From c19bb69a889547c3a9e71475ec4f20375f59cffb Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Fri, 16 Oct 2026 16:21:50 -0500 Subject: [PATCH 01/57] An LDAP search that does not end with success is refused as an outage, not read as no entry (BACKLOG #3010) _search now refuses every result code but success through a named helper, _refuse_unanswered, which raises LdapUnansweredError (an LdapError). The reconciler reads it as UNAVAILABLE, a sign-in as auth.login_error. This covers the user search, the nested group search and the bind-account read. Row item 4 is deferred to vault BACKLOG #3100 item 3. Proposed PR title: An LDAP search that does not end with success is an outage, not "no such account" (BACKLOG #3010) Proposed vault banner: BUILT on branch b257-3010. _search in auth/ldap.py refuses any result code but success, with or without entries, through _refuse_unanswered; the reconciler reads it as unavailable and sign-in as a login error. Closing items 1-3 built and tested; item 4 deferred to #3100 item 3. --- changelog.d/3010.security.md | 8 + messagefoundry/auth/ldap.py | 116 ++++++- tests/test_ldap_search_result_codes.py | 385 ++++++++++++++++++++++ tests/test_vault_ldap_privilege_probes.py | 3 +- 4 files changed, 499 insertions(+), 13 deletions(-) create mode 100644 changelog.d/3010.security.md create mode 100644 tests/test_ldap_search_result_codes.py diff --git a/changelog.d/3010.security.md b/changelog.d/3010.security.md new file mode 100644 index 0000000000..996a4407d1 --- /dev/null +++ b/changelog.d/3010.security.md @@ -0,0 +1,8 @@ +- **A directory search that does not end with success is an outage, never "no such account".** + Every LDAP search now refuses any result code but success, with or without entries: busy, + unavailable, unwillingToPerform, adminLimitExceeded, a time or size limit, and noSuchObject on + a missing search base. Before, such a search read as no entries, so the session reconciler would + have struck a present account toward revocation, and a busy nested group search read as fewer + groups. The reconciler now reads these as unavailable, which never revokes, and a sign-in writes + `auth.login_error` naming the code. `check-privileges` names the code for the bind-account read. + No directory diagnostic text reaches the error, the log or the audit row. (vault `BACKLOG #3010`) diff --git a/messagefoundry/auth/ldap.py b/messagefoundry/auth/ldap.py index 76ac90d328..93f2255d4f 100644 --- a/messagefoundry/auth/ldap.py +++ b/messagefoundry/auth/ldap.py @@ -16,6 +16,8 @@ local-only deployment never touches them. **No referral is followed, and one is refused** (BACKLOG #2530); :func:`_refuse_referral` says why. +**A search that ends with any result code but success is refused too** (BACKLOG #3010), never read +as no entries; :func:`_refuse_unanswered` says why. """ from __future__ import annotations @@ -116,6 +118,22 @@ class LdapReferralError(LdapError): """ +class LdapUnansweredError(LdapError): + """The directory ended an operation with a result code that is not an answer (BACKLOG #3010). + + A subclass, so every ``except LdapError`` reads it as the directory not answering: the session + reconciler as unavailable, which never strikes, and a sign-in as ``auth.login_error``. It + carries the code so a caller can tell codes apart without parsing the text. ``code`` is the result code, or ``None`` + when the result held none that is an integer; ``code_name`` is ldap3's name for it, never text + the directory sent. + """ + + def __init__(self, message: str, *, code: int | None, code_name: str) -> None: + super().__init__(message) + self.code = code + self.code_name = code_name + + class DirectoryAnswer(Enum): """What one password-free lookup of one account established (ADR 0195 rule item 2). @@ -128,8 +146,10 @@ class DirectoryAnswer(Enum): #: The entry was found and ``userAccountControl`` proved it enabled. FOUND = "found" - #: The search matched nothing, or an id-keyed probe held an id the filter builder cannot parse, - #: so no search ran, or an id-keyed search's entry did not read back that id (BACKLOG #2027). + #: The search ended with success and matched nothing, or an id-keyed probe held an id the + #: filter builder cannot parse, so no search ran, or an id-keyed search's entry did not read + #: back that id (BACKLOG #2027). A search that ended with any other result code raises + #: :class:`LdapUnansweredError` instead (BACKLOG #3010). NOT_FOUND = "not_found" #: The entry was found and ``userAccountControl`` read, with ACCOUNTDISABLE (0x2) set. DISABLED = "disabled" @@ -738,11 +758,82 @@ def _refuse_referral(conn: Any, operation: str) -> None: raise LdapReferralError(message) +#: What a refused code most likely means, where the code points at the configuration rather than at +#: a passing load. Engine text only. Each names what the operation was aimed at, which for a search +#: is its configured base and for the bind-account read is ``[auth].ad_bind_dn``. +_UNANSWERED_HINTS: dict[int, str] = { + 32: " noSuchObject usually means the configured base or DN does not exist, or the service " + "account cannot see it.", + 34: " invalidDNSyntax usually means a configured DN is not a distinguished name, such as a UPN " + "or DOMAIN\\user.", + 50: " insufficientAccessRights means the service account may not read it.", +} + + +def _refuse_unanswered(conn: Any, operation: str, *, answers: frozenset[int] | None = None) -> None: + """Raise :class:`LdapUnansweredError` unless ``conn``'s last operation ended with a result code + in ``answers``, which is success alone unless the caller names another set. + + BACKLOG #3010. ldap3 raises on none of these codes here. Every ``Connection`` in this module + keeps ldap3's default ``raise_exceptions=False``, and even under ``True`` ldap3 2.9.1 passes + sizeLimitExceeded (4) and timeLimitExceeded (3) through (``DO_NOT_RAISE_EXCEPTIONS``). So a + search the directory did not finish -- busy (51), unavailable (52), unwillingToPerform (53), + adminLimitExceeded (11), a time or size limit -- came back with no entries, or some. Read on, the + user search answered ``NOT_FOUND``, which the session reconciler counts toward revoking the + account, and the nested group search answered fewer groups, which narrows the account's roles. + + **Which code is an answer.** For a search, success alone. A search that ended with success and + matched nothing is the one result that means "no such entry", so only it reads ``NOT_FOUND``, + and offboarding still revokes. Every other code says the directory did not answer the question + asked, so any entries that came with it are not the whole answer and none is used. That + includes noSuchObject (32): on these searches it names the search BASE, not the account, so + read as absence it would read every account absent at once, which the reconciler's mass-revoke + breaker does not stop at or under its floor. + + **The cost, named.** The reconciler's unavailable never revokes, so a code that recurs on every + pass, such as a deleted search base, stops offboarding for as long as it lasts. Making a + standing one visible is vault BACKLOG #3100's item 3; #3010's item 4 was deferred to it. This + warns once per operation and code, and the text carries no directory text: the code, ldap3's + name for it and the operation, never the diagnostic message the directory sent. + + **A result that is not a dict is let through.** ldap3's synchronous strategy, the only one built + here, reads ``self.result['type']`` straight after every search, so a real search cannot return + without one. Only a test double can. + + ``answers`` is a parameter so an operation whose answers differ, such as a bind, can reuse this + with its own set. Run :func:`_refuse_referral` first: a referral is not success either, and only + that refusal reads as referred. + """ + from ldap3.core.results import RESULT_CODES, RESULT_SUCCESS # lazy, like every ldap3 import + + result = conn.result + if not isinstance(result, dict): + return + code = result.get("result") + if type(code) is not int: # not isinstance: a bool is an int, and is no result code + code, name, shown, hint = None, "no code", "no readable result code", "" + elif code in (answers if answers is not None else (RESULT_SUCCESS,)): + return + else: + name = RESULT_CODES.get(code, "an unknown result code") + shown, hint = f"result code {code} ({name})", _UNANSWERED_HINTS.get(code, "") + message = ( + f"AD ended the {operation} with {shown}, which is not an answer; the engine reads it as " + f"the directory not answering (BACKLOG #3010).{hint}" + ) + # Keyed on ldap3's name, not the raw code: a directory chooses the code, and ldap3's table is + # finite, so the set of warnings stays bounded. + _warn_once(f"unanswered: {operation}: {name}", "%s Reported once per code.", message) + raise LdapUnansweredError(message, code=code, code_name=name) + + def _search(conn: Any, operation: str, **kwargs: Any) -> None: - """``conn.search(**kwargs)``, refusing a referral result. Every search in this module goes - through here, so none can read one as "no entries"; a test pins that.""" + """``conn.search(**kwargs)``, refusing a referral result and then any result but success + (:func:`_refuse_unanswered`). Every search in this module goes through here, so none can read + either as "no entries"; a test pins that.""" conn.search(**kwargs) _refuse_referral(conn, operation) + _refuse_unanswered(conn, operation) #: What opening an ldap3 socket can raise that is not an ldap3 error (BACKLOG #2566). ldap3's @@ -1209,6 +1300,8 @@ def _resolve_groups(self, conn: Any, user_dn: str, member_of: list[str]) -> froz # Direct membership from the user's memberOf attribute. found = list(member_of) if self._s.ad_use_nested_groups and self._s.ad_group_search_base: + # A search the directory did not finish raises here (BACKLOG #3010), so a busy answer + # is an outage to every caller, never a shorter group list that narrows the roles. _search( conn, "group search", @@ -1423,27 +1516,26 @@ def read_bind_account(self) -> BindAccountReading: attributes=["tokenGroups", "memberOf", "primaryGroupID"], ) except (ldap3.core.exceptions.LDAPException, LdapError) as exc: - # An LdapError is a refused referral, whose text is the engine's own. + # An LdapError is a refused referral, or a read that ended with a result code that + # is not success (BACKLOG #3010): noSuchObject, invalidDNSyntax for a bind identity + # that is a UPN rather than a DN, insufficientAccessRights. Both texts are the + # engine's own, naming the code and never the directory's diagnostic message. why = str(exc) if isinstance(exc, LdapError) else type(exc).__name__ return BindAccountReading( who, problem=f"group membership not read: {why}", whoami_error=whoami_error ) entry = svc.entries[0] if svc.entries else None - result = svc.result if isinstance(svc.result, dict) else {} except ldap3.core.exceptions.LDAPException as exc: raise LdapError(str(exc)) from exc finally: with contextlib.suppress(ldap3.core.exceptions.LDAPException): svc.unbind() if entry is None: - # ldap3 does not raise on a failed search here, so the result code is the only record of - # why: invalidDNSyntax for a bind identity that is a UPN or DOMAIN\\user rather than a - # DN, insufficientAccessRights for an entry the account may not read. - why = result.get("description") or "no result code" + # A base read that ended with success and no entry: no directory should answer so. return BindAccountReading( who, - problem=f"group membership not read: the base read of [auth].ad_bind_dn returned " - f"no entry ({why}); the read needs a distinguished name the account may read", + problem="group membership not read: the base read of [auth].ad_bind_dn returned " + "no entry; the read needs a distinguished name the account may read", whoami_error=whoami_error, ) raw = entry["tokenGroups"].raw_values if "tokenGroups" in entry else () diff --git a/tests/test_ldap_search_result_codes.py b/tests/test_ldap_search_result_codes.py new file mode 100644 index 0000000000..643e966b76 --- /dev/null +++ b/tests/test_ldap_search_result_codes.py @@ -0,0 +1,385 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +# Copyright (C) 2026 MessageFoundry Foundation, LLC and contributors +"""BACKLOG #3010: a search that ends with any result code but success is the directory not +answering, never "no such entry". + +ldap3 raises on none of these codes here (every connection keeps ``raise_exceptions=False``), so a +busy or admin-limit-exceeded search came back with no entries. ``_search_user`` read that as +``NOT_FOUND``, the session reconciler read ``NOT_FOUND`` as ``ABSENT``, and after the strike +threshold it revoked a present account. The nested group search read the same answer as fewer +groups. ``_search`` now refuses every code but success with :class:`LdapUnansweredError`, which +every caller already reads as an outage. + +Driven at three depths: ``_search`` itself over a bare double; the REAL ``LdapAuthenticator`` over +the ldap3 doubles of ``tests/test_ad_user_account_control.py`` and ``tests/test_ldap_timeouts.py``; +and the REAL ``AuthService`` reconciler and Kerberos sign-in over those. Each refusal has a control +on the same doubles: success with no entries still reads absent, and still revokes. + +**Not exercised:** a real domain controller. The doubles set the result dict the way ldap3 2.9.1's +synchronous strategy leaves it. Synthetic data only; nothing here touches PHI. +""" + +from __future__ import annotations + +import logging +from collections.abc import Callable +from types import SimpleNamespace +from typing import Any + +import ldap3 +import pytest +from ldap3.core.results import ( + RESULT_ADMIN_LIMIT_EXCEEDED, + RESULT_BUSY, + RESULT_INSUFFICIENT_ACCESS_RIGHTS, + RESULT_INVALID_DN_SYNTAX, + RESULT_NO_SUCH_OBJECT, + RESULT_OPERATIONS_ERROR, + RESULT_REFERRAL, + RESULT_SIZE_LIMIT_EXCEEDED, + RESULT_SUCCESS, + RESULT_TIME_LIMIT_EXCEEDED, + RESULT_UNAVAILABLE, + RESULT_UNWILLING_TO_PERFORM, +) + +from messagefoundry.auth import ldap as ldap_module +from messagefoundry.auth.ldap import ( + DirectoryAnswer, + LdapAuthenticator, + LdapError, + LdapReferralError, + LdapUnansweredError, +) +from messagefoundry.auth.reconcile import ProbeOutcome +from messagefoundry.auth.service import AuthService +from messagefoundry.store.store import MessageStore +from tests.test_ad_user_account_control import _alive, _audited, _pass, _signed_in_estate +from tests.test_ldap_timeouts import _ad_settings, _install_fakes + +#: Stands for text the directory sends with a result: its diagnostic message and matched DN. It +#: must never reach an error, a log line or an audit row. +_LEAK = "CN=synthetic-leak-3010,OU=Staff,DC=test,DC=invalid" + +_GROUPS = "OU=Groups,DC=test,DC=invalid" + +#: The codes the row names, and others a directory can end a search with. None is an answer. +_NOT_ANSWERS = [ + pytest.param(RESULT_BUSY, "busy", id="51-busy"), + pytest.param(RESULT_ADMIN_LIMIT_EXCEEDED, "adminLimitExceeded", id="11-adminLimitExceeded"), + pytest.param(RESULT_TIME_LIMIT_EXCEEDED, "timeLimitExceeded", id="3-timeLimitExceeded"), + pytest.param(RESULT_SIZE_LIMIT_EXCEEDED, "sizeLimitExceeded", id="4-sizeLimitExceeded"), + pytest.param(RESULT_UNAVAILABLE, "unavailable", id="52-unavailable"), + pytest.param(RESULT_UNWILLING_TO_PERFORM, "unwillingToPerform", id="53-unwillingToPerform"), + pytest.param(RESULT_NO_SUCH_OBJECT, "noSuchObject", id="32-noSuchObject"), + pytest.param(RESULT_OPERATIONS_ERROR, "operationsError", id="1-operationsError"), + pytest.param(4242, "an unknown result code", id="4242-unknown"), +] + + +@pytest.fixture(autouse=True) +def _fresh_warnings(monkeypatch: pytest.MonkeyPatch) -> None: + """Each test sees its own once-per-process warnings.""" + monkeypatch.setattr(ldap_module, "_object_guid_shapes_warned", set()) + + +def _result(code: object) -> dict[str, Any]: + """A searchResDone as ldap3 leaves it in ``Connection.result``, with directory text attached.""" + return { + "result": code, + "description": "x", + "message": _LEAK, + "dn": _LEAK, + "type": "searchResDone", + } + + +def _conn(result: object, entries: list[object] | None = None) -> SimpleNamespace: + def search(**_kw: object) -> bool: + return False + + return SimpleNamespace(search=search, result=result, entries=entries or []) + + +def _answer_with( + monkeypatch: pytest.MonkeyPatch, + code: int, + *, + when: Callable[[dict[str, Any]], bool], + keep_entries: bool = False, +) -> None: + """Over the ldap3 double already installed, end every search ``when`` picks with ``code``. + + The entries are dropped, as a refused search leaves them, unless ``keep_entries``: a size or + admin limit can arrive with part of the answer.""" + base = ldap3.Connection + + class Answering(base): # type: ignore[misc, valid-type] + def search(self, **kwargs: Any) -> bool: + super().search(**kwargs) + if when(kwargs): + self.result = _result(code) + if not keep_entries: + dropped: list[Any] = [] + self.entries = dropped + return False + return True + + monkeypatch.setattr(ldap3, "Connection", Answering) + + +def _user_search(kwargs: dict[str, Any]) -> bool: + return "member:" not in str(kwargs["search_filter"]) + + +def _group_search(kwargs: dict[str, Any]) -> bool: + return "member:" in str(kwargs["search_filter"]) + + +# --- _search: which code is an answer -------------------------------------------------------------- + + +@pytest.mark.parametrize(("code", "name"), _NOT_ANSWERS) +@pytest.mark.parametrize("partial", [False, True], ids=["no-entries", "partial-entries"]) +def test_a_search_that_did_not_end_with_success_is_refused( + code: int, name: str, partial: bool +) -> None: + """Refused whether or not entries came with it: a limit that cut the answer short leaves some, + and they are not the whole answer.""" + conn = _conn(_result(code), entries=[object()] if partial else None) + with pytest.raises(LdapUnansweredError) as refused: + ldap_module._search(conn, "user search", search_base="OU=Staff") + assert refused.value.code == code and refused.value.code_name == name + message = str(refused.value) + assert f"user search with result code {code} ({name})" in message + assert "which is not an answer" in message + assert _LEAK not in message + # Every caller reads LdapError as an outage; a referral keeps its own subclass. + assert isinstance(refused.value, LdapError) + assert not isinstance(refused.value, LdapReferralError) + + +def test_success_is_the_only_answer_and_reads_on() -> None: + """THE CONTROL: the same double ending with success raises nothing, with or without entries.""" + ldap_module._search(_conn(_result(RESULT_SUCCESS)), "user search") + ldap_module._search(_conn(_result(RESULT_SUCCESS), [object()]), "user search") + + +def test_a_caller_can_name_its_own_answers() -> None: + """The set is a parameter, for an operation such as a bind whose answers differ.""" + invalid_credentials = 49 + conn = _conn(_result(invalid_credentials)) + ldap_module._refuse_unanswered(conn, "user bind", answers=frozenset({0, invalid_credentials})) + with pytest.raises(LdapUnansweredError): + ldap_module._refuse_unanswered(conn, "user bind") + + +def test_a_referral_is_still_refused_as_a_referral() -> None: + """The referral check runs first, so the reconciler still reads it as referred (#2538).""" + conn = _conn({"result": RESULT_REFERRAL, "referrals": ["ldap://dc1.other.test.invalid/"]}) + with pytest.raises(LdapReferralError): + ldap_module._search(conn, "user search") + + +@pytest.mark.parametrize("raw", [None, "0", True, False, 0.0], ids=repr) +def test_a_result_code_that_is_not_an_integer_is_refused(raw: object) -> None: + """``False == 0``, so a bool must not pass as success; nor a string or a float.""" + with pytest.raises(LdapUnansweredError, match="with no readable result code") as refused: + ldap_module._search(_conn(_result(raw)), "group search") + assert refused.value.code is None + + +def test_a_result_that_is_not_a_dict_is_let_through() -> None: + """ldap3's synchronous strategy cannot return from a search without one; only a double can.""" + ldap_module._search(_conn(None), "user search") + + +def test_no_such_object_names_the_search_base() -> None: + with pytest.raises(LdapUnansweredError, match="configured base or DN does not exist"): + ldap_module._search(_conn(_result(RESULT_NO_SUCH_OBJECT)), "user search") + with pytest.raises(LdapUnansweredError) as busy: + ldap_module._search(_conn(_result(RESULT_BUSY)), "user search") + assert "base" not in str(busy.value) + + +def test_the_refusal_warns_once_per_code_and_carries_no_directory_text( + caplog: pytest.LogCaptureFixture, +) -> None: + with caplog.at_level(logging.WARNING, logger="messagefoundry.auth.ldap"): + for code in (RESULT_BUSY, RESULT_BUSY, RESULT_UNAVAILABLE, 4242, 4243): + with pytest.raises(LdapUnansweredError): + ldap_module._search(_conn(_result(code)), "user search") + lines = [r.getMessage() for r in caplog.records] + # Two unknown codes share ldap3's one name, so a directory cannot grow the warned set. + assert len(lines) == 3, lines + assert all(_LEAK not in line for line in lines) + + +# --- the real authenticator: the user search and the nested group search ------------------------- + + +@pytest.mark.parametrize(("code", "name"), _NOT_ANSWERS) +def test_a_user_search_the_directory_did_not_answer_is_an_outage_not_an_absence( + monkeypatch: pytest.MonkeyPatch, code: int, name: str +) -> None: + """Closing items 1 and 2. Before the fix, ``probe_principal`` answered ``NOT_FOUND`` here, + which the reconciler strikes, and the Kerberos lookup answered ``None``, an unknown user.""" + _install_fakes(monkeypatch) + _answer_with(monkeypatch, code, when=_user_search) + authenticator = LdapAuthenticator(_ad_settings()) + with pytest.raises(LdapUnansweredError, match="user search"): + authenticator.probe_principal("alice") + with pytest.raises(LdapUnansweredError): + authenticator.resolve_principal("alice") + with pytest.raises(LdapUnansweredError): + authenticator.authenticate("alice", "synthetic-pw-3010") + + +def test_success_with_no_entries_still_reads_not_found(monkeypatch: pytest.MonkeyPatch) -> None: + """Closing item 3, THE CONTROL: the one result that means "no such entry" still says so.""" + _install_fakes(monkeypatch) + _answer_with(monkeypatch, RESULT_SUCCESS, when=_user_search) + probe = LdapAuthenticator(_ad_settings()).probe_principal("alice") + assert probe.answer is DirectoryAnswer.NOT_FOUND + + +@pytest.mark.parametrize("partial", [False, True], ids=["no-entries", "partial-entries"]) +def test_a_group_search_the_directory_did_not_finish_is_an_outage_not_fewer_groups( + monkeypatch: pytest.MonkeyPatch, partial: bool +) -> None: + """A busy or limited nested group search used to read as fewer groups, which narrows the + account's roles. It raises on every path that resolves groups.""" + _install_fakes(monkeypatch) + code = RESULT_SIZE_LIMIT_EXCEEDED if partial else RESULT_BUSY + _answer_with( + monkeypatch, + code, + when=_group_search, + keep_entries=partial, + ) + authenticator = LdapAuthenticator(_ad_settings()) + with pytest.raises(LdapUnansweredError, match="group search"): + authenticator.probe_principal("alice") + with pytest.raises(LdapUnansweredError, match="group search"): + authenticator.authenticate("alice", "synthetic-pw-3010") + + +# --- the reconciler: no strike, no revocation ------------------------------------------------------ + + +@pytest.mark.parametrize( + ("estate", "when", "code", "partial"), + [ + ({}, _user_search, RESULT_BUSY, False), + ({}, _user_search, RESULT_ADMIN_LIMIT_EXCEEDED, False), + ({}, _user_search, RESULT_NO_SUCH_OBJECT, False), + ({}, _user_search, RESULT_SIZE_LIMIT_EXCEEDED, True), + ({"ad_group_search_base": _GROUPS}, _group_search, RESULT_BUSY, False), + ], + ids=["busy", "adminLimitExceeded", "noSuchObject", "sizeLimit-partial", "group-search-busy"], +) +async def test_a_directory_that_did_not_answer_never_revokes_a_present_account( + monkeypatch: pytest.MonkeyPatch, + estate: dict[str, Any], + when: Callable[[dict[str, Any]], bool], + code: int, + partial: bool, +) -> None: + """The row's defect, end to end. One signed-in account, under the breaker's floor, whose search + the directory answers with ``code`` on every pass. Before the fix the user search read ABSENT + and was revoked at the strike threshold, and the nested group search read as fewer groups, + which feeds the role re-diff. Now each probe is unavailable and the session survives.""" + async with _signed_in_estate(monkeypatch, ["jdoe"], **estate) as ( + _directory, + service, + store, + tokens, + ): + _answer_with(monkeypatch, code, when=when, keep_entries=partial) + user = await store.get_user_by_username("jdoe") + assert user is not None + assert (await service._probe_principal(user)).outcome is ProbeOutcome.UNAVAILABLE + for _ in range(5): # far past ad_session_recheck_strikes + assert (await _pass(service)).revocations == () + assert await _alive(service, tokens) == {"jdoe"} + assert await _audited(store, "auth.ad_session_revoked") == [] + + +async def test_an_account_that_left_the_directory_is_still_revoked( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """THE CONTROL for the test above, on the same doubles: success with no entries strikes and + revokes, so the fix did not stop offboarding.""" + async with _signed_in_estate(monkeypatch, ["jdoe"]) as (directory, service, store, tokens): + directory.delete("jdoe") + user = await store.get_user_by_username("jdoe") + assert user is not None + assert (await service._probe_principal(user)).outcome is ProbeOutcome.ABSENT + revoked = [r.username for _ in range(5) for r in (await _pass(service)).revocations] + assert revoked == ["jdoe"] + assert await _alive(service, tokens) == set() + + +# --- sign-in: a directory error, not an unknown user --------------------------------------------- + + +async def test_a_busy_kerberos_sign_in_is_audited_as_a_login_error( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Closing item 2. The row names the code and never the directory's text.""" + + async def no_sleep(_deadline: float) -> None: + return None + + monkeypatch.setattr("messagefoundry.auth.service._sleep_until", no_sleep) + monkeypatch.setattr("messagefoundry.auth.service.kerberos_principal", lambda _t, _s: "alice") + _install_fakes(monkeypatch) + _answer_with(monkeypatch, RESULT_BUSY, when=lambda _kw: True) + settings = _ad_settings(kerberos_enabled=True) + store = await MessageStore.open(":memory:") + try: + service = AuthService(store, settings, ldap=LdapAuthenticator(settings)) + await service.initialize() + out = await service.authenticate_kerberos(b"spnego-token") + + assert not out.ok + rows = [str(dict(r)["detail"]) for r in await store.list_audit(action="auth.login_error")] + assert len(rows) == 1 and "result code 51 (busy)" in rows[0], rows + assert _LEAK not in rows[0] + assert await store.list_audit(action="auth.login_success") == [] + finally: + await store.close() + + +# --- check-privileges: the bind-account read names the code --------------------------------------- + + +@pytest.mark.parametrize( + ("code", "name", "hint"), + [ + (RESULT_NO_SUCH_OBJECT, "noSuchObject", "does not exist"), + (RESULT_INVALID_DN_SYNTAX, "invalidDNSyntax", "is not a distinguished name"), + (RESULT_INSUFFICIENT_ACCESS_RIGHTS, "insufficientAccessRights", "may not read it"), + (RESULT_BUSY, "busy", None), + ], +) +def test_the_bind_account_read_names_the_code_and_keeps_the_identity( + monkeypatch: pytest.MonkeyPatch, code: int, name: str, hint: str | None +) -> None: + """A configuration code carries its hint; busy carries none, so check-privileges does not send + an operator to fix ``[auth].ad_bind_dn`` when the directory was only loaded.""" + conn = _conn(_result(code)) + conn.extend = SimpleNamespace(standard=SimpleNamespace(who_am_i=lambda: "u:EXAMPLE\\svc")) + conn.unbind = lambda: True + authenticator = LdapAuthenticator(_ad_settings()) + monkeypatch.setattr(authenticator, "_service_conn", lambda: conn) + reading = authenticator.read_bind_account() + assert reading.authzid == "u:EXAMPLE\\svc" + problem = reading.problem + assert problem is not None and problem.startswith("group membership not read: ") + assert f"bind-account group read with result code {code} ({name})" in problem + assert _LEAK not in problem + if hint is None: + assert "distinguished name" not in problem and "may not read" not in problem + else: + assert hint in problem diff --git a/tests/test_vault_ldap_privilege_probes.py b/tests/test_vault_ldap_privilege_probes.py index 20f6839c6e..67c43fcb62 100644 --- a/tests/test_vault_ldap_privilege_probes.py +++ b/tests/test_vault_ldap_privilege_probes.py @@ -870,7 +870,8 @@ def test_a_missing_entry_keeps_the_identity_and_is_unobservable( ) -> None: reading = _read(monkeypatch, _FakeLdapConn(None)) assert reading.authzid == "u:EXAMPLE\\svc" - assert reading.problem is not None and "returned no entry (noSuchObject)" in reading.problem + # BACKLOG #3010: a read that did not end with success is refused naming ldap3's code name. + assert reading.problem is not None and "result code 32 (noSuchObject)" in reading.problem assert _judge(reading).state is HopState.UNOBSERVABLE From 6f2f4f5081d6f6fa503f5b0c0c83b2d6900ccdef Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Fri, 16 Oct 2026 16:32:19 -0500 Subject: [PATCH 02/57] The start-up credential probe runs off the event loop (BACKLOG #3164) The api/app.py lifespan called AuthService.probe_credential_generation synchronously, so the password generator's policy screens ran on the loop thread at start. A pathological password_extra_context_words list makes the generator retry many times. The lifespan now awaits asyncio.to_thread(auth.probe_credential_generation), the same shape as the Kerberos preflight a few lines below, and keeps its except guard. The method stays synchronous, so the existing synchronous test in tests/test_generated_credential_failure.py is untouched, and the verify CLI caller, which has no loop, is untouched too. tests/test_credential_probe_off_loop.py drives the real lifespan and records where the generator's screen ran; it fails on the old code. A second test pins that the guard still logs a raise and the start continues. --- messagefoundry/api/app.py | 6 +- tests/test_credential_probe_off_loop.py | 95 +++++++++++++++++++++++++ 2 files changed, 99 insertions(+), 2 deletions(-) create mode 100644 tests/test_credential_probe_off_loop.py diff --git a/messagefoundry/api/app.py b/messagefoundry/api/app.py index 052fb355f5..3fe2b485f8 100644 --- a/messagefoundry/api/app.py +++ b/messagefoundry/api/app.py @@ -10489,9 +10489,11 @@ async def _audit_upload_prune(meta: UploadedFileMeta) -> None: # ADR 0197 Amendment A, Manager decision 2026-09-29: learn at start, not at the first # account creation, that the password policy leaves no credential to issue. It logs # an ERROR and returns, and is guarded like the census: a policy-level fact must - # never refuse the start, whatever a later policy clause raises. + # never refuse the start, whatever a later policy clause raises. It runs off the loop + # (vault BACKLOG #3164): a pathological context-word list makes the generator retry + # its policy screens many times, and the loop already serves the API by now. try: - auth.probe_credential_generation() + await asyncio.to_thread(auth.probe_credential_generation) except Exception: # noqa: BLE001 -- see the comment above _log.exception( "the credential-generation probe could not run; startup continues" diff --git a/tests/test_credential_probe_off_loop.py b/tests/test_credential_probe_off_loop.py new file mode 100644 index 0000000000..fc008e4a93 --- /dev/null +++ b/tests/test_credential_probe_off_loop.py @@ -0,0 +1,95 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +# Copyright (C) 2026 MessageFoundry Foundation, LLC and contributors +"""The start-up credential-generation probe runs off the event loop (vault BACKLOG #3164). + +``AuthService.probe_credential_generation`` runs the temporary-credential generator once at start. +A pathological ``[auth].password_extra_context_words`` list makes that generator retry its policy +screens many times, so the lifespan must not run it on the loop thread, the way the request-path +callers already do not (BACKLOG #2359). These drive the real lifespan and record where the +generator's screen ran. + +Severity is conditional (CLAUDE.md section 0): zero deployments, so a held loop is what a deploying +site would meet at start, never a live exposure. +""" + +from __future__ import annotations + +import asyncio +import logging +from pathlib import Path + +import pytest +from fastapi import FastAPI + +from messagefoundry.api import create_managed_app +from messagefoundry.auth import service as service_module +from messagefoundry.auth.policy import PasswordPolicy +from messagefoundry.config.settings import ( + AlertsSettings, + AuthSettings, + EgressSettings, + SecurityEnforcement, + SecuritySettings, + StoreSettings, +) +from messagefoundry.store.crypto import generate_key + + +def _app(db: Path) -> FastAPI: + """An engine that starts on an empty store: ``warn`` lets the ADR 0167 gate pass with no + Administrator, so the lifespan reaches the probe.""" + return create_managed_app( + store_settings=StoreSettings(path=str(db), encryption_key=generate_key()), + poll_interval=0.05, + auth_settings=AuthSettings(), + alerts_settings=AlertsSettings( + email_smtp_host="smtp.example.invalid", email_from="alerts@example.invalid" + ), + security_settings=SecuritySettings(enforcement=SecurityEnforcement.WARN), + egress_settings=EgressSettings(deny_by_default=False), + ) + + +async def _start(app: FastAPI) -> None: + async with app.router.lifespan_context(app): + pass + + +def test_the_lifespan_runs_the_generator_off_the_loop_thread( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + on_loop: list[bool] = [] + + def recording(policy: PasswordPolicy) -> str | None: + # A worker thread has no running loop; the loop thread does. + try: + asyncio.get_running_loop() + except RuntimeError: + on_loop.append(False) + else: + on_loop.append(True) + return None + + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(service_module, "credential_generation_problem", recording) + asyncio.run(_start(_app(tmp_path / "probe.db"))) + # The count is the control: an empty list would pass the assertion below vacuously. + assert on_loop == [False], "the probe ran once, and not on the event loop thread" + + +def test_a_probe_that_raises_off_the_loop_still_lets_the_start_continue( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + """The guard still wraps the call now that it is awaited: a raise in the worker thread comes + back through ``asyncio.to_thread`` and is logged, and the start does not refuse.""" + + def broken(policy: PasswordPolicy) -> str | None: + raise ValueError("a later policy clause raised") + + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(service_module, "credential_generation_problem", broken) + with caplog.at_level(logging.ERROR, logger="messagefoundry.api.app"): + asyncio.run(_start(_app(tmp_path / "probe.db"))) # must not raise + assert any( + "the credential-generation probe could not run" in r.getMessage() for r in caplog.records + ) From 251080a9f1bac5bb80b671fa5bbf5e7a13d78b3a Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Fri, 16 Oct 2026 16:32:52 -0500 Subject: [PATCH 03/57] Directory role and disable notices name the directory, not an administrator (BACKLOG #3121) The directory reconciler's ACCOUNT_DISABLED and ROLES_CHANGED notices said an administrator made the change and closed with "If this was you". _apply_reconcile_revocation now tags its notice detail with source "directory", and so does the sign-in role re-sync in _complete_ad_login, so the two directory ROLES_CHANGED notices read alike. In pipeline/security_notify.py the source tag alone now decides from_directory, for any kind, so a new directory notice cannot fall back to the administrator wording by missing from a list. ROLES_CHANGED and ACCOUNT_DISABLED take directory descriptions that name no administrator, gain the directory line, and close with "If you did not expect this change". The administrator path keeps its wording, as the row says; that wording is not judged correct (BACKLOG #1141, vault #3072 item 10). Tests pin the descriptions whole, not only words the subject would satisfy: the directory variants, the unchanged administrator variants, the sign-in address line on the re-sync notice, and the existing directory_sessions_ended description. The reconciler and login-path tests assert the source tag. docs/SECURITY.md says how directory notices are worded. --- docs/SECURITY.md | 11 ++- messagefoundry/auth/notifications.py | 8 ++- messagefoundry/auth/service.py | 16 +++-- messagefoundry/pipeline/security_notify.py | 29 ++++++-- tests/test_ad_session_reconcile.py | 8 ++- tests/test_auth_service.py | 4 ++ tests/test_security_notify.py | 84 +++++++++++++++++++++- 7 files changed, 141 insertions(+), 19 deletions(-) diff --git a/docs/SECURITY.md b/docs/SECURITY.md index d0a9314cc6..089cc02aa2 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -4384,9 +4384,14 @@ Users are notified of security-relevant changes to their account through **two** alert distribution list). Fired on: account **lockout** and the **first successful login after ≥3 failed attempts**, or a step-up re-auth that clears such a run (suspicious-login signals, 6.3.5); and **password change**, **email change**, **role change**, **account disable**, and a **directory rename** of the username (credential changes, - 6.3.7). When the directory session reconciler ends an account's sessions because it could not - match the account or could not read its `userAccountControl`, it sends `directory_sessions_ended`, - not `account_disabled` (vault BACKLOG #2140). + 6.3.7). + A notice for a change the directory made names the directory and no administrator, and it closes + with "If you did not expect this change" rather than "If this was you". That covers at least the + directory rename, the directory's email change, a role change applied by the session reconciler + or by a directory sign-in's re-sync, and the reconciler's `account_disabled` when it read the + account disabled (vault BACKLOG #3121). When the reconciler ends an account's sessions because it + could not match the account or could not read its `userAccountControl`, it sends + `directory_sessions_ended`, not `account_disabled` (vault BACKLOG #2140). The rename notice, `username_changed`, names the old and the new name, and is sent only when the new name was written (BACKLOG #2017). An **unreplaced temporary password** near its deadline (ASVS 6.4.5, BACKLOG #2007) sends two reminders, beside the operator's `initial_credential_expiring` diff --git a/messagefoundry/auth/notifications.py b/messagefoundry/auth/notifications.py index a772bc1381..9d108998b4 100644 --- a/messagefoundry/auth/notifications.py +++ b/messagefoundry/auth/notifications.py @@ -50,7 +50,9 @@ TEMPORARY_CREDENTIAL_EXPIRING = "temporary_credential_expiring" TEMPORARY_CREDENTIAL_EXPIRING_FOR_ISSUER = "temporary_credential_expiring_issuer" EMAIL_CHANGED = "email_changed" # 6.3.7 — the account's email address was changed -ROLES_CHANGED = "roles_changed" # 6.3.7 — an admin changed the account's roles +# 6.3.7 -- the account's roles changed: an administrator's act, or the directory's, which the +# reconciler and the sign-in re-sync tag with ``detail["source"] == "directory"`` (vault BACKLOG #3121). +ROLES_CHANGED = "roles_changed" # 6.3.7 -- the directory renamed the account and the engine copied the new name onto the row # (BACKLOG #2017). When it is sent is stated once, at the send site in # ``AuthService._refresh_cached_username``. ``detail`` carries ``old_username``, ``new_username`` and @@ -67,7 +69,9 @@ # address the account held BEFORE the takeover, because a moved address belongs to the operator who # ran the command, not to the holder being told. FIRST_ADMINISTRATOR_TAKEOVER = "first_administrator_takeover" -ACCOUNT_DISABLED = "account_disabled" # 6.3.7 — an admin disabled the account +# 6.3.7 -- an administrator disabled the account, or the reconciler read it disabled in the +# directory, tagged ``detail["source"] == "directory"`` (vault BACKLOG #3121). +ACCOUNT_DISABLED = "account_disabled" # 6.3.7 -- the directory session reconciler ended the account's sessions for a reason that is NOT a # read disabled bit: the probe could not match the account (``directory_absent``: not found, or its # stored id did not match or could not be parsed), or its ``userAccountControl`` could not be read diff --git a/messagefoundry/auth/service.py b/messagefoundry/auth/service.py index da69512cfb..c05f6b0c9e 100644 --- a/messagefoundry/auth/service.py +++ b/messagefoundry/auth/service.py @@ -5448,12 +5448,14 @@ async def _complete_ad_login( # A directory-pushed privilege change is the same privilege change to the same user as a # local one, so notify the affected user out-of-band too (ASVS 6.3.7), matching set_roles(). # Best-effort; the change is also visible in the audited /me/security-events feed. + # ``source`` words it for the directory, as the reconciler's notice is (vault BACKLOG + # #3121): no administrator made this change, so "if this was you" cannot apply. await self._notify_security( ROLES_CHANGED, username=user.username, email=user.notify_email, client=client, - detail={"roles": role_ids}, + detail={"roles": role_ids, "source": "directory"}, ) ad_roles = _roles_from_ids(role_ids) ad_custom_permissions = await self._custom_permissions_for_ids(role_ids) @@ -7382,9 +7384,13 @@ async def _apply_reconcile_revocation(self, revocation: reconcile.SessionRevocat # ``ad_session_revoked`` alert record it. # # ACCOUNT_DISABLED only when the pass READ the disabled bit (vault BACKLOG #2140). That - # notice says an administrator disabled the account, which an absent account or an unreadable - # attribute does not establish, so every other whole-account reason, including any added - # later, gets the neutral DIRECTORY_SESSIONS_ENDED. + # notice says the directory reports the account disabled, which an absent account or an + # unreadable attribute does not establish, so every other whole-account reason, including + # any added later, gets the neutral DIRECTORY_SESSIONS_ENDED. + # + # ``source`` makes the renderer word ROLES_CHANGED and ACCOUNT_DISABLED for the directory, + # with no administrator in them and no "if this was you" (vault BACKLOG #3121). The + # login-path re-sync tags its ROLES_CHANGED the same way. if revocation.role_ids is not None: kind = ROLES_CHANGED elif revocation.reason == reconcile.REVOKE_REASONS[reconcile.ProbeOutcome.DISABLED]: @@ -7396,7 +7402,7 @@ async def _apply_reconcile_revocation(self, revocation: reconcile.SessionRevocat kind, username=user.username, email=user.notify_email, - detail={"reason": revocation.reason}, + detail={"reason": revocation.reason, "source": "directory"}, ) return True diff --git a/messagefoundry/pipeline/security_notify.py b/messagefoundry/pipeline/security_notify.py index 1652eae709..9c82556f3d 100644 --- a/messagefoundry/pipeline/security_notify.py +++ b/messagefoundry/pipeline/security_notify.py @@ -166,6 +166,19 @@ ), } +# Vault BACKLOG #3121. A directory-driven role change or disable names no administrator: the +# directory made it. The directory line and the "If you did not expect this change" closing follow +# from ``from_directory`` in :func:`_build_body`, so these say only what changed. The disable reads +# what the reconciler did, which is end the sessions; it does not claim MessageFoundry disabled +# anything. +_DIRECTORY_DESCRIPTIONS = { + ROLES_CHANGED: "Your account's roles were changed.", + ACCOUNT_DISABLED: ( + "Your organization's directory reports your account as disabled, so MessageFoundry ended " + "your sessions." + ), +} + _LOCAL_REMEDY = ( "ask your MessageFoundry administrator for a password reset, or the host operator to run " @@ -257,9 +270,9 @@ def _build_body(event: SecurityEvent) -> str: event.event_type == NOTIFY_EMAIL_SET and event.detail.get("set_by") == "administrator" ) # BACKLOG #1139, #2291: the directory made this change, not the holder and not the console. - from_directory = event.event_type in (EMAIL_CHANGED, USERNAME_CHANGED) and ( - event.detail.get("source") == "directory" - ) + # The sender's tag decides, whatever the kind (vault BACKLOG #3121), so a new directory notice + # cannot fall back to the administrator wording by missing from a list here. + from_directory = event.detail.get("source") == "directory" if moved_by_admin: description = ( "An administrator changed the address that receives security notices for your account." @@ -268,6 +281,9 @@ def _build_body(event: SecurityEvent) -> str: description = ( "An administrator set this address to receive security notices about your account." ) + elif from_directory and event.event_type in _DIRECTORY_DESCRIPTIONS: + # Only these two kinds name an administrator in their own text; the others need no swap. + description = _DIRECTORY_DESCRIPTIONS[event.event_type] else: description = _DESCRIPTIONS.get( event.event_type, "A security event occurred on your account." @@ -437,7 +453,9 @@ def _build_body(event: SecurityEvent) -> str: # and a bare "Source IP" line would read as the address of whoever made it. Measured # 2026-10-06, the directory notices that pass an address come from ``_upsert_ad_user``, on # a directory sign-in to this account, which copies the change down BEFORE the sign-in is - # finished: a later step can still refuse it, or it can still owe a second factor. So the + # finished: a later step can still refuse it, or it can still owe a second factor. Since + # vault BACKLOG #3121 the role re-sync in ``_complete_ad_login`` is one more such sender, + # and it too runs inside a directory sign-in before the session is issued. So the # line says a sign-in was under way, never that one succeeded. Nothing on the event says # the address came from a sign-in, so a new sender passing another kind of address on a # directory event would make this line false; such a sender must change it. @@ -482,7 +500,8 @@ def _build_body(event: SecurityEvent) -> str: ) ): # A directory rename or email change is an administrator's act, so "if this was you" - # cannot apply to it (the email half is BACKLOG #2291). Nor to a federated link or unlink: + # cannot apply to it (the email half is BACKLOG #2291), nor to a directory role change or + # disable (vault BACKLOG #3121). Nor to a federated link or unlink: # both routes refuse an administrator changing their own account's binding. Nor to the # engine ending sessions on a directory answer (vault BACKLOG #2140). closing = "If you did not expect this change, contact your MessageFoundry administrator." diff --git a/tests/test_ad_session_reconcile.py b/tests/test_ad_session_reconcile.py index b8d7952136..5dbe0199e8 100644 --- a/tests/test_ad_session_reconcile.py +++ b/tests/test_ad_session_reconcile.py @@ -3470,8 +3470,8 @@ async def test_a_scope_revocation_sends_no_account_disabled_notice() -> None: #: vault BACKLOG #2140. The notice each revocation reason sends, decided once. ``None`` is no notice. -#: ACCOUNT_DISABLED says an administrator disabled the account, so only the reason that READ the -#: disabled bit may send it. +#: ACCOUNT_DISABLED says the directory reports the account disabled, so only the reason that READ +#: the disabled bit may send it. _NOTICE_FOR_REASON: dict[str, str | None] = { "directory_disabled": ACCOUNT_DISABLED, "directory_absent": DIRECTORY_SESSIONS_ENDED, @@ -3511,7 +3511,9 @@ async def test_each_revocation_reason_sends_its_decided_notice(reason: str) -> N expected = _NOTICE_FOR_REASON[reason] assert [e.event_type for e in notifier.sent] == ([] if expected is None else [expected]) if expected is not None: - assert notifier.sent[0].detail == {"reason": reason} + # ``source`` words ROLES_CHANGED and ACCOUNT_DISABLED for the directory (vault + # BACKLOG #3121); every reconciler notice carries it. + assert notifier.sent[0].detail == {"reason": reason, "source": "directory"} finally: await store.close() diff --git a/tests/test_auth_service.py b/tests/test_auth_service.py index 0955336679..5d94184cc8 100644 --- a/tests/test_auth_service.py +++ b/tests/test_auth_service.py @@ -828,6 +828,8 @@ def role_changes() -> list[SecurityEvent]: assert len(changes) == 2 assert changes[-1].username == "jdoe" and changes[-1].email == "jdoe@example.org" assert changes[-1].detail.get("roles") == ["viewer"] + # Worded for the directory, as the reconciler's notice is (vault BACKLOG #3121). + assert changes[-1].detail.get("source") == "directory" finally: await store.close() @@ -945,6 +947,8 @@ async def test_clearing_the_profile_address_leaves_the_account_still_notifiable( later = notifier.events[before:] assert [e.event_type for e in later] == [ROLES_CHANGED] assert later[0].email == "bob@example.org" + # An administrator's act keeps the administrator wording (vault BACKLOG #3121). + assert "source" not in later[0].detail finally: await store.close() diff --git a/tests/test_security_notify.py b/tests/test_security_notify.py index be36691ee5..beea972807 100644 --- a/tests/test_security_notify.py +++ b/tests/test_security_notify.py @@ -15,6 +15,7 @@ import pytest from messagefoundry.auth.notifications import ( + ACCOUNT_DISABLED, ACCOUNT_LOCKED, ADMIN_NEW_IP, DIRECTORY_SESSIONS_ENDED, @@ -24,6 +25,7 @@ NOTIFY_EMAIL_SET, PASSWORD_RESET, RECOVERY_CODE_USED, + ROLES_CHANGED, TEMPORARY_CREDENTIAL_EXPIRING, TEMPORARY_CREDENTIAL_EXPIRING_FOR_ISSUER, USERNAME_CHANGED, @@ -210,7 +212,11 @@ def test_a_directory_sessions_ended_notice_renders_its_own_text_and_claims_no_di body = _build_body(SecurityEvent(DIRECTORY_SESSIONS_ENDED, username="jdoe", email=None)) text = (_SUBJECTS[DIRECTORY_SESSIONS_ENDED] + "\n" + body).lower() assert "a security event occurred on your account." not in text - assert "directory" in text and "sessions" in text + # The description itself, not words the subject alone would satisfy (vault BACKLOG #3121). + assert ( + "A check against your organization's directory ended your MessageFoundry sessions." + in body.splitlines() + ) for claim in ("disabled", "deleted", "if this was you"): assert claim not in text assert body.endswith( @@ -218,6 +224,82 @@ def test_a_directory_sessions_ended_notice_renders_its_own_text_and_claims_no_di ) +#: vault BACKLOG #3121. The description each directory-driven kind carries, pinned whole. +_DIRECTORY_TEXT = { + ROLES_CHANGED: "Your account's roles were changed.", + ACCOUNT_DISABLED: ( + "Your organization's directory reports your account as disabled, so MessageFoundry ended " + "your sessions." + ), +} +_DIRECTORY_LINE = ( + "This change came from your organization's directory, not from the MessageFoundry console." +) +_DIRECTORY_CLOSING = "If you did not expect this change, contact your MessageFoundry administrator." +_ADMIN_TEXT = { + ROLES_CHANGED: "Your account's roles were changed by an administrator.", + ACCOUNT_DISABLED: "Your account was disabled by an administrator.", +} +_IF_THIS_WAS_YOU = ( + "If this was you, no action is needed. If not, contact your MessageFoundry administrator." +) + + +@pytest.mark.parametrize("kind", [ROLES_CHANGED, ACCOUNT_DISABLED]) +def test_a_directory_role_or_disable_notice_names_the_directory_and_no_administrator( + kind: str, +) -> None: + """vault BACKLOG #3121. The reconciler and the sign-in re-sync tag these ``source`` directory: + no administrator made the change and the holder did nothing, so neither the administrator + wording nor "if this was you" may appear.""" + body = _build_body( + SecurityEvent( + kind, + username="jdoe", + email="j@example.org", + detail={"reason": "directory_disabled", "source": "directory"}, + ) + ) + lines = body.splitlines() + assert _DIRECTORY_TEXT[kind] in lines + assert _DIRECTORY_LINE in lines + assert _ADMIN_TEXT[kind] not in lines + assert "if this was you" not in body.lower() + assert body.endswith(_DIRECTORY_CLOSING) + + +@pytest.mark.parametrize("kind", [ROLES_CHANGED, ACCOUNT_DISABLED]) +def test_an_administrator_role_or_disable_notice_keeps_its_wording(kind: str) -> None: + """vault BACKLOG #3121, closing item 2: the notice an administrator's own act sends keeps its + text. Kept by the row's instruction, not judged correct: "if this was you" on an administrator's + act is the shape BACKLOG #1141 fixed for the reset notice, and vault #3072 item 10 names.""" + body = _build_body(SecurityEvent(kind, username="jdoe", email="j@example.org")) + lines = body.splitlines() + assert _ADMIN_TEXT[kind] in lines + assert _DIRECTORY_TEXT[kind] not in lines + assert "directory" not in body.lower() + assert body.endswith(_IF_THIS_WAS_YOU) + + +def test_the_sign_in_role_re_sync_notice_says_where_the_sign_in_came_from() -> None: + """vault BACKLOG #3121. The sign-in re-sync passes the client address, and a directory notice + must not print it as the address of whoever made the change.""" + body = _build_body( + SecurityEvent( + ROLES_CHANGED, + username="jdoe", + email="j@example.org", + client_ip="203.0.113.7", + detail={"roles": ["viewer"], "source": "directory"}, + ) + ) + assert "Source IP" not in body + assert ( + "MessageFoundry picked up this change during a sign-in to your account from 203.0.113.7." + in body.splitlines() + ) + + def test_body_says_a_directory_repoint_came_from_the_directory() -> None: """BACKLOG #1139 (ASVS 6.3.7): where the change came from decides what the reader can DO. A directory-driven repoint is not editable in the console, so an unexplained one reads as a From c2337e6134cb6b9ee128614bdeb26624fe1c90d6 Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Fri, 16 Oct 2026 16:34:45 -0500 Subject: [PATCH 04/57] TOTP enrolment follow-ups from PR 2080 (BACKLOG #3164 #3121 #3113) This commit is the BACKLOG #3113 layer; the two before it are #3164 and #3121. Item 1: provision_first_administrator's refusal when enable_totp matches no row no longer tells the operator to remove the entry it showed. Under the re-stage race (vault #3025) another run can turn on this run's secret, so the text says to keep the entry until a sign-in shows which works. A race test drives two runs on one row in a fixed order and measures that the loser's secret is the live one. Items 2 and 7: in confirm_mfa_enrollment a store refusal of revoke_session is logged via write_audit_soft and the auth.mfa_enroll_refused row is still written; session_ended records the revoke's outcome instead of a constant. Items 3 and 5: the Elevation docstring, the "a retry works" comment and two long SECURITY.md sentences now say what the code does. Item 4: the docs/ASVS-L2-PHASE0-CHANGES.md TOTP key row no longer says a live code proves the copy that is enabled; it names #3025. Item 6: test_webui asserts enable_totp's result. Items 8 and 9 are declined in code comments: 8 would undo #1902's rotate-first order; 9 needs enable_totp to return a reason on all three store backends. Proposed PR title: Auth follow-ups: start-up probe off the loop, directory notice wording, TOTP enrolment refusal fixes (BACKLOG #3164 #3121 #3113) Proposed banner, #3164: BUILT on engine branch b257-d1. The lifespan awaits asyncio.to_thread(auth.probe_credential_generation) with its guard kept; tests/test_credential_probe_off_loop.py proves the generator runs off the loop thread. Proposed banner, #3121: BUILT on engine branch b257-d1. Reconciler and sign-in re-sync ROLES_CHANGED and ACCOUNT_DISABLED notices carry source directory and render directory wording with the "If you did not expect this change" closing; administrator wording kept; tests pin the descriptions. Proposed banner, #3113: BUILT on engine branch b257-d1. Items 1 to 7 done with tests (race test for item 1, raise test for items 2 and 7); items 8 and 9 declined in writing in confirm_mfa_enrollment. The PHASE0 line is corrected; no vault docs/security file quotes it, so no cell re-read is owed unless the scorecard holder finds one. --- docs/ASVS-L2-PHASE0-CHANGES.md | 2 +- docs/SECURITY.md | 12 ++-- messagefoundry/auth/service.py | 55 +++++++++++---- .../tests/test_webui.py | 3 +- tests/_admin_account.py | 9 +-- tests/test_mfa.py | 53 ++++++++++++++- tests/test_provision_first_administrator.py | 68 +++++++++++++++++++ 7 files changed, 177 insertions(+), 25 deletions(-) diff --git a/docs/ASVS-L2-PHASE0-CHANGES.md b/docs/ASVS-L2-PHASE0-CHANGES.md index 653036ec04..4bcdd77e74 100644 --- a/docs/ASVS-L2-PHASE0-CHANGES.md +++ b/docs/ASVS-L2-PHASE0-CHANGES.md @@ -468,7 +468,7 @@ is right. | Off-box log-forward client key | 4, on the same condition as the API TLS server key | Loads the combined PEM in `_build_tls_context` ([`logging_setup.py`](../messagefoundry/logging_setup.py)). There is no passphrase setting | As for the API TLS server key. The file's permissions are the key's only protection, so setting them is the site's job | | Native API client key | 4, on the same condition as the API TLS server key | Holds no copy. The client process loads it (`_build_verify_context` in [`apiclient/client.py`](../messagefoundry/apiclient/client.py)) | Every function in this row belongs to the operator of the client machine | | Non-production self-signed key | Two keys, each 4, private authentication key, like the API TLS placeholder key. The `cert self-signed` key is a placeholder a site may serve. The load-test harness pair is also test keying material under section 8.1.4 | Mints both with `make_self_signed` in [`pki.py`](../messagefoundry/pki.py), the CLI key on request. The CLI writes its key through `_write_private_key`, which refuses to overwrite an existing file. The harness writes its pair with a plain write and `chmod`, as the lifecycle row above says. Nothing is audited | The `cert self-signed` key is a placeholder key, like the API TLS placeholder key: a site may serve it until it supplies a chain. Set its cryptoperiod under section 5.3.6 item 4 through `--days`, because the engine never renews it. Replace it with an operator-supplied chain, then delete both files. The load-test harness pair is test keying material, which section 8.1.4 says shall not be used operationally, and the engine does not stop that | -| TOTP shared secret | 3, symmetric authentication key: an HMAC key shared by exactly two parties | **Pre-activation (7.1):** `begin_mfa_enrollment` in [`auth/service.py`](../messagefoundry/auth/service.py) stores a fresh staged secret and audits `auth.mfa_enroll_started`. A later call replaces it. A staged secret nobody confirms stays in the column until then, or until a disable or reset; nothing moves it on by time. **Key confirmation, then active (7.1, transition 4):** `confirm_mfa_enrollment` turns MFA on only after a live code proves the user's copy, and audits `auth.mfa_enrolled`. **Out of use:** `disable_totp` sets the live column to NULL on a disable or an administrator reset, audited as `auth.mfa_disabled` or `auth.mfa_reset`. That removes the live copy. It is not destruction to the 8.3.4 bar (point 3), because older copies can survive. **Backup:** where a DR archive carries the store, it carries the secret; the lifecycle row above says when discarding the DEK erases those copies. Table 7 allows backup of this key type, and B.3.2 (its case 3) says a key that can be re-established need not be backed up. Re-enrolment re-establishes this one. **Cryptoperiod: none.** The secret is exempt from a calendar lifetime (owner ruling 2026-09-27, BACKLOG #1931). Section 5.3.6 item 3 suggests an originator-usage period of no more than two years. So this is a recorded departure from that suggestion, not an alignment with it | Keep the user's copy in one authenticator. After an administrator reset, tell the user to delete the old entry. Section 8.3.5 says the party revoking a key shared by two shall inform the other | +| TOTP shared secret | 3, symmetric authentication key: an HMAC key shared by exactly two parties | **Pre-activation (7.1):** `begin_mfa_enrollment` in [`auth/service.py`](../messagefoundry/auth/service.py) stores a fresh staged secret and audits `auth.mfa_enroll_started`. A later call replaces it. A staged secret nobody confirms stays in the column until then, or until a disable or reset; nothing moves it on by time. **Key confirmation, then active (7.1, transition 4):** `confirm_mfa_enrollment` turns MFA on only after a live code checks out against the secret it read, and audits `auth.mfa_enrolled`. The enable itself checks only that some secret is still staged and TOTP is still off, not that the staged secret is the one the code proved, so a second enrolment start that re-stages the secret inside the confirm's window can turn on a secret no code proved (vault BACKLOG #3025, open; corrected under vault BACKLOG #3113). **Out of use:** `disable_totp` sets the live column to NULL on a disable or an administrator reset, audited as `auth.mfa_disabled` or `auth.mfa_reset`. That removes the live copy. It is not destruction to the 8.3.4 bar (point 3), because older copies can survive. **Backup:** where a DR archive carries the store, it carries the secret; the lifecycle row above says when discarding the DEK erases those copies. Table 7 allows backup of this key type, and B.3.2 (its case 3) says a key that can be re-established need not be backed up. Re-enrolment re-establishes this one. **Cryptoperiod: none.** The secret is exempt from a calendar lifetime (owner ruling 2026-09-27, BACKLOG #1931). Section 5.3.6 item 3 suggests an originator-usage period of no more than two years. So this is a recorded departure from that suggestion, not an alignment with it | Keep the user's copy in one authenticator. After an administrator reset, tell the user to delete the old entry. Section 8.3.5 says the party revoking a key shared by two shall inform the other | | Anonymizer re-identification salt | None of the 19 types in 5.1.1 fits. It keys a hash that pseudonymizes one dataset. The nearest fit is 5.1.2's *other secret information*, so this row borrows 5.3.7's rule for that and takes no cryptoperiod from 5.3.6 | Never mints or stores it. `Keyer` in [`anon/keying.py`](../messagefoundry/anon/keying.py) refuses a visibly weak salt, and the export reads it from the environment for one run | Mint it with a CSPRNG, one per dataset, and hold it with one person. Section 5.3.7 says other secret information shall not be kept longer than necessary, so its period is one dataset. Keep no backup unless the dataset must be regenerated, and destroy it when that need ends | | Sealed-cache key | 6, symmetric data-encryption key. It protects the confidentiality and integrity of decoded cache values in the engine's own memory | Mints it per process (`_Sealer` in [`store/sealed_cache.py`](../messagefoundry/store/sealed_cache.py)) and uses it only to seal and open cache values, with each value bound to its cache key as associated data. Pre-activation and active are one step, and the key is destroyed with the process, so its cryptoperiod (5.3.6) is the process lifetime. There is no backup or archive of it (Table 7 is moot) | Protect the engine process's memory: the in-use residual in `docs/PHI.md` §3 and the host requirements in §10 apply, because this key sits in the same memory as the values it seals | diff --git a/docs/SECURITY.md b/docs/SECURITY.md index 089cc02aa2..cd9bb91528 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -2768,11 +2768,13 @@ way past the sign-in lock (ADR 0197 Amendment A). Such an account's first factor An administrator clears a lost authenticator via `POST /users/{id}/reset-mfa` (which also revokes the user's sessions). A confirm turns TOTP on only while a secret is still staged and TOTP is still off (BACKLOG -#2224). If a reset clears the secret during the confirm, or another confirm turns TOTP on first, -the confirm turns nothing on and returns no recovery codes. It ends the session it just rotated and -answers as a lost session: `401` on `POST /me/mfa/confirm`, and the sign-in page in the web -console. It is audited `auth.mfa_enroll_refused`, whose `reason` is `secret_cleared`, -`already_enabled` or `account_gone`, read after the refusal. The check is that some secret is +#2224). A reset can clear the secret during the confirm, or another confirm can turn TOTP on +first. Either way the confirm turns nothing on and returns no recovery codes. It ends the session +it just rotated and answers as a lost session: `401` on `POST /me/mfa/confirm`, and the sign-in +page in the web console. It is audited `auth.mfa_enroll_refused`. Its `reason` is +`secret_cleared`, `already_enabled` or `account_gone`. That reason is read after the refusal, so a +later write can blur it. Its `session_ended` is false when the store refused to end the rotated +session; the token is still not handed back (vault BACKLOG #3113). The check is that some secret is staged, not that it is the one the confirm verified: a second enrolment start that re-stages the secret inside the same window is not caught by it. With `[security].require_mfa` on — **the default since BACKLOG #187 (secure-by-default, including the diff --git a/messagefoundry/auth/service.py b/messagefoundry/auth/service.py index c05f6b0c9e..485a2b6cab 100644 --- a/messagefoundry/auth/service.py +++ b/messagefoundry/auth/service.py @@ -824,9 +824,12 @@ class Elevation: the session was ended because its temporary password lapsed, with the proof checked or not, as that ``auth.temp_password_expired`` row records; or (``confirm_mfa_enrollment`` only, BACKLOG #2224) the proof was good but turning TOTP on matched no row, so the confirm ended - the session it had just rotated, as its ``auth.mfa_enroll_refused`` row records. Fails - CLOSED: no token is handed back. Held apart from a wrong proof so a route answers 401 rather than re-prompting on a - session that no longer exists. The ``auth.reauth`` row's ``session_revoked`` says which. + the session it had just rotated, as its ``auth.mfa_enroll_refused`` row records. That row's + ``session_ended`` is false when the store refused the revoke (vault BACKLOG #3113): the + rotated session then still exists, but its token was never handed back. Fails CLOSED: no + token is handed back. Held apart from a wrong proof so a route answers 401 rather than + re-prompting on a session that is gone or unreachable. For a password re-auth, the + ``auth.reauth`` row's ``session_revoked`` says which cause applied. ``recovery_codes`` is populated only by :meth:`AuthService.confirm_mfa_enrollment` (shown once). @@ -835,7 +838,7 @@ class Elevation: correct code as incorrect. A password re-proof is never refused for a lock (BACKLOG #1138). ``ok`` is DERIVED, not stored, and that is load-bearing rather than tidiness. Held as a field it - was a second spelling of ``token is not None`` -- true of all 19 constructions -- so the type + was a second spelling of ``token is not None`` -- true of every construction then -- so the type could represent a state the system never produces, mypy could not narrow ``token`` from ``ok``, and every consuming site paid ``if not elevation.ok or elevation.token is None``: a second clause whose only job was to re-derive the first in a form the checker accepts. As a property the @@ -3120,13 +3123,20 @@ async def provision_first_administrator( # Conditional (BACKLOG #2224): a factor clear by another run's repair branch, or another # run that enabled TOTP first, since the secret was staged above makes it match no row. # Refused before the role, so the row stays roleless and a re-run completes it. + # + # The words must not say to remove the entry this run showed (vault BACKLOG #3113). + # Staging is unconditional, so the last run to stage owns the staged secret, and another + # run's enable can turn on THIS run's secret before this enable is refused (the re-stage + # race, vault BACKLOG #3025). So this run cannot tell whether its entry is live. if not await self._store.enable_totp(user_id, recovery_code_hashes=hashes): raise FirstAdministratorRefused( "the authenticator enrolment was changed while this command ran, so this run " - "did not turn TOTP on and granted no role; remove the authenticator entry it " - "showed. " + "did not turn TOTP on and granted no role. Another run may have turned on the " + "secret this run showed, so keep the authenticator entry it showed until a " + "sign-in to this account shows which entry works. " "If another provision-admin run is in progress, let it finish. Then run the " - "command again" + "command again: it reads that account and either completes it or says why it " + "cannot" ) if notify_email is not None: # Unconditional rather than fresh-path-only, because the invariant "the supplied address @@ -10066,7 +10076,8 @@ async def confirm_mfa_enrollment( still OFF (BACKLOG #1902), so a lost session never leaves MFA on with codes nobody saw. So does a good code whose activation matches no row, because a reset cleared the staged secret or a second confirm enabled first (BACKLOG #2224). That confirm ends the session it rotated, - hands back no codes, and audits ``auth.mfa_enroll_refused``. + hands back no codes, and audits ``auth.mfa_enroll_refused``, whose ``session_ended`` says + whether the store took the revoke (vault BACKLOG #3113). This is one of the two legs that turn an MFA-pending session into an MFA-satisfied one for a FIRST enrolment, so it rotates for the same reason ``verify_mfa`` does: without it a pre-MFA @@ -10117,7 +10128,10 @@ async def confirm_mfa_enrollment( # the rotation has succeeded; on a lost session MFA stays off and the staged secret stays put, # and the user signs in again and confirms with a code from a later step (this one is spent). # The cost: if enable_totp itself fails after a good rotation, the new token is never handed - # back, so the user is signed out with MFA still off. That fails closed, and a retry works. + # back, so the user is signed out with MFA still off. That fails closed. A retry is not + # always a plain one (vault BACKLOG #3113): it needs a code from a later step, since this one + # is spent, and where the enable matched no row (#2224, below) a reset may have cleared the + # staged secret, so the user starts the enrolment again. # Stamp against the OLD hash, then rotate — never the other way round. async with self._sealed_stamp(hash_token(token)): await self._store.mark_session_mfa_verified(hash_token(token)) @@ -10140,10 +10154,25 @@ async def confirm_mfa_enrollment( # # Known cost: ``_elevated`` already ran the session cap for the rotated session, so # at the cap that run may have ended the account's oldest full session, and it stays - # ended. The order is #1902's, rotate first and enable last, and is kept. - await self._store.revoke_session(hash_token(elevation.token)) + # ended. The order is #1902's, rotate first and enable last, and is kept. Vault BACKLOG + # #3113 item 8 is declined for that reason: enabling first would undo #1902. + # + # A store refusal of the revoke is logged, not raised, so the refusal row below is + # still written (vault BACKLOG #3113). ``session_ended`` records what the revoke did. + # The token is not handed back either way, so the caller still cannot use the session. + rotated = hash_token(elevation.token) + session_ended = await write_audit_soft( + lambda: self._store.revoke_session(rotated), + log=_log, + message="a refused TOTP enrolment for %s could not end the session it rotated", + args=lambda: (scrub_log_argument(identity.username),), + errors=_audit_write_errors(), + defects=AUDIT_WRITE_DEFECTS, + ) # The reason is the row's state read AFTER the refusal, not the write's own verdict, - # so a later write can blur it; it is a pointer for an operator, not a proof. + # so a later write can blur it; it is a pointer for an operator, not a proof. Vault + # BACKLOG #3113 item 9 is declined: the write's own verdict would need ``enable_totp`` + # to return a reason on all three store backends. after = await self._load_user(identity.user_id) if after is None: reason = "account_gone" @@ -10154,7 +10183,7 @@ async def confirm_mfa_enrollment( await self._audit( "auth.mfa_enroll_refused", actor=identity.username, - detail=_json({"reason": reason, "session_ended": True}), + detail=_json({"reason": reason, "session_ended": session_ended}), client=client, ) return Elevation(session_lost=True) diff --git a/packaging/messagefoundry-webconsole/tests/test_webui.py b/packaging/messagefoundry-webconsole/tests/test_webui.py index eebfef122e..a1a9bb67d8 100644 --- a/packaging/messagefoundry-webconsole/tests/test_webui.py +++ b/packaging/messagefoundry-webconsole/tests/test_webui.py @@ -6254,7 +6254,8 @@ async def test_reauth_both_factors_enrolled_renders_code_and_passkey(engine: Eng secret = totp_mod.generate_secret() await service.store.set_totp_secret(boss_id, secret=secret) - await service.store.enable_totp(boss_id, recovery_code_hashes=[]) + # Conditional since BACKLOG #2224, so the premise is asserted (vault BACKLOG #3113 item 6). + assert await service.store.enable_totp(boss_id, recovery_code_hashes=[]) is True await _cookie_login(c, "boss") # fresh MFA-pending session boss_creds = await service.store.list_webauthn_credentials(boss_id) diff --git a/tests/_admin_account.py b/tests/_admin_account.py index 3fce51f421..df1e6331cc 100644 --- a/tests/_admin_account.py +++ b/tests/_admin_account.py @@ -129,16 +129,17 @@ async def create_local_user_chosen(service: AuthService, *, password: str, **kwa _provision_steps = itertools.count(1) -def provision_totp() -> dict[str, Any]: +def provision_totp(secret: str = PROVISION_TOTP_SECRET) -> dict[str, Any]: """The TOTP arguments ``provision_first_administrator`` needs under the shipped ``require_mfa`` (ADR 0197 Amendment A): the secret, a code for it, and the instant that code was "read". Each call lands on its own later 30-second step, so repeated provisions of one row - (the repair tests) never replay a step the row already consumed.""" + (the repair tests) never replay a step the row already consumed. ``secret`` is for a test + whose runs must show different entries.""" from messagefoundry.auth import totp at = time.time() + totp.DEFAULT_PERIOD * next(_provision_steps) return { - "totp_secret": PROVISION_TOTP_SECRET, - "totp_code": totp.totp(PROVISION_TOTP_SECRET, now=at), + "totp_secret": secret, + "totp_code": totp.totp(secret, now=at), "totp_code_read_at": at, } diff --git a/tests/test_mfa.py b/tests/test_mfa.py index 8b2eb0c907..92a43c5f45 100644 --- a/tests/test_mfa.py +++ b/tests/test_mfa.py @@ -18,6 +18,8 @@ import asyncio import heapq import inspect +import json +import logging import time import unicodedata from collections.abc import Awaitable @@ -2485,7 +2487,12 @@ async def reset_then_enable( rows = await store.list_audit( actor=ADMIN_USERNAME, action="auth.mfa_enroll_refused", limit=10 ) - assert len(rows) == 1 and '"reason": "secret_cleared"' in str(rows[0]["detail"]) + # ``session_ended`` is read from the revoke's outcome, not a constant (vault BACKLOG #3113). + assert len(rows) == 1 + assert json.loads(str(rows[0]["detail"])) == { + "reason": "secret_cleared", + "session_ended": True, + } actions = [e["action"] for e in await service.security_events_for(ADMIN_USERNAME)] assert "auth.mfa_enrolled" not in actions finally: @@ -2564,6 +2571,50 @@ async def other_confirm_first( await store.close() +async def test_a_refused_enable_whose_revoke_raises_still_writes_its_refusal_row( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + """RED when: a store refusal of the revoke skips the ``auth.mfa_enroll_refused`` row (vault + BACKLOG #3113 item 2), or the row claims the session ended when it did not (item 7).""" + store = await _store() + try: + service = AuthService(store, AuthSettings(mfa_verify_min_elapsed_seconds=0)) + identity, token, _ = await login_admin(service) + enroll = await service.begin_mfa_enrollment(identity) + + real_enable = store.enable_totp + + async def reset_then_enable( + user_id: str, *, recovery_code_hashes: list[str], now: float | None = None + ) -> bool: + await store.disable_totp(user_id) + return await real_enable(user_id, recovery_code_hashes=recovery_code_hashes, now=now) + + async def refused_revoke(token_hash: str, *, now: float | None = None) -> None: + raise RuntimeError("the store refused the revoke") + + monkeypatch.setattr(store, "enable_totp", reset_then_enable) + monkeypatch.setattr(store, "revoke_session", refused_revoke) + with caplog.at_level(logging.ERROR, logger="messagefoundry.auth.service"): + refused = await service.confirm_mfa_enrollment( + identity, fresh_totp(enroll.secret), token=token + ) + # Fails closed all the same: no token and no codes come back. + assert refused.ok is False and refused.session_lost is True + assert refused.token is None and refused.recovery_codes == () + rows = await store.list_audit( + actor=ADMIN_USERNAME, action="auth.mfa_enroll_refused", limit=10 + ) + assert len(rows) == 1, "the refusal row was skipped when the revoke raised" + detail = json.loads(str(rows[0]["detail"])) + assert detail == {"reason": "secret_cleared", "session_ended": False} + assert any( + "could not end the session it rotated" in r.getMessage() for r in caplog.records + ), "the refused revoke was swallowed without a log line" + finally: + await store.close() + + # --- ADR 0197 Amendment A, wave 1 (BACKLOG #1131): every account is born with a way past ---------- # # A lock exists to bound guesses. On an engine-generated 192-bit credential it bounds nothing, so diff --git a/tests/test_provision_first_administrator.py b/tests/test_provision_first_administrator.py index 5e0b88e4a7..bc826e98d2 100644 --- a/tests/test_provision_first_administrator.py +++ b/tests/test_provision_first_administrator.py @@ -13,12 +13,14 @@ from __future__ import annotations import asyncio +import base64 import io import json import sys import time import types from collections.abc import Callable, Sequence +from contextvars import ContextVar from pathlib import Path import pytest @@ -2708,3 +2710,69 @@ def test_the_last_wrong_code_is_not_told_to_try_again( != 0 ) assert sum("did not match" in text for text in terminal.shown) == 4 + + +# --- vault BACKLOG #3113 item 1: the run that loses an enrolment race -------------------------- + +#: A second synthetic base32 secret, so the two runs show different entries. Encoded here rather +#: than written as a literal, which the gitleaks generic-api-key rule reads as a credential. +_OTHER_TOTP_SECRET = base64.b32encode(b"1234567890").decode("ascii") + + +async def test_the_run_that_loses_an_enrolment_race_is_not_told_to_remove_a_live_entry( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Two runs repair one roleless row. The second stages its secret after the first, so the + first run's enable turns on the SECOND run's secret, and the second run's enable is refused. + Its refusal must not tell its operator to remove the entry it showed: that entry is the live + one. This is the re-stage race of vault BACKLOG #3025, driven in a fixed order.""" + store = await MessageStore.open(":memory:") + try: + service = AuthService(store, AuthSettings()) + await service.initialize() + await _roleless_account(store, email=None) + first = provision_totp() + second = provision_totp(_OTHER_TOTP_SECRET) + real_enable = store.enable_totp + second_at_enable = asyncio.Event() + first_enabled = asyncio.Event() + run: ContextVar[str] = ContextVar("run", default="first") + tasks: list[asyncio.Task[object]] = [] + + async def run_second() -> object: + run.set("second") + return await service.provision_first_administrator( + username="site-admin", password=_PASSWORD, actor="second", **second + ) + + async def racing_enable(user_id: str, *, recovery_code_hashes: list[str]) -> bool: + if run.get() == "first": + # The first run has staged its secret. The second run now stages its own over it, + # and stops at its own enable. + tasks.append(asyncio.create_task(run_second())) + await second_at_enable.wait() + try: + return await real_enable(user_id, recovery_code_hashes=recovery_code_hashes) + finally: + first_enabled.set() + second_at_enable.set() + await first_enabled.wait() + return await real_enable(user_id, recovery_code_hashes=recovery_code_hashes) + + monkeypatch.setattr(store, "enable_totp", racing_enable) + won = await service.provision_first_administrator( + username="site-admin", password=_PASSWORD, actor="first", **first + ) + assert won.repaired + with pytest.raises(FirstAdministratorRefused) as lost: + await tasks[0] + # The premise, measured: the secret the losing run showed is the one turned on. + row = await store.get_user("roleless") + assert row is not None and row.totp_enabled + assert await store.get_totp_secret("roleless") == _OTHER_TOTP_SECRET + message = str(lost.value) + assert "did not turn TOTP on and granted no role" in message + assert "remove" not in message.lower() + assert "keep the authenticator entry it showed" in message + finally: + await store.close() From c6113d9451acc9d990ed098224cdc5608e5190b2 Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Fri, 16 Oct 2026 16:34:58 -0500 Subject: [PATCH 05/57] LDAP result-code refusal: referral checked inside the helper, picklable error, named costs (BACKLOG #3010) Review round one. _refuse_unanswered now refuses a referral first itself, so no caller can reorder the two. LdapUnansweredError survives copy and pickle. The docstring says the #3100 item 3 deferral holds only once that item covers the reconciler's except LdapError arm. The changelog names the costs: a standing code stops revocation unalerted, and an unpaged nested group search over a directory's cap would refuse sign-in. New tests: a busy account beside a departed one in a mixed pass, copy and pickle. Proposed PR title: An LDAP search that does not end with success is an outage, not "no such account" (BACKLOG #3010) Proposed vault banner: BUILT on branch b257-3010. _search in auth/ldap.py refuses any result code but success, with or without entries, through _refuse_unanswered; the reconciler reads it as unavailable and sign-in as a login error. Closing items 1-3 built and tested; item 4 deferred to #3100 item 3, which must widen to the except LdapError arm. Shipped open: bind result codes (#3169), unpaged group search, stale reconcile.py docstring. --- changelog.d/3010.security.md | 8 ++++- messagefoundry/auth/ldap.py | 48 ++++++++++++++++---------- tests/test_ldap_search_result_codes.py | 40 ++++++++++++++++++++- 3 files changed, 75 insertions(+), 21 deletions(-) diff --git a/changelog.d/3010.security.md b/changelog.d/3010.security.md index 996a4407d1..7ae138d201 100644 --- a/changelog.d/3010.security.md +++ b/changelog.d/3010.security.md @@ -5,4 +5,10 @@ have struck a present account toward revocation, and a busy nested group search read as fewer groups. The reconciler now reads these as unavailable, which never revokes, and a sign-in writes `auth.login_error` naming the code. `check-privileges` names the code for the bind-account read. - No directory diagnostic text reaches the error, the log or the audit row. (vault `BACKLOG #3010`) + No directory diagnostic text reaches the error, the log or the audit row. + **The cost:** a code that recurs on every pass, such as a deleted search base, now stops + revocation and the role re-diff for as long as it lasts, with one WARNING per code per process + and no latched alert. The nested group search is not paged. Where a directory caps an unpaged + search (AD's MaxPageSize, 1000 by default), an account in more groups than the cap would end + it with sizeLimitExceeded and could no longer sign in, where it used to sign in with a truncated + group list. That was not measured against a domain controller. (vault `BACKLOG #3010`) diff --git a/messagefoundry/auth/ldap.py b/messagefoundry/auth/ldap.py index 93f2255d4f..b42787fe5d 100644 --- a/messagefoundry/auth/ldap.py +++ b/messagefoundry/auth/ldap.py @@ -121,14 +121,16 @@ class LdapReferralError(LdapError): class LdapUnansweredError(LdapError): """The directory ended an operation with a result code that is not an answer (BACKLOG #3010). - A subclass, so every ``except LdapError`` reads it as the directory not answering: the session - reconciler as unavailable, which never strikes, and a sign-in as ``auth.login_error``. It - carries the code so a caller can tell codes apart without parsing the text. ``code`` is the result code, or ``None`` - when the result held none that is an integer; ``code_name`` is ldap3's name for it, never text - the directory sent. + A subclass, so every ``except LdapError`` handles it as before; :func:`_refuse_unanswered` says + what that means for each caller. ``code`` is the result code, or ``None`` when the result held + none that is an integer; ``code_name`` is ldap3's name for it, never text the directory sent. + Both default, so the exception survives a copy or a pickle, which rebuild it from its message + and then restore both from its ``__dict__``. """ - def __init__(self, message: str, *, code: int | None, code_name: str) -> None: + def __init__( + self, message: str, *, code: int | None = None, code_name: str = "no code" + ) -> None: super().__init__(message) self.code = code self.code_name = code_name @@ -771,8 +773,9 @@ def _refuse_referral(conn: Any, operation: str) -> None: def _refuse_unanswered(conn: Any, operation: str, *, answers: frozenset[int] | None = None) -> None: - """Raise :class:`LdapUnansweredError` unless ``conn``'s last operation ended with a result code - in ``answers``, which is success alone unless the caller names another set. + """Raise :class:`LdapReferralError` for a referral, and otherwise :class:`LdapUnansweredError` + unless ``conn``'s last operation ended with a result code in ``answers``, which is success alone + unless the caller names another set. BACKLOG #3010. ldap3 raises on none of these codes here. Every ``Connection`` in this module keeps ldap3's default ``raise_exceptions=False``, and even under ``True`` ldap3 2.9.1 passes @@ -790,22 +793,30 @@ def _refuse_unanswered(conn: Any, operation: str, *, answers: frozenset[int] | N read as absence it would read every account absent at once, which the reconciler's mass-revoke breaker does not stop at or under its floor. - **The cost, named.** The reconciler's unavailable never revokes, so a code that recurs on every - pass, such as a deleted search base, stops offboarding for as long as it lasts. Making a - standing one visible is vault BACKLOG #3100's item 3; #3010's item 4 was deferred to it. This - warns once per operation and code, and the text carries no directory text: the code, ldap3's - name for it and the operation, never the diagnostic message the directory sent. + **What each caller makes of it.** The session reconciler reads an :class:`LdapError` as + unavailable, which never strikes; a sign-in writes ``auth.login_error``; the nested group search + raising means no caller sees a shorter group list. + + **The cost, named.** Unavailable never revokes, so a code that recurs on every pass, such as a + deleted search base, stops offboarding and the role re-diff for as long as it lasts, and nothing + latches an alert for it. Making a standing one visible is #3010's item 4, which the batch + deferred to vault BACKLOG #3100's item 3. As filed, that item names the reconciler's broad + ``except Exception`` arm, which this error does not reach, so it must widen to cover the + ``except LdapError`` arm before the deferral holds. This warns once per operation and code, and + the text carries no directory text: the code, ldap3's name for it and the operation, never the + diagnostic message the directory sent. **A result that is not a dict is let through.** ldap3's synchronous strategy, the only one built here, reads ``self.result['type']`` straight after every search, so a real search cannot return without one. Only a test double can. ``answers`` is a parameter so an operation whose answers differ, such as a bind, can reuse this - with its own set. Run :func:`_refuse_referral` first: a referral is not success either, and only - that refusal reads as referred. + with its own set. A referral is refused first, as a referral, whatever ``answers`` holds, so + the reconciler still reads it as referred (BACKLOG #2538). """ from ldap3.core.results import RESULT_CODES, RESULT_SUCCESS # lazy, like every ldap3 import + _refuse_referral(conn, operation) result = conn.result if not isinstance(result, dict): return @@ -828,11 +839,10 @@ def _refuse_unanswered(conn: Any, operation: str, *, answers: frozenset[int] | N def _search(conn: Any, operation: str, **kwargs: Any) -> None: - """``conn.search(**kwargs)``, refusing a referral result and then any result but success + """``conn.search(**kwargs)``, refusing a referral result and any other result but success (:func:`_refuse_unanswered`). Every search in this module goes through here, so none can read either as "no entries"; a test pins that.""" conn.search(**kwargs) - _refuse_referral(conn, operation) _refuse_unanswered(conn, operation) @@ -1300,8 +1310,8 @@ def _resolve_groups(self, conn: Any, user_dn: str, member_of: list[str]) -> froz # Direct membership from the user's memberOf attribute. found = list(member_of) if self._s.ad_use_nested_groups and self._s.ad_group_search_base: - # A search the directory did not finish raises here (BACKLOG #3010), so a busy answer - # is an outage to every caller, never a shorter group list that narrows the roles. + # A search the directory did not finish raises here (BACKLOG #3010), never a shorter + # group list. _search( conn, "group search", diff --git a/tests/test_ldap_search_result_codes.py b/tests/test_ldap_search_result_codes.py index 643e966b76..fb9ee39c31 100644 --- a/tests/test_ldap_search_result_codes.py +++ b/tests/test_ldap_search_result_codes.py @@ -21,7 +21,9 @@ from __future__ import annotations +import copy import logging +import pickle from collections.abc import Callable from types import SimpleNamespace from typing import Any @@ -175,10 +177,26 @@ def test_a_caller_can_name_its_own_answers() -> None: def test_a_referral_is_still_refused_as_a_referral() -> None: - """The referral check runs first, so the reconciler still reads it as referred (#2538).""" + """The referral check runs first, so the reconciler still reads it as referred (#2538). It runs + inside the helper, so a caller that names its own answers cannot skip it or reorder it.""" conn = _conn({"result": RESULT_REFERRAL, "referrals": ["ldap://dc1.other.test.invalid/"]}) with pytest.raises(LdapReferralError): ldap_module._search(conn, "user search") + with pytest.raises(LdapReferralError): + ldap_module._refuse_unanswered(conn, "user bind", answers=frozenset({RESULT_REFERRAL})) + + +def test_the_refusal_survives_a_copy_and_a_pickle() -> None: + """``BaseException`` rebuilds from its args alone, so required keywords would raise here.""" + with pytest.raises(LdapUnansweredError) as refused: + ldap_module._search(_conn(_result(RESULT_BUSY)), "user search") + for clone in (copy.copy(refused.value), pickle.loads(pickle.dumps(refused.value))): + assert isinstance(clone, LdapUnansweredError) + assert (str(clone), clone.code, clone.code_name) == ( + str(refused.value), + RESULT_BUSY, + "busy", + ) @pytest.mark.parametrize("raw", [None, "0", True, False, 0.0], ids=repr) @@ -320,6 +338,26 @@ async def test_an_account_that_left_the_directory_is_still_revoked( assert await _alive(service, tokens) == set() +async def test_a_busy_account_beside_a_departed_one_is_left_alone_while_the_other_is_revoked( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The partial outage: the pass is not aborted, so each probe is judged. The account whose + search the directory did not answer keeps its session; the one that left is revoked.""" + async with _signed_in_estate(monkeypatch, ["jdoe", "asmith"]) as ( + directory, + service, + store, + tokens, + ): + busy_id = ldap_module.object_guid_filter_value(directory.ids["jdoe"]) + assert busy_id is not None + _answer_with(monkeypatch, RESULT_BUSY, when=lambda kw: busy_id in str(kw["search_filter"])) + directory.delete("asmith") + revoked = [r.username for _ in range(5) for r in (await _pass(service)).revocations] + assert revoked == ["asmith"] + assert await _alive(service, tokens) == {"jdoe"} + + # --- sign-in: a directory error, not an unknown user --------------------------------------------- From d2c88da1aea5971dbc02277293fc10498624dbc0 Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Fri, 16 Oct 2026 16:49:07 -0500 Subject: [PATCH 06/57] Review round one on the D1 auth follow-ups (BACKLOG #3164 #3121 #3113) #3121: the directory line "not from the MessageFoundry console" is gone from the role re-sync and disable notices. A re-sync's roles come from the directory's groups through the group-to-role map an administrator edits in the console, so that line could be false. The role notice now says MessageFoundry changed the roles to match the directory's groups. from_directory is gated on the kind again, so the reconciler's directory_sessions_ended, which now carries the source tag, does not claim the directory changed anything. A directory ACCOUNT_DISABLED gets its own subject, since the administrator subject says the MessageFoundry account was disabled. Tests render the as-sent shapes and pin the subject. Comments that called a directory change an administrator's act are reworded. #3113: the race test cancels the second run on failure and uses a low-entropy synthetic secret; the PHASE0 row links SECURITY.md rather than restating it; the revoke comment states it covers a refusal of the revoke alone. #3164: the lifespan comment names what the loop is already running. Proposed PR title: Auth follow-ups: start-up probe off the loop, directory notice wording, TOTP enrolment refusal fixes (BACKLOG #3164 #3121 #3113) Proposed banner, #3164: BUILT on engine branch b257-d1. The lifespan awaits asyncio.to_thread(auth.probe_credential_generation) with its guard kept; tests/test_credential_probe_off_loop.py proves the generator runs off the loop thread. Proposed banner, #3121: BUILT on engine branch b257-d1. Reconciler and sign-in re-sync ROLES_CHANGED and ACCOUNT_DISABLED notices carry source directory, name no administrator, and close with "If you did not expect this change"; the directory disable has its own subject; administrator wording kept; tests pin descriptions and subject. Proposed banner, #3113: BUILT on engine branch b257-d1. Items 1 to 7 done with tests (race test for item 1, raise test for items 2 and 7); items 8 and 9 declined in writing in confirm_mfa_enrollment. The PHASE0 line is corrected; no vault docs/security file quotes it, so no cell re-read is owed unless the scorecard holder finds one. --- docs/ASVS-L2-PHASE0-CHANGES.md | 2 +- docs/SECURITY.md | 6 ++- messagefoundry/api/app.py | 3 +- messagefoundry/auth/notifications.py | 5 +- messagefoundry/auth/service.py | 7 ++- messagefoundry/pipeline/security_notify.py | 56 ++++++++++++++------ tests/test_provision_first_administrator.py | 13 +++-- tests/test_security_notify.py | 58 ++++++++++++++------- 8 files changed, 103 insertions(+), 47 deletions(-) diff --git a/docs/ASVS-L2-PHASE0-CHANGES.md b/docs/ASVS-L2-PHASE0-CHANGES.md index 4bcdd77e74..a5eb73d18e 100644 --- a/docs/ASVS-L2-PHASE0-CHANGES.md +++ b/docs/ASVS-L2-PHASE0-CHANGES.md @@ -468,7 +468,7 @@ is right. | Off-box log-forward client key | 4, on the same condition as the API TLS server key | Loads the combined PEM in `_build_tls_context` ([`logging_setup.py`](../messagefoundry/logging_setup.py)). There is no passphrase setting | As for the API TLS server key. The file's permissions are the key's only protection, so setting them is the site's job | | Native API client key | 4, on the same condition as the API TLS server key | Holds no copy. The client process loads it (`_build_verify_context` in [`apiclient/client.py`](../messagefoundry/apiclient/client.py)) | Every function in this row belongs to the operator of the client machine | | Non-production self-signed key | Two keys, each 4, private authentication key, like the API TLS placeholder key. The `cert self-signed` key is a placeholder a site may serve. The load-test harness pair is also test keying material under section 8.1.4 | Mints both with `make_self_signed` in [`pki.py`](../messagefoundry/pki.py), the CLI key on request. The CLI writes its key through `_write_private_key`, which refuses to overwrite an existing file. The harness writes its pair with a plain write and `chmod`, as the lifecycle row above says. Nothing is audited | The `cert self-signed` key is a placeholder key, like the API TLS placeholder key: a site may serve it until it supplies a chain. Set its cryptoperiod under section 5.3.6 item 4 through `--days`, because the engine never renews it. Replace it with an operator-supplied chain, then delete both files. The load-test harness pair is test keying material, which section 8.1.4 says shall not be used operationally, and the engine does not stop that | -| TOTP shared secret | 3, symmetric authentication key: an HMAC key shared by exactly two parties | **Pre-activation (7.1):** `begin_mfa_enrollment` in [`auth/service.py`](../messagefoundry/auth/service.py) stores a fresh staged secret and audits `auth.mfa_enroll_started`. A later call replaces it. A staged secret nobody confirms stays in the column until then, or until a disable or reset; nothing moves it on by time. **Key confirmation, then active (7.1, transition 4):** `confirm_mfa_enrollment` turns MFA on only after a live code checks out against the secret it read, and audits `auth.mfa_enrolled`. The enable itself checks only that some secret is still staged and TOTP is still off, not that the staged secret is the one the code proved, so a second enrolment start that re-stages the secret inside the confirm's window can turn on a secret no code proved (vault BACKLOG #3025, open; corrected under vault BACKLOG #3113). **Out of use:** `disable_totp` sets the live column to NULL on a disable or an administrator reset, audited as `auth.mfa_disabled` or `auth.mfa_reset`. That removes the live copy. It is not destruction to the 8.3.4 bar (point 3), because older copies can survive. **Backup:** where a DR archive carries the store, it carries the secret; the lifecycle row above says when discarding the DEK erases those copies. Table 7 allows backup of this key type, and B.3.2 (its case 3) says a key that can be re-established need not be backed up. Re-enrolment re-establishes this one. **Cryptoperiod: none.** The secret is exempt from a calendar lifetime (owner ruling 2026-09-27, BACKLOG #1931). Section 5.3.6 item 3 suggests an originator-usage period of no more than two years. So this is a recorded departure from that suggestion, not an alignment with it | Keep the user's copy in one authenticator. After an administrator reset, tell the user to delete the old entry. Section 8.3.5 says the party revoking a key shared by two shall inform the other | +| TOTP shared secret | 3, symmetric authentication key: an HMAC key shared by exactly two parties | **Pre-activation (7.1):** `begin_mfa_enrollment` in [`auth/service.py`](../messagefoundry/auth/service.py) stores a fresh staged secret and audits `auth.mfa_enroll_started`. A later call replaces it. A staged secret nobody confirms stays in the column until then, or until a disable or reset; nothing moves it on by time. **Key confirmation, then active (7.1, transition 4):** `confirm_mfa_enrollment` turns MFA on only after a live code checks out against the secret it read, and audits `auth.mfa_enrolled`. The enable does not check that the staged secret is the one the code proved; [`SECURITY.md`](SECURITY.md), in its TOTP paragraph, states that gap (vault BACKLOG #3025; this sentence corrected under vault BACKLOG #3113). **Out of use:** `disable_totp` sets the live column to NULL on a disable or an administrator reset, audited as `auth.mfa_disabled` or `auth.mfa_reset`. That removes the live copy. It is not destruction to the 8.3.4 bar (point 3), because older copies can survive. **Backup:** where a DR archive carries the store, it carries the secret; the lifecycle row above says when discarding the DEK erases those copies. Table 7 allows backup of this key type, and B.3.2 (its case 3) says a key that can be re-established need not be backed up. Re-enrolment re-establishes this one. **Cryptoperiod: none.** The secret is exempt from a calendar lifetime (owner ruling 2026-09-27, BACKLOG #1931). Section 5.3.6 item 3 suggests an originator-usage period of no more than two years. So this is a recorded departure from that suggestion, not an alignment with it | Keep the user's copy in one authenticator. After an administrator reset, tell the user to delete the old entry. Section 8.3.5 says the party revoking a key shared by two shall inform the other | | Anonymizer re-identification salt | None of the 19 types in 5.1.1 fits. It keys a hash that pseudonymizes one dataset. The nearest fit is 5.1.2's *other secret information*, so this row borrows 5.3.7's rule for that and takes no cryptoperiod from 5.3.6 | Never mints or stores it. `Keyer` in [`anon/keying.py`](../messagefoundry/anon/keying.py) refuses a visibly weak salt, and the export reads it from the environment for one run | Mint it with a CSPRNG, one per dataset, and hold it with one person. Section 5.3.7 says other secret information shall not be kept longer than necessary, so its period is one dataset. Keep no backup unless the dataset must be regenerated, and destroy it when that need ends | | Sealed-cache key | 6, symmetric data-encryption key. It protects the confidentiality and integrity of decoded cache values in the engine's own memory | Mints it per process (`_Sealer` in [`store/sealed_cache.py`](../messagefoundry/store/sealed_cache.py)) and uses it only to seal and open cache values, with each value bound to its cache key as associated data. Pre-activation and active are one step, and the key is destroyed with the process, so its cryptoperiod (5.3.6) is the process lifetime. There is no backup or archive of it (Table 7 is moot) | Protect the engine process's memory: the in-use residual in `docs/PHI.md` §3 and the host requirements in §10 apply, because this key sits in the same memory as the values it seals | diff --git a/docs/SECURITY.md b/docs/SECURITY.md index cd9bb91528..f3fd6c4198 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -4387,11 +4387,13 @@ Users are notified of security-relevant changes to their account through **two** failed attempts**, or a step-up re-auth that clears such a run (suspicious-login signals, 6.3.5); and **password change**, **email change**, **role change**, **account disable**, and a **directory rename** of the username (credential changes, 6.3.7). - A notice for a change the directory made names the directory and no administrator, and it closes + A notice for a change the directory drove names the directory and no administrator, and it closes with "If you did not expect this change" rather than "If this was you". That covers at least the directory rename, the directory's email change, a role change applied by the session reconciler or by a directory sign-in's re-sync, and the reconciler's `account_disabled` when it read the - account disabled (vault BACKLOG #3121). When the reconciler ends an account's sessions because it + account disabled (vault BACKLOG #3121). The role notice says the roles were changed to match the + directory's groups, not that the console played no part: those groups map to roles through the + group-to-role map an administrator edits. When the reconciler ends an account's sessions because it could not match the account or could not read its `userAccountControl`, it sends `directory_sessions_ended`, not `account_disabled` (vault BACKLOG #2140). The rename notice, `username_changed`, names the old and the new name, and is sent only when diff --git a/messagefoundry/api/app.py b/messagefoundry/api/app.py index 3fe2b485f8..6370ef36ed 100644 --- a/messagefoundry/api/app.py +++ b/messagefoundry/api/app.py @@ -10491,7 +10491,8 @@ async def _audit_upload_prune(meta: UploadedFileMeta) -> None: # an ERROR and returns, and is guarded like the census: a policy-level fact must # never refuse the start, whatever a later policy clause raises. It runs off the loop # (vault BACKLOG #3164): a pathological context-word list makes the generator retry - # its policy screens many times, and the loop already serves the API by now. + # its policy screens many times, and the engine's listeners and workers already run + # on the loop by now. try: await asyncio.to_thread(auth.probe_credential_generation) except Exception: # noqa: BLE001 -- see the comment above diff --git a/messagefoundry/auth/notifications.py b/messagefoundry/auth/notifications.py index 9d108998b4..015e50cfc0 100644 --- a/messagefoundry/auth/notifications.py +++ b/messagefoundry/auth/notifications.py @@ -50,8 +50,9 @@ TEMPORARY_CREDENTIAL_EXPIRING = "temporary_credential_expiring" TEMPORARY_CREDENTIAL_EXPIRING_FOR_ISSUER = "temporary_credential_expiring_issuer" EMAIL_CHANGED = "email_changed" # 6.3.7 — the account's email address was changed -# 6.3.7 -- the account's roles changed: an administrator's act, or the directory's, which the -# reconciler and the sign-in re-sync tag with ``detail["source"] == "directory"`` (vault BACKLOG #3121). +# 6.3.7 -- the account's roles changed: an administrator's act, or a re-sync from the directory's +# groups, which the reconciler and the sign-in re-sync tag ``detail["source"] == "directory"`` +# (vault BACKLOG #3121). ROLES_CHANGED = "roles_changed" # 6.3.7 -- the directory renamed the account and the engine copied the new name onto the row # (BACKLOG #2017). When it is sent is stated once, at the send site in diff --git a/messagefoundry/auth/service.py b/messagefoundry/auth/service.py index 485a2b6cab..279c57b0e3 100644 --- a/messagefoundry/auth/service.py +++ b/messagefoundry/auth/service.py @@ -5459,7 +5459,7 @@ async def _complete_ad_login( # local one, so notify the affected user out-of-band too (ASVS 6.3.7), matching set_roles(). # Best-effort; the change is also visible in the audited /me/security-events feed. # ``source`` words it for the directory, as the reconciler's notice is (vault BACKLOG - # #3121): no administrator made this change, so "if this was you" cannot apply. + # #3121): the holder did not make this change, so "if this was you" cannot apply. await self._notify_security( ROLES_CHANGED, username=user.username, @@ -10159,7 +10159,10 @@ async def confirm_mfa_enrollment( # # A store refusal of the revoke is logged, not raised, so the refusal row below is # still written (vault BACKLOG #3113). ``session_ended`` records what the revoke did. - # The token is not handed back either way, so the caller still cannot use the session. + # The token is not handed back either way, so the caller still cannot use the session; + # a session the revoke left live expires on its own timers. A fault that also refuses + # the read and the audit write below still loses the row: this covers a refusal of the + # revoke alone. rotated = hash_token(elevation.token) session_ended = await write_audit_soft( lambda: self._store.revoke_session(rotated), diff --git a/messagefoundry/pipeline/security_notify.py b/messagefoundry/pipeline/security_notify.py index 9c82556f3d..e62f795d78 100644 --- a/messagefoundry/pipeline/security_notify.py +++ b/messagefoundry/pipeline/security_notify.py @@ -166,18 +166,42 @@ ), } -# Vault BACKLOG #3121. A directory-driven role change or disable names no administrator: the -# directory made it. The directory line and the "If you did not expect this change" closing follow -# from ``from_directory`` in :func:`_build_body`, so these say only what changed. The disable reads -# what the reconciler did, which is end the sessions; it does not claim MessageFoundry disabled -# anything. +# Vault BACKLOG #3121. The reconciler's and the sign-in re-sync's role change and disable name no +# administrator and close with "If you did not expect this change". Each carries its own source in +# its words, so neither takes the generic directory line: a re-sync's roles come from the +# directory's groups through the group-to-role map, which an administrator edits in the console, +# so "not from the MessageFoundry console" would be false for it. The disable says what the +# reconciler did, which is end the sessions; no MessageFoundry row was disabled. _DIRECTORY_DESCRIPTIONS = { - ROLES_CHANGED: "Your account's roles were changed.", + ROLES_CHANGED: ( + "MessageFoundry read your groups in your organization's directory and changed your " + "account's roles to match them." + ), ACCOUNT_DISABLED: ( "Your organization's directory reports your account as disabled, so MessageFoundry ended " "your sessions." ), } +# The administrator subject says the MessageFoundry account was disabled, which is false here. +_DIRECTORY_SUBJECTS = { + ACCOUNT_DISABLED: "Your MessageFoundry sessions were ended: your directory account is disabled", +} + + +def _from_directory(event: SecurityEvent) -> bool: + """Whether the directory, not the holder or an administrator, made this change (BACKLOG #1139, + #2291, vault #3121). Gated on the kind as well as the tag: the reconciler tags every notice it + sends, and its ``directory_sessions_ended`` must not claim the directory changed anything.""" + return ( + event.event_type in (EMAIL_CHANGED, USERNAME_CHANGED, *_DIRECTORY_DESCRIPTIONS) + and event.detail.get("source") == "directory" + ) + + +def _subject(event: SecurityEvent) -> str: + if _from_directory(event) and event.event_type in _DIRECTORY_SUBJECTS: + return _DIRECTORY_SUBJECTS[event.event_type] + return _SUBJECTS.get(event.event_type, "MessageFoundry security alert") _LOCAL_REMEDY = ( @@ -269,10 +293,7 @@ def _build_body(event: SecurityEvent) -> str: set_by_admin = ( event.event_type == NOTIFY_EMAIL_SET and event.detail.get("set_by") == "administrator" ) - # BACKLOG #1139, #2291: the directory made this change, not the holder and not the console. - # The sender's tag decides, whatever the kind (vault BACKLOG #3121), so a new directory notice - # cannot fall back to the administrator wording by missing from a list here. - from_directory = event.detail.get("source") == "directory" + from_directory = _from_directory(event) if moved_by_admin: description = ( "An administrator changed the address that receives security notices for your account." @@ -282,7 +303,6 @@ def _build_body(event: SecurityEvent) -> str: "An administrator set this address to receive security notices about your account." ) elif from_directory and event.event_type in _DIRECTORY_DESCRIPTIONS: - # Only these two kinds name an administrator in their own text; the others need no swap. description = _DIRECTORY_DESCRIPTIONS[event.event_type] else: description = _DESCRIPTIONS.get( @@ -389,11 +409,12 @@ def _build_body(event: SecurityEvent) -> str: scrub_log_argument(event.username), key, ) - if from_directory: + if from_directory and event.event_type not in _DIRECTORY_DESCRIPTIONS: # BACKLOG #1139. Say WHERE the change came from, because it changes what the reader can do # about it: a directory-driven change is not editable in the console, so "contact your # administrator" is the only action, and an unexplained change the holder cannot find a - # cause for reads as a compromise. + # cause for reads as a compromise. The vault #3121 kinds name their source in their own + # description instead (see ``_DIRECTORY_DESCRIPTIONS``). lines.append( "This change came from your organization's directory, not from the MessageFoundry " "console." @@ -499,9 +520,10 @@ def _build_body(event: SecurityEvent) -> str: DIRECTORY_SESSIONS_ENDED, ) ): - # A directory rename or email change is an administrator's act, so "if this was you" - # cannot apply to it (the email half is BACKLOG #2291), nor to a directory role change or - # disable (vault BACKLOG #3121). Nor to a federated link or unlink: + # The holder did not make a directory change, so "if this was you" cannot apply to a + # directory rename or email change (the email half is BACKLOG #2291), nor to a role + # change or disable the directory drove (vault BACKLOG #3121). Nor to a federated link or + # unlink: # both routes refuse an administrator changing their own account's binding. Nor to the # engine ending sessions on a directory answer (vault BACKLOG #2140). closing = "If you did not expect this change, contact your MessageFoundry administrator." @@ -677,7 +699,7 @@ def _send(self, event: SecurityEvent) -> None: port=self._port, sender=self._sender, recipients=[event.email], - subject=_SUBJECTS.get(event.event_type, "MessageFoundry security alert"), + subject=_subject(event), body=_build_body(event), use_tls=self._use_tls, username=self._username, diff --git a/tests/test_provision_first_administrator.py b/tests/test_provision_first_administrator.py index bc826e98d2..345f82a79a 100644 --- a/tests/test_provision_first_administrator.py +++ b/tests/test_provision_first_administrator.py @@ -13,7 +13,6 @@ from __future__ import annotations import asyncio -import base64 import io import json import sys @@ -2714,9 +2713,9 @@ def test_the_last_wrong_code_is_not_told_to_try_again( # --- vault BACKLOG #3113 item 1: the run that loses an enrolment race -------------------------- -#: A second synthetic base32 secret, so the two runs show different entries. Encoded here rather -#: than written as a literal, which the gitleaks generic-api-key rule reads as a credential. -_OTHER_TOTP_SECRET = base64.b32encode(b"1234567890").decode("ascii") +#: A second synthetic base32 secret, so the two runs show different entries. Two symbols on +#: purpose, for the reason ``_PASSWORD`` above is low-entropy: it must not look like a secret. +_OTHER_TOTP_SECRET = "ABABABABABABABAB" async def test_the_run_that_loses_an_enrolment_race_is_not_told_to_remove_a_live_entry( @@ -2770,9 +2769,15 @@ async def racing_enable(user_id: str, *, recovery_code_hashes: list[str]) -> boo row = await store.get_user("roleless") assert row is not None and row.totp_enabled assert await store.get_totp_secret("roleless") == _OTHER_TOTP_SECRET + # The winning run reported success over a secret it did not show. That is the race + # itself, vault BACKLOG #3025, pinned here as the premise and not as a wanted outcome. message = str(lost.value) assert "did not turn TOTP on and granted no role" in message assert "remove" not in message.lower() assert "keep the authenticator entry it showed" in message finally: + # A failure above must not leave the second run running against a closed store. + for task in tasks: + task.cancel() + await asyncio.gather(*tasks, return_exceptions=True) await store.close() diff --git a/tests/test_security_notify.py b/tests/test_security_notify.py index beea972807..7572319bae 100644 --- a/tests/test_security_notify.py +++ b/tests/test_security_notify.py @@ -37,6 +37,7 @@ _SUBJECTS, SecurityEventNotifier, _build_body, + _subject, security_notifier_from_settings, ) @@ -204,19 +205,38 @@ def test_a_non_last_factor_removal_renders_its_own_subject_and_body() -> None: assert "disabled" not in body.lower() -def test_a_directory_sessions_ended_notice_renders_its_own_text_and_claims_no_disable() -> None: +#: The generic line a directory email change or rename carries (BACKLOG #1139). +_DIRECTORY_LINE = ( + "This change came from your organization's directory, not from the MessageFoundry console." +) + + +@pytest.mark.parametrize( + "detail", + [ + pytest.param({}, id="untagged"), + # The shape the reconciler mails since vault BACKLOG #3121. + pytest.param({"reason": "directory_absent", "source": "directory"}, id="as-sent"), + ], +) +def test_a_directory_sessions_ended_notice_renders_its_own_text_and_claims_no_disable( + detail: dict[str, Any], +) -> None: """vault BACKLOG #2140. Both renderers fall back silently, so a half-wired kind would mail the generic wording. The directory never answered that the account is disabled or gone, so the text - must not say either, and an engine act cannot carry the "if this was you" closing.""" + must not say either, nor that the directory changed anything, and an engine act cannot carry + the "if this was you" closing.""" assert DIRECTORY_SESSIONS_ENDED in _SUBJECTS - body = _build_body(SecurityEvent(DIRECTORY_SESSIONS_ENDED, username="jdoe", email=None)) - text = (_SUBJECTS[DIRECTORY_SESSIONS_ENDED] + "\n" + body).lower() + event = SecurityEvent(DIRECTORY_SESSIONS_ENDED, username="jdoe", email=None, detail=detail) + body = _build_body(event) + text = (_subject(event) + "\n" + body).lower() assert "a security event occurred on your account." not in text # The description itself, not words the subject alone would satisfy (vault BACKLOG #3121). assert ( "A check against your organization's directory ended your MessageFoundry sessions." in body.splitlines() ) + assert _DIRECTORY_LINE not in body.splitlines() for claim in ("disabled", "deleted", "if this was you"): assert claim not in text assert body.endswith( @@ -226,15 +246,15 @@ def test_a_directory_sessions_ended_notice_renders_its_own_text_and_claims_no_di #: vault BACKLOG #3121. The description each directory-driven kind carries, pinned whole. _DIRECTORY_TEXT = { - ROLES_CHANGED: "Your account's roles were changed.", + ROLES_CHANGED: ( + "MessageFoundry read your groups in your organization's directory and changed your " + "account's roles to match them." + ), ACCOUNT_DISABLED: ( "Your organization's directory reports your account as disabled, so MessageFoundry ended " "your sessions." ), } -_DIRECTORY_LINE = ( - "This change came from your organization's directory, not from the MessageFoundry console." -) _DIRECTORY_CLOSING = "If you did not expect this change, contact your MessageFoundry administrator." _ADMIN_TEXT = { ROLES_CHANGED: "Your account's roles were changed by an administrator.", @@ -250,22 +270,24 @@ def test_a_directory_role_or_disable_notice_names_the_directory_and_no_administr kind: str, ) -> None: """vault BACKLOG #3121. The reconciler and the sign-in re-sync tag these ``source`` directory: - no administrator made the change and the holder did nothing, so neither the administrator - wording nor "if this was you" may appear.""" - body = _build_body( - SecurityEvent( - kind, - username="jdoe", - email="j@example.org", - detail={"reason": "directory_disabled", "source": "directory"}, - ) + the holder did nothing, so neither the administrator wording nor "if this was you" may appear. + Nor the generic directory line: a role re-sync runs through the group-to-role map an + administrator edits in the console, so "not from the MessageFoundry console" could be false.""" + event = SecurityEvent( + kind, + username="jdoe", + email="j@example.org", + detail={"reason": "directory_disabled", "source": "directory"}, ) + body = _build_body(event) lines = body.splitlines() assert _DIRECTORY_TEXT[kind] in lines - assert _DIRECTORY_LINE in lines + assert _DIRECTORY_LINE not in lines assert _ADMIN_TEXT[kind] not in lines assert "if this was you" not in body.lower() assert body.endswith(_DIRECTORY_CLOSING) + # The administrator subject says the MessageFoundry account was disabled, which is false here. + assert "was disabled" not in _subject(event) @pytest.mark.parametrize("kind", [ROLES_CHANGED, ACCOUNT_DISABLED]) From 1d5ff351a13851e093d97381d4ef4343aca46db4 Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Fri, 16 Oct 2026 17:14:59 -0500 Subject: [PATCH 07/57] D1 round-two follow-ups: the enrolment refusal row survives any revoke fault (BACKLOG #3113) Round-two findings 1, 3, 5 and 6 from the D1 review, carried to this branch. - Finding 1: confirm_mfa_enrollment writes auth.mfa_enroll_refused in a finally around the revoke, so a defect the soft write re-raises (NotImplementedError, sqlite3.ProgrammingError) no longer skips the row, and aiosqlite's closed-connection ValueError is logged as a store refusal, so the confirm still answers session_lost. The row's read and write move into _audit_enroll_refused. - Finding 3: the directory role notice says the sessions already open on the account were ended, as the disable notice does; both senders (reconciler pass, sign-in re-sync) revoke first. - Finding 5: the DIRECTORY_SESSIONS_ENDED comment in auth/notifications.py no longer says ACCOUNT_DISABLED names an administrator. - Finding 6: docs/SECURITY.md says the directory notice names no administrator as what made the change (its closing still names one), and no longer says flatly that the refused confirm ends the session. Findings 2, 4 and 7 stay open by brief. --- docs/SECURITY.md | 17 +++-- messagefoundry/auth/notifications.py | 4 +- messagefoundry/auth/service.py | 74 +++++++++++++--------- messagefoundry/pipeline/security_notify.py | 8 ++- tests/test_mfa.py | 52 ++++++++++++++- tests/test_security_notify.py | 4 +- 6 files changed, 112 insertions(+), 47 deletions(-) diff --git a/docs/SECURITY.md b/docs/SECURITY.md index f3fd6c4198..9aa5de2ebc 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -2769,12 +2769,13 @@ An administrator clears a lost authenticator via `POST /users/{id}/reset-mfa` (which also revokes the user's sessions). A confirm turns TOTP on only while a secret is still staged and TOTP is still off (BACKLOG #2224). A reset can clear the secret during the confirm, or another confirm can turn TOTP on -first. Either way the confirm turns nothing on and returns no recovery codes. It ends the session -it just rotated and answers as a lost session: `401` on `POST /me/mfa/confirm`, and the sign-in -page in the web console. It is audited `auth.mfa_enroll_refused`. Its `reason` is +first. Either way the confirm turns nothing on and returns no recovery codes. It asks the store to +end the session it just rotated, and answers as a lost session: `401` on `POST /me/mfa/confirm`, +and the sign-in page in the web console. It is audited `auth.mfa_enroll_refused`. Its `reason` is `secret_cleared`, `already_enabled` or `account_gone`. That reason is read after the refusal, so a later write can blur it. Its `session_ended` is false when the store refused to end the rotated -session; the token is still not handed back (vault BACKLOG #3113). The check is that some secret is +session. The token is still not handed back, and the row is written whatever the revoke raised +(vault BACKLOG #3113). The check is that some secret is staged, not that it is the one the confirm verified: a second enrolment start that re-stages the secret inside the same window is not caught by it. With `[security].require_mfa` on — **the default since BACKLOG #187 (secure-by-default, including the @@ -4387,13 +4388,15 @@ Users are notified of security-relevant changes to their account through **two** failed attempts**, or a step-up re-auth that clears such a run (suspicious-login signals, 6.3.5); and **password change**, **email change**, **role change**, **account disable**, and a **directory rename** of the username (credential changes, 6.3.7). - A notice for a change the directory drove names the directory and no administrator, and it closes - with "If you did not expect this change" rather than "If this was you". That covers at least the + A notice for a change the directory drove names the directory, not an administrator, as what made + the change. It closes with "If you did not expect this change, contact your MessageFoundry + administrator" rather than "If this was you". That covers at least the directory rename, the directory's email change, a role change applied by the session reconciler or by a directory sign-in's re-sync, and the reconciler's `account_disabled` when it read the account disabled (vault BACKLOG #3121). The role notice says the roles were changed to match the directory's groups, not that the console played no part: those groups map to roles through the - group-to-role map an administrator edits. When the reconciler ends an account's sessions because it + group-to-role map an administrator edits. Like the disable notice, it says the sessions already + open on the account were ended, which both of its senders do first. When the reconciler ends an account's sessions because it could not match the account or could not read its `userAccountControl`, it sends `directory_sessions_ended`, not `account_disabled` (vault BACKLOG #2140). The rename notice, `username_changed`, names the old and the new name, and is sent only when diff --git a/messagefoundry/auth/notifications.py b/messagefoundry/auth/notifications.py index 015e50cfc0..0a7015e8dc 100644 --- a/messagefoundry/auth/notifications.py +++ b/messagefoundry/auth/notifications.py @@ -76,8 +76,8 @@ # 6.3.7 -- the directory session reconciler ended the account's sessions for a reason that is NOT a # read disabled bit: the probe could not match the account (``directory_absent``: not found, or its # stored id did not match or could not be parsed), or its ``userAccountControl`` could not be read -# (vault BACKLOG #2140). ACCOUNT_DISABLED would tell the holder an administrator disabled the account, -# which nothing established. Which reason sends which kind is decided once, in +# (vault BACKLOG #2140). ACCOUNT_DISABLED would tell the holder the account is disabled, by an +# administrator or in the directory, which nothing established. Which reason sends which kind is decided once, in # ``AuthService._apply_reconcile_revocation``. DIRECTORY_SESSIONS_ENDED = "directory_sessions_ended" MFA_ENABLED = "mfa_enabled" # 6.3.7 — a second factor (TOTP) was enrolled on the account diff --git a/messagefoundry/auth/service.py b/messagefoundry/auth/service.py index 279c57b0e3..4b6eb9b71c 100644 --- a/messagefoundry/auth/service.py +++ b/messagefoundry/auth/service.py @@ -10157,38 +10157,27 @@ async def confirm_mfa_enrollment( # ended. The order is #1902's, rotate first and enable last, and is kept. Vault BACKLOG # #3113 item 8 is declined for that reason: enabling first would undo #1902. # - # A store refusal of the revoke is logged, not raised, so the refusal row below is - # still written (vault BACKLOG #3113). ``session_ended`` records what the revoke did. - # The token is not handed back either way, so the caller still cannot use the session; - # a session the revoke left live expires on its own timers. A fault that also refuses - # the read and the audit write below still loses the row: this covers a refusal of the - # revoke alone. + # A store refusal of the revoke is logged, not raised, so the confirm still answers + # ``session_lost`` (vault BACKLOG #3113). That covers aiosqlite's closed connection + # too, which it raises as a bare ``ValueError`` before sqlite3 is reached. A defect + # the helper re-raises still propagates, and the ``finally`` writes the refusal row + # first, so no exception the revoke raises can skip it. ``session_ended`` records + # what the revoke did. The token is not handed back either way, so the caller still + # cannot use the session; a session the revoke left live expires on its own timers. + # A fault that also refuses the read and the audit write below still loses the row. rotated = hash_token(elevation.token) - session_ended = await write_audit_soft( - lambda: self._store.revoke_session(rotated), - log=_log, - message="a refused TOTP enrolment for %s could not end the session it rotated", - args=lambda: (scrub_log_argument(identity.username),), - errors=_audit_write_errors(), - defects=AUDIT_WRITE_DEFECTS, - ) - # The reason is the row's state read AFTER the refusal, not the write's own verdict, - # so a later write can blur it; it is a pointer for an operator, not a proof. Vault - # BACKLOG #3113 item 9 is declined: the write's own verdict would need ``enable_totp`` - # to return a reason on all three store backends. - after = await self._load_user(identity.user_id) - if after is None: - reason = "account_gone" - elif after.totp_enabled: - reason = "already_enabled" - else: - reason = "secret_cleared" - await self._audit( - "auth.mfa_enroll_refused", - actor=identity.username, - detail=_json({"reason": reason, "session_ended": session_ended}), - client=client, - ) + session_ended = False + try: + session_ended = await write_audit_soft( + lambda: self._store.revoke_session(rotated), + log=_log, + message="a refused TOTP enrolment for %s could not end the session it rotated", + args=lambda: (scrub_log_argument(identity.username),), + errors=(*_audit_write_errors(), ValueError), + defects=AUDIT_WRITE_DEFECTS, + ) + finally: + await self._audit_enroll_refused(identity, session_ended, client=client) return Elevation(session_lost=True) await self._audit("auth.mfa_enrolled", actor=identity.username, client=client) # ADR 0197 Amendment A: enrolment now comes BEFORE the first rotation, so whoever intercepts @@ -10208,6 +10197,29 @@ async def confirm_mfa_enrollment( await self._mark_login_address_known(identity.user_id, client, enrolment=user) return elevation + async def _audit_enroll_refused( + self, identity: Identity, session_ended: bool, *, client: str | None + ) -> None: + """Write the ``auth.mfa_enroll_refused`` row for a confirm whose enable matched no row. + + The reason is the row's state read AFTER the refusal, not the write's own verdict, so a + later write can blur it; it is a pointer for an operator, not a proof. Vault BACKLOG #3113 + item 9 is declined: the write's own verdict would need ``enable_totp`` to return a reason + on all three store backends.""" + after = await self._load_user(identity.user_id) + if after is None: + reason = "account_gone" + elif after.totp_enabled: + reason = "already_enabled" + else: + reason = "secret_cleared" + await self._audit( + "auth.mfa_enroll_refused", + actor=identity.username, + detail=_json({"reason": reason, "session_ended": session_ended}), + client=client, + ) + async def verify_mfa( self, token: str | None, code: str, *, client: str | None = None ) -> Elevation: diff --git a/messagefoundry/pipeline/security_notify.py b/messagefoundry/pipeline/security_notify.py index e62f795d78..e06fa1ecf2 100644 --- a/messagefoundry/pipeline/security_notify.py +++ b/messagefoundry/pipeline/security_notify.py @@ -171,11 +171,13 @@ # its words, so neither takes the generic directory line: a re-sync's roles come from the # directory's groups through the group-to-role map, which an administrator edits in the console, # so "not from the MessageFoundry console" would be false for it. The disable says what the -# reconciler did, which is end the sessions; no MessageFoundry row was disabled. +# reconciler did, which is end the sessions; no MessageFoundry row was disabled. Both senders +# of the role change end the sessions already open on the account first, the reconciler's pass +# and the sign-in re-sync alike, so the role notice says so as the disable notice does. _DIRECTORY_DESCRIPTIONS = { ROLES_CHANGED: ( - "MessageFoundry read your groups in your organization's directory and changed your " - "account's roles to match them." + "MessageFoundry read your groups in your organization's directory, changed your " + "account's roles to match them, and ended any sessions already open on your account." ), ACCOUNT_DISABLED: ( "Your organization's directory reports your account as disabled, so MessageFoundry ended " diff --git a/tests/test_mfa.py b/tests/test_mfa.py index 92a43c5f45..0c4b56cb0b 100644 --- a/tests/test_mfa.py +++ b/tests/test_mfa.py @@ -2571,8 +2571,17 @@ async def other_confirm_first( await store.close() +@pytest.mark.parametrize( + "fault", + [ + RuntimeError("the store refused the revoke"), + # aiosqlite's own refusal of a closed connection, raised before sqlite3 is reached. + ValueError("Connection closed"), + ], + ids=["store-refusal", "closed-connection"], +) async def test_a_refused_enable_whose_revoke_raises_still_writes_its_refusal_row( - monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, fault: Exception ) -> None: """RED when: a store refusal of the revoke skips the ``auth.mfa_enroll_refused`` row (vault BACKLOG #3113 item 2), or the row claims the session ended when it did not (item 7).""" @@ -2591,7 +2600,7 @@ async def reset_then_enable( return await real_enable(user_id, recovery_code_hashes=recovery_code_hashes, now=now) async def refused_revoke(token_hash: str, *, now: float | None = None) -> None: - raise RuntimeError("the store refused the revoke") + raise fault monkeypatch.setattr(store, "enable_totp", reset_then_enable) monkeypatch.setattr(store, "revoke_session", refused_revoke) @@ -2615,6 +2624,45 @@ async def refused_revoke(token_hash: str, *, now: float | None = None) -> None: await store.close() +async def test_a_refused_enable_whose_revoke_hits_a_defect_still_writes_its_refusal_row( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """RED when: a defect the revoke raises, which the soft write re-raises rather than logs, skips + the ``auth.mfa_enroll_refused`` row (vault BACKLOG #3113 item 2, round two). The defect still + propagates, so a bug is not passed over as a store refusal.""" + store = await _store() + try: + service = AuthService(store, AuthSettings(mfa_verify_min_elapsed_seconds=0)) + identity, token, _ = await login_admin(service) + enroll = await service.begin_mfa_enrollment(identity) + + real_enable = store.enable_totp + + async def reset_then_enable( + user_id: str, *, recovery_code_hashes: list[str], now: float | None = None + ) -> bool: + await store.disable_totp(user_id) + return await real_enable(user_id, recovery_code_hashes=recovery_code_hashes, now=now) + + async def broken_revoke(token_hash: str, *, now: float | None = None) -> None: + raise NotImplementedError("a backend without the revoke") + + monkeypatch.setattr(store, "enable_totp", reset_then_enable) + monkeypatch.setattr(store, "revoke_session", broken_revoke) + with pytest.raises(NotImplementedError): + await service.confirm_mfa_enrollment(identity, fresh_totp(enroll.secret), token=token) + rows = await store.list_audit( + actor=ADMIN_USERNAME, action="auth.mfa_enroll_refused", limit=10 + ) + assert len(rows) == 1, "the refusal row was skipped when the revoke raised a defect" + assert json.loads(str(rows[0]["detail"])) == { + "reason": "secret_cleared", + "session_ended": False, + } + finally: + await store.close() + + # --- ADR 0197 Amendment A, wave 1 (BACKLOG #1131): every account is born with a way past ---------- # # A lock exists to bound guesses. On an engine-generated 192-bit credential it bounds nothing, so diff --git a/tests/test_security_notify.py b/tests/test_security_notify.py index 7572319bae..f9c5a3a124 100644 --- a/tests/test_security_notify.py +++ b/tests/test_security_notify.py @@ -247,8 +247,8 @@ def test_a_directory_sessions_ended_notice_renders_its_own_text_and_claims_no_di #: vault BACKLOG #3121. The description each directory-driven kind carries, pinned whole. _DIRECTORY_TEXT = { ROLES_CHANGED: ( - "MessageFoundry read your groups in your organization's directory and changed your " - "account's roles to match them." + "MessageFoundry read your groups in your organization's directory, changed your " + "account's roles to match them, and ended any sessions already open on your account." ), ACCOUNT_DISABLED: ( "Your organization's directory reports your account as disabled, so MessageFoundry ended " From 3010fd98f95e67837219233ea86f4dc681fcfd1d Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Fri, 16 Oct 2026 17:17:39 -0500 Subject: [PATCH 08/57] The nested group search is paged, and an unknown LDAP result code is refused, not a KeyError (BACKLOG #3010) Follow-up limb of #3010, findings 3, 6, 8 and the ldap side of 10 from the first Builder's review. - Finding 3: the nested group search goes through new _search_paged, the simple paged results control at 500 per page. Each page passes _search, so a page that does not end with success refuses the whole search and the pages already read are dropped. A search still paging after 20 pages is refused too (each page may take up to ad_receive_timeout). - Finding 6, verified in the installed ldap3 2.9.1: the default fast decoder does RESULT_CODES[code] (utils/asn1.py ldap_result_to_dict_fast, operation/bind.py), so an unknown code raises KeyError(code) from inside conn.search or a bind. _unknown_result_code recognises only that shape; _search and _socket_faults_as_ldap_error map it to LdapUnansweredError. A real-ldap3 loopback test measures the premise. - Finding 8: the test module docstring is in the conditional. - Finding 10 (ldap side): the group-search-busy reconciler case is its own test, reading roles through a mapped group; the stored roles survive. --- changelog.d/3010.security.md | 14 +- messagefoundry/auth/ldap.py | 119 +++++- tests/test_ldap_search_result_codes.py | 505 +++++++++++++++++++++++-- 3 files changed, 591 insertions(+), 47 deletions(-) diff --git a/changelog.d/3010.security.md b/changelog.d/3010.security.md index 7ae138d201..369267a621 100644 --- a/changelog.d/3010.security.md +++ b/changelog.d/3010.security.md @@ -8,7 +8,13 @@ No directory diagnostic text reaches the error, the log or the audit row. **The cost:** a code that recurs on every pass, such as a deleted search base, now stops revocation and the role re-diff for as long as it lasts, with one WARNING per code per process - and no latched alert. The nested group search is not paged. Where a directory caps an unpaged - search (AD's MaxPageSize, 1000 by default), an account in more groups than the cap would end - it with sizeLimitExceeded and could no longer sign in, where it used to sign in with a truncated - group list. That was not measured against a domain controller. (vault `BACKLOG #3010`) + and no latched alert. + **The nested group search is paged** (the simple paged results control, 500 per page), so an + account in more groups than a directory's unpaged cap (AD's MaxPageSize, 1000 by default) would + not end it with sizeLimitExceeded and fail sign-in. A page that does not end with success refuses + the whole search, and so does a search still paging after 20 pages; the pages already read are + never used as the whole answer. Paging was measured against a loopback server through ldap3, not + against a domain controller. + **A result code ldap3 has no name for is the same refusal.** ldap3's default decoder raises + `KeyError` on one from inside a search or a bind, which would have surfaced as an unhandled + error at sign-in; it now reads as the directory not answering. (vault `BACKLOG #3010`) diff --git a/messagefoundry/auth/ldap.py b/messagefoundry/auth/ldap.py index b42787fe5d..4470584f20 100644 --- a/messagefoundry/auth/ldap.py +++ b/messagefoundry/auth/ldap.py @@ -814,7 +814,7 @@ def _refuse_unanswered(conn: Any, operation: str, *, answers: frozenset[int] | N with its own set. A referral is refused first, as a referral, whatever ``answers`` holds, so the reconciler still reads it as referred (BACKLOG #2538). """ - from ldap3.core.results import RESULT_CODES, RESULT_SUCCESS # lazy, like every ldap3 import + from ldap3.core.results import RESULT_SUCCESS # lazy, like every ldap3 import here _refuse_referral(conn, operation) result = conn.result @@ -822,9 +822,20 @@ def _refuse_unanswered(conn: Any, operation: str, *, answers: frozenset[int] | N return code = result.get("result") if type(code) is not int: # not isinstance: a bool is an int, and is no result code - code, name, shown, hint = None, "no code", "no readable result code", "" - elif code in (answers if answers is not None else (RESULT_SUCCESS,)): - return + raise _unanswered(operation, None) + if code not in (answers if answers is not None else (RESULT_SUCCESS,)): + raise _unanswered(operation, code) + + +def _unanswered(operation: str, code: int | None) -> LdapUnansweredError: + """The refusal for ``operation`` ending with ``code``, warned once per operation and code name. + + The text carries the code, ldap3's name for it and the operation, never directory text. + """ + from ldap3.core.results import RESULT_CODES # lazy, like every ldap3 import here + + if code is None: + name, shown, hint = "no code", "no readable result code", "" else: name = RESULT_CODES.get(code, "an unknown result code") shown, hint = f"result code {code} ({name})", _UNANSWERED_HINTS.get(code, "") @@ -835,17 +846,97 @@ def _refuse_unanswered(conn: Any, operation: str, *, answers: frozenset[int] | N # Keyed on ldap3's name, not the raw code: a directory chooses the code, and ldap3's table is # finite, so the set of warnings stays bounded. _warn_once(f"unanswered: {operation}: {name}", "%s Reported once per code.", message) - raise LdapUnansweredError(message, code=code, code_name=name) + return LdapUnansweredError(message, code=code, code_name=name) def _search(conn: Any, operation: str, **kwargs: Any) -> None: """``conn.search(**kwargs)``, refusing a referral result and any other result but success (:func:`_refuse_unanswered`). Every search in this module goes through here, so none can read - either as "no entries"; a test pins that.""" - conn.search(**kwargs) + either as "no entries"; a test pins that. + + A result code ldap3 has no name for is the same refusal; :func:`_unknown_result_code` says + why it never reaches :func:`_refuse_unanswered`. + """ + try: + conn.search(**kwargs) + except KeyError as exc: + code = _unknown_result_code(exc) + if code is None: + raise + raise _unanswered(operation, code) from exc _refuse_unanswered(conn, operation) +def _unknown_result_code(exc: KeyError) -> int | None: + """The result code ``exc`` names when it is ldap3 failing to name one, else ``None``. + + BACKLOG #3010. ldap3 2.9.1 decodes with its fast decoder by default + (``Connection(fast_decoder=True)``), and that decoder looks a bind, search or extended result's + code up in ``RESULT_CODES`` (``utils/asn1.py``, ``ldap_result_to_dict_fast``, and + ``operation/bind.py``), so a code outside the table raises ``KeyError(code)`` from inside the + ldap3 call. Unmapped, that would escape every ``except LdapError`` and surface as an unhandled + error at sign-in. Only a ``KeyError`` whose sole argument is an integer missing from that table + is ldap3's; any other is the engine's own defect, and the caller lets it propagate. + """ + from ldap3.core.results import RESULT_CODES # lazy, like every ldap3 import here + + code = exc.args[0] if len(exc.args) == 1 else None + return code if type(code) is int and code not in RESULT_CODES else None + + +#: The simple paged results control (RFC 2696), which AD and ldap3 both name by this OID. +_PAGED_RESULTS_OID = "1.2.840.113556.1.4.319" + +#: Entries asked for per page of a search that can return many. Below AD's default MaxPageSize +#: (1000), the cap an unpaged search ends at with sizeLimitExceeded. A directory whose cap is lower +#: returns smaller pages, which is still the whole answer. +_PAGE_SIZE = 500 + +#: Pages read before a paged search is refused as unfinished: 10,000 entries. Each page may take up +#: to ``[auth].ad_receive_timeout``, so a directory that keeps returning a cookie would otherwise +#: hold the sign-in or reconcile worker for as long as it keeps answering. +_MAX_PAGES = 20 + + +def _paged_cookie(conn: Any) -> bytes | None: + """The cookie ``conn``'s last search returned for the next page, or ``None`` at the last page. + + ldap3 decodes the control into ``result['controls'][oid]['value']['cookie']``. Absent, the + directory ignored the request for pages, which is allowed because it is sent non-critical; that + answer is then the whole answer or ends with a code :func:`_refuse_unanswered` refuses. + """ + result = conn.result + controls = result.get("controls") if isinstance(result, dict) else None + control = controls.get(_PAGED_RESULTS_OID) if isinstance(controls, dict) else None + value = control.get("value") if isinstance(control, dict) else None + cookie = value.get("cookie") if isinstance(value, dict) else None + return cookie if isinstance(cookie, bytes) and cookie else None + + +def _search_paged(conn: Any, operation: str, **kwargs: Any) -> list[Any]: + """Every entry of a search that can outrun a directory's unpaged cap, read page by page. + + BACKLOG #3010. Each page goes through :func:`_search`, so a page that does not end with success + refuses the whole search, and the pages already read are dropped with it: a part of the answer + is never used as the whole. Unpaged, an account in more groups than the cap would end the + nested group search with sizeLimitExceeded, which that refusal turns into a failed sign-in. + """ + entries: list[Any] = [] + cookie: bytes | None = None + for _ in range(_MAX_PAGES): + _search(conn, operation, paged_size=_PAGE_SIZE, paged_cookie=cookie, **kwargs) + entries.extend(conn.entries) + cookie = _paged_cookie(conn) + if cookie is None: + return entries + message = ( + f"AD returned more than {_MAX_PAGES} pages of {_PAGE_SIZE} for the {operation}; the engine " + "reads it as the directory not answering (BACKLOG #3010)." + ) + _warn_once(f"unanswered: {operation}: pages", "%s Reported once.", message) + raise LdapUnansweredError(message) + + #: What opening an ldap3 socket can raise that is not an ldap3 error (BACKLOG #2566). ldap3's #: ``_open_socket`` wraps a ``socket.error`` in its own ``LDAPSocketOpenError``, but not what #: ``settimeout`` and ``setsockopt`` raise on a bad value: ``OverflowError``, ``TypeError``, or the @@ -880,6 +971,10 @@ def _socket_faults_as_ldap_error() -> Iterator[None]: of ``tests/test_ldap_referrals.py`` build that connection, so one would fail there first. And ``authenticate`` wraps the whole decoy bind, which must neither map nor swallow by itself. + It also maps a bind that ended with a result code ldap3 has no name for + (:func:`_unknown_result_code`) to the refusal a search gets for one, so it reads as an outage + and not as an unhandled error. Any other ``KeyError`` propagates. + An ``LDAPException`` passes through untouched, even one that is also an ``OSError``, ``TypeError`` or ``ValueError``, so its own handler keeps its own text. The new text is fixed and names only the type: a fault's message is not the engine's to repeat. **That holds only for @@ -902,6 +997,11 @@ def _socket_faults_as_ldap_error() -> Iterator[None]: else f"{kind.__module__}.{kind.__qualname__}" ) raise LdapError(f"AD directory call failed: {name}") from exc + except KeyError as exc: + code = _unknown_result_code(exc) + if code is None: + raise + raise _unanswered("bind", code) from exc class LdapAuthenticator: @@ -1312,7 +1412,8 @@ def _resolve_groups(self, conn: Any, user_dn: str, member_of: list[str]) -> froz if self._s.ad_use_nested_groups and self._s.ad_group_search_base: # A search the directory did not finish raises here (BACKLOG #3010), never a shorter # group list. - _search( + # Paged, so an account in more groups than the directory's unpaged cap still resolves. + groups = _search_paged( conn, "group search", search_base=self._s.ad_group_search_base, @@ -1320,7 +1421,7 @@ def _resolve_groups(self, conn: Any, user_dn: str, member_of: list[str]) -> froz search_scope=ldap3.SUBTREE, attributes=["distinguishedName"], ) - found.extend(str(e.entry_dn) for e in conn.entries) + found.extend(str(e.entry_dn) for e in groups) canonical = [canonical_group_dn(dn) for dn in found] dropped = canonical.count(None) if dropped: diff --git a/tests/test_ldap_search_result_codes.py b/tests/test_ldap_search_result_codes.py index fb9ee39c31..cd1b2899d7 100644 --- a/tests/test_ldap_search_result_codes.py +++ b/tests/test_ldap_search_result_codes.py @@ -3,17 +3,24 @@ """BACKLOG #3010: a search that ends with any result code but success is the directory not answering, never "no such entry". -ldap3 raises on none of these codes here (every connection keeps ``raise_exceptions=False``), so a -busy or admin-limit-exceeded search came back with no entries. ``_search_user`` read that as -``NOT_FOUND``, the session reconciler read ``NOT_FOUND`` as ``ABSENT``, and after the strike -threshold it revoked a present account. The nested group search read the same answer as fewer -groups. ``_search`` now refuses every code but success with :class:`LdapUnansweredError`, which -every caller already reads as an outage. - -Driven at three depths: ``_search`` itself over a bare double; the REAL ``LdapAuthenticator`` over +ldap3 raises on none of these codes here (every connection keeps ``raise_exceptions=False``), so in +the code before this change a busy or admin-limit-exceeded search would come back with no entries. +``_search_user`` would read that as ``NOT_FOUND``, the session reconciler would read ``NOT_FOUND`` +as ``ABSENT``, and after the strike threshold it would revoke a present account. The nested group +search would read the same answer as fewer groups. ``_search`` now refuses every code but success +with :class:`LdapUnansweredError`, which every caller already reads as an outage. + +Two follow-ups ride here. The nested group search is paged, so an account in more groups than a +directory's unpaged cap would not end it with sizeLimitExceeded, and a page that does not end with +success refuses the whole search. And a result code ldap3 has no name for, which its default fast +decoder raises ``KeyError`` on from inside ``conn.search``, is mapped to the same refusal. + +Driven at four depths: ``_search`` itself over a bare double; the REAL ``LdapAuthenticator`` over the ldap3 doubles of ``tests/test_ad_user_account_control.py`` and ``tests/test_ldap_timeouts.py``; -and the REAL ``AuthService`` reconciler and Kerberos sign-in over those. Each refusal has a control -on the same doubles: success with no entries still reads absent, and still revokes. +the REAL ``AuthService`` reconciler and Kerberos sign-in over those; and a REAL ldap3 stack against +a loopback server, for the paging control and the unknown code, which no double can reproduce. Each +refusal has a control on the same doubles: success with no entries still reads absent, and still +revokes. **Not exercised:** a real domain controller. The doubles set the result dict the way ldap3 2.9.1's synchronous strategy leaves it. Synthetic data only; nothing here touches PHI. @@ -21,10 +28,13 @@ from __future__ import annotations +import contextlib import copy import logging import pickle -from collections.abc import Callable +import socket +import threading +from collections.abc import Callable, Iterator from types import SimpleNamespace from typing import Any @@ -33,6 +43,7 @@ from ldap3.core.results import ( RESULT_ADMIN_LIMIT_EXCEEDED, RESULT_BUSY, + RESULT_CODES, RESULT_INSUFFICIENT_ACCESS_RIGHTS, RESULT_INVALID_DN_SYNTAX, RESULT_NO_SUCH_OBJECT, @@ -53,10 +64,19 @@ LdapReferralError, LdapUnansweredError, ) +from messagefoundry.auth.permissions import Role from messagefoundry.auth.reconcile import ProbeOutcome from messagefoundry.auth.service import AuthService from messagefoundry.store.store import MessageStore from tests.test_ad_user_account_control import _alive, _audited, _pass, _signed_in_estate +from tests.test_ldap_referrals import ( + _BIND_REQUEST, + _BIND_RESPONSE, + _SEARCH_DONE, + _SEARCH_REQUEST, + _read_message, + _tlv, +) from tests.test_ldap_timeouts import _ad_settings, _install_fakes #: Stands for text the directory sends with a result: its diagnostic message and matched DN. It @@ -65,6 +85,9 @@ _GROUPS = "OU=Groups,DC=test,DC=invalid" +#: A group only the nested group search returns, which the role test maps to the operator role. +_OPERATORS = f"CN=Operators,{_GROUPS}" + #: The codes the row names, and others a directory can end a search with. None is an answer. _NOT_ANSWERS = [ pytest.param(RESULT_BUSY, "busy", id="51-busy"), @@ -286,34 +309,24 @@ def test_a_group_search_the_directory_did_not_finish_is_an_outage_not_fewer_grou @pytest.mark.parametrize( - ("estate", "when", "code", "partial"), + ("code", "partial"), [ - ({}, _user_search, RESULT_BUSY, False), - ({}, _user_search, RESULT_ADMIN_LIMIT_EXCEEDED, False), - ({}, _user_search, RESULT_NO_SUCH_OBJECT, False), - ({}, _user_search, RESULT_SIZE_LIMIT_EXCEEDED, True), - ({"ad_group_search_base": _GROUPS}, _group_search, RESULT_BUSY, False), + (RESULT_BUSY, False), + (RESULT_ADMIN_LIMIT_EXCEEDED, False), + (RESULT_NO_SUCH_OBJECT, False), + (RESULT_SIZE_LIMIT_EXCEEDED, True), ], - ids=["busy", "adminLimitExceeded", "noSuchObject", "sizeLimit-partial", "group-search-busy"], + ids=["busy", "adminLimitExceeded", "noSuchObject", "sizeLimit-partial"], ) async def test_a_directory_that_did_not_answer_never_revokes_a_present_account( - monkeypatch: pytest.MonkeyPatch, - estate: dict[str, Any], - when: Callable[[dict[str, Any]], bool], - code: int, - partial: bool, + monkeypatch: pytest.MonkeyPatch, code: int, partial: bool ) -> None: - """The row's defect, end to end. One signed-in account, under the breaker's floor, whose search - the directory answers with ``code`` on every pass. Before the fix the user search read ABSENT - and was revoked at the strike threshold, and the nested group search read as fewer groups, - which feeds the role re-diff. Now each probe is unavailable and the session survives.""" - async with _signed_in_estate(monkeypatch, ["jdoe"], **estate) as ( - _directory, - service, - store, - tokens, - ): - _answer_with(monkeypatch, code, when=when, keep_entries=partial) + """The row's defect, end to end. One signed-in account, under the breaker's floor, whose user + search the directory answers with ``code`` on every pass. Before the fix the user search read + ABSENT and the account would be revoked at the strike threshold. Now each probe is unavailable + and the session survives. The group search has its own test below, which also reads roles.""" + async with _signed_in_estate(monkeypatch, ["jdoe"]) as (_directory, service, store, tokens): + _answer_with(monkeypatch, code, when=_user_search, keep_entries=partial) user = await store.get_user_by_username("jdoe") assert user is not None assert (await service._probe_principal(user)).outcome is ProbeOutcome.UNAVAILABLE @@ -358,6 +371,430 @@ async def test_a_busy_account_beside_a_departed_one_is_left_alone_while_the_othe assert await _alive(service, tokens) == {"jdoe"} +def _groups_answer(monkeypatch: pytest.MonkeyPatch, dns: list[str]) -> None: + """Over the ldap3 double already installed, answer every nested group search with ``dns``.""" + base = ldap3.Connection + + class Grouped(base): # type: ignore[misc, valid-type] + def search(self, **kwargs: Any) -> bool: + answered: bool = super().search(**kwargs) + if _group_search(kwargs): + self.entries = [SimpleNamespace(entry_dn=dn) for dn in dns] + return answered + + monkeypatch.setattr(ldap3, "Connection", Grouped) + + +async def test_a_group_search_the_directory_did_not_finish_never_narrows_a_signed_in_role( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The nested group search, end to end, reading roles. One signed-in account holds the operator + role through a group only that search returns. Read as fewer groups, as the code before #3010 + would, a busy group search would find the role withdrawn and revoke the session as + ``roles_changed``. Now each probe is unavailable, the stored roles are not re-diffed, and the + session survives. The control runs first, on the same doubles: readable, the directory grants + what is stored, so a pass changes nothing.""" + async with _signed_in_estate(monkeypatch, ["jdoe"], ad_group_search_base=_GROUPS) as ( + _directory, + service, + store, + tokens, + ): + _groups_answer(monkeypatch, [_OPERATORS]) + canonical = ldap_module.canonical_group_dn(_OPERATORS) + assert canonical is not None + await store.set_ad_group_role_map([(canonical, Role.OPERATOR.value)]) + user = await store.get_user_by_username("jdoe") + assert user is not None + await store.set_user_roles(user.id, [Role.OPERATOR.value]) + assert (await service._probe_principal(user)).outcome is ProbeOutcome.PRESENT + assert (await _pass(service)).revocations == () + + _answer_with(monkeypatch, RESULT_BUSY, when=_group_search) + assert (await service._probe_principal(user)).outcome is ProbeOutcome.UNAVAILABLE + for _ in range(5): # far past ad_session_recheck_strikes + assert (await _pass(service)).revocations == () + assert await store.get_user_role_ids(user.id) == [Role.OPERATOR.value] + assert await _alive(service, tokens) == {"jdoe"} + assert await _audited(store, "auth.ad_session_revoked") == [] + + +# --- the nested group search is paged --------------------------------------------------------------- + + +def _page(code: int, entries: list[object], cookie: bytes | None) -> dict[str, Any]: + """A searchResDone carrying the paged results control, as ldap3's fast decoder leaves it, with + the page's entries under ``_entries`` for :func:`_replay`.""" + result = _result(code) + if cookie is not None: + value = {"size": 0, "cookie": cookie} + result["controls"] = {ldap_module._PAGED_RESULTS_OID: {"value": value}} + result["_entries"] = entries + return result + + +def _replay(conn: Any, page: dict[str, Any], wrap: Callable[[Any], object] = lambda e: e) -> bool: + """Leave ``conn`` as a search answered with ``page`` leaves it, each entry passed to ``wrap``.""" + result = dict(page) + conn.entries = [wrap(e) for e in result.pop("_entries")] + conn.result = result + return bool(result["result"] == RESULT_SUCCESS) + + +class _PagedConn: + """A bare double that answers its n-th search with ``pages[n]``, recording each search.""" + + def __init__(self, pages: list[dict[str, Any]]) -> None: + self._pages = pages + self.calls: list[dict[str, Any]] = [] + self.result: dict[str, Any] | None = None + self.entries: list[object] = [] + + def search(self, **kwargs: Any) -> bool: + self.calls.append(kwargs) + return _replay(self, self._pages[min(len(self.calls), len(self._pages)) - 1]) + + +def test_a_paged_search_reads_every_page_and_sends_each_cookie_back() -> None: + a, b, c = object(), object(), object() + conn = _PagedConn( + [ + _page(RESULT_SUCCESS, [a], b"synthetic-cookie-1"), + _page(RESULT_SUCCESS, [b], b"synthetic-cookie-2"), + _page(RESULT_SUCCESS, [c], b""), # an empty cookie marks the last page (RFC 2696) + ] + ) + assert ldap_module._search_paged(conn, "group search", search_base=_GROUPS) == [a, b, c] + assert [call["paged_cookie"] for call in conn.calls] == [ + None, + b"synthetic-cookie-1", + b"synthetic-cookie-2", + ] + assert {call["paged_size"] for call in conn.calls} == {ldap_module._PAGE_SIZE} + assert all(call["search_base"] == _GROUPS for call in conn.calls) + + +def test_a_directory_that_ignores_the_page_request_is_read_once() -> None: + """The control is sent non-critical, so a directory may answer without it. That answer ended + with success, so it is the whole answer.""" + entry = object() + conn = _PagedConn([_page(RESULT_SUCCESS, [entry], None)]) + assert ldap_module._search_paged(conn, "group search") == [entry] + assert len(conn.calls) == 1 + + +@pytest.mark.parametrize( + ("code", "partial"), + [(RESULT_BUSY, False), (RESULT_SIZE_LIMIT_EXCEEDED, True), (4242, False)], + ids=["51-busy", "4-sizeLimitExceeded-partial", "4242-unknown"], +) +def test_a_later_page_that_did_not_end_with_success_refuses_the_whole_search( + code: int, partial: bool +) -> None: + """No silent truncation: the first page's entries are not returned as the whole answer.""" + conn = _PagedConn( + [ + _page(RESULT_SUCCESS, [object()], b"synthetic-cookie-1"), + _page(code, [object()] if partial else [], b"synthetic-cookie-2"), + ] + ) + with pytest.raises(LdapUnansweredError, match="group search") as refused: + ldap_module._search_paged(conn, "group search") + assert refused.value.code == code + assert len(conn.calls) == 2 + + +def test_a_directory_that_never_stops_paging_is_refused(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(ldap_module, "_MAX_PAGES", 3) + conn = _PagedConn([_page(RESULT_SUCCESS, [object()], b"synthetic-again")]) + with pytest.raises(LdapUnansweredError, match="more than 3 pages"): + ldap_module._search_paged(conn, "group search") + assert len(conn.calls) == 3 + + +@pytest.mark.parametrize( + "result", + [ + None, + {"result": 0}, + {"result": 0, "controls": None}, + {"result": 0, "controls": {ldap_module._PAGED_RESULTS_OID: None}}, + {"result": 0, "controls": {ldap_module._PAGED_RESULTS_OID: {"value": {"cookie": "x"}}}}, + ], + ids=["no-result", "no-controls", "controls-none", "control-none", "cookie-not-bytes"], +) +def test_a_cookie_that_is_absent_or_unreadable_ends_the_paging(result: object) -> None: + assert ldap_module._paged_cookie(SimpleNamespace(result=result)) is None + + +def _paged_groups(monkeypatch: pytest.MonkeyPatch, pages: list[dict[str, Any]]) -> list[object]: + """Over the ``_install_fakes`` doubles, answer the n-th group search with ``pages[n]``. Returns + the cookies each group search sent.""" + base = ldap3.Connection + sent: list[object] = [] + + class Paging(base): # type: ignore[misc, valid-type] + def search(self, **kwargs: Any) -> bool: + answered: bool = super().search(**kwargs) + if not _group_search(kwargs): + return answered + sent.append(kwargs.get("paged_cookie")) + return _replay(self, pages[len(sent) - 1], lambda dn: SimpleNamespace(entry_dn=dn)) + + monkeypatch.setattr(ldap3, "Connection", Paging) + return sent + + +def _group_dn(n: int) -> str: + return f"CN=Synthetic Group {n},OU=Groups,DC=example,DC=com" + + +def test_an_account_in_more_groups_than_one_page_resolves_every_group( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Unpaged, a directory that caps a search would end this one with sizeLimitExceeded, which + the refusal turns into a failed sign-in. Paged, every page is read.""" + _install_fakes(monkeypatch) + sent = _paged_groups( + monkeypatch, + [ + _page(RESULT_SUCCESS, [_group_dn(1), _group_dn(2)], b"synthetic-cookie-1"), + _page(RESULT_SUCCESS, [_group_dn(3)], b""), + ], + ) + principal = LdapAuthenticator(_ad_settings()).authenticate("alice", "synthetic-pw").principal + assert principal is not None + for n in (1, 2, 3): + assert ldap_module.canonical_group_dn(_group_dn(n)) in principal.groups + assert sent == [None, b"synthetic-cookie-1"] + + +def test_a_busy_later_page_refuses_the_sign_in_rather_than_dropping_its_groups( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _install_fakes(monkeypatch) + _paged_groups( + monkeypatch, + [ + _page(RESULT_SUCCESS, [_group_dn(1)], b"synthetic-cookie-1"), + _page(RESULT_BUSY, [], None), + ], + ) + with pytest.raises(LdapUnansweredError, match=r"group search with result code 51 \(busy\)"): + LdapAuthenticator(_ad_settings()).authenticate("alice", "synthetic-pw") + + +# --- a result code ldap3 has no name for ----------------------------------------------------------- + + +def _raising(*args: object) -> Callable[..., bool]: + def call(*_a: object, **_kw: object) -> bool: + raise KeyError(*args) + + return call + + +def test_a_key_error_naming_an_unknown_code_is_the_same_refusal() -> None: + """ldap3's fast decoder raises ``KeyError(code)`` for a code outside ``RESULT_CODES``.""" + conn = SimpleNamespace(search=_raising(4242), result=None, entries=[]) + with pytest.raises(LdapUnansweredError, match="an unknown result code") as refused: + ldap_module._search(conn, "user search") + assert refused.value.code == 4242 + assert isinstance(refused.value.__cause__, KeyError) + + +def test_a_bind_ending_with_an_unknown_code_is_the_same_refusal() -> None: + """The decoder raises the same ``KeyError`` on a bind response, so the wrap every bind runs in + maps it too.""" + with ( + pytest.raises(LdapUnansweredError, match="bind with result code 4242") as refused, + ldap_module._socket_faults_as_ldap_error(), + ): + _raising(4242)() + assert refused.value.code == 4242 + + +_OTHER_KEY_ERRORS = pytest.mark.parametrize( + "args", + [("objectGUID",), (RESULT_BUSY,), (True,), (), (4242, "x")], + ids=["a-name", "a-known-code", "a-bool", "no-args", "two-args"], +) + + +@_OTHER_KEY_ERRORS +def test_any_other_key_error_is_the_engines_own_and_propagates(args: tuple[object, ...]) -> None: + """Mapped, an engine defect would read as an outage, which the reconciler logs at debug.""" + conn = SimpleNamespace(search=_raising(*args), result=None, entries=[]) + with pytest.raises(KeyError) as raised: + ldap_module._search(conn, "user search") + assert not isinstance(raised.value, LdapError) + with pytest.raises(KeyError) as raised, ldap_module._socket_faults_as_ldap_error(): + _raising(*args)() + assert not isinstance(raised.value, LdapError) + + +# --- through a real ldap3 stack: the paging control and the unknown code -------------------------- + +_IN_CHAIN = b"1.2.840.113556.1.4.1941" # the nested group search's matching rule, on the wire + +_USER_DN = "CN=alice,OU=Users,DC=example,DC=com" + +#: One message a loopback server sends: its protocolOp, and its controls (empty for none). +_Reply = tuple[bytes, bytes] + + +def _enumerated(code: int) -> bytes: + """A non-negative resultCode as BER ENUMERATED content, two's complement, minimal. + ``tests/test_ldap_referrals.py``'s ``_ldap_result`` writes one byte, so it cannot carry 4242.""" + return code.to_bytes(max(1, (code.bit_length() + 8) // 8), "big") + + +def _done(code: int, cookie: bytes | None = None, *, tag: int = _SEARCH_DONE) -> _Reply: + """An LDAPResult with ``code``, and the paged results control when ``cookie`` is given.""" + op = _tlv(tag, _tlv(0x0A, _enumerated(code)) + _tlv(0x04, b"") + _tlv(0x04, b"")) + if cookie is None: + return op, b"" + value = _tlv(0x30, _tlv(0x02, b"\x00") + _tlv(0x04, cookie)) + control = _tlv(0x30, _tlv(0x04, ldap_module._PAGED_RESULTS_OID.encode()) + _tlv(0x04, value)) + return op, _tlv(0xA0, control) + + +def _entry(dn: str) -> _Reply: + return _tlv(0x64, _tlv(0x04, dn.encode()) + _tlv(0x30, b"")), b"" + + +class _ScriptedDc: + """A loopback LDAP server, after ``tests/test_ldap_referrals.py``'s ``_FakeDc``. Binds end with + ``bind_code``; each search is answered by ``answer(raw request)``. ``searches`` holds every + search request's bytes.""" + + def __init__(self, answer: Callable[[bytes], list[_Reply]], bind_code: int) -> None: + self._answer, self._bind_code = answer, bind_code + self.searches: list[bytes] = [] + self._listener = socket.create_server(("127.0.0.1", 0)) + self.port: int = self._listener.getsockname()[1] + threading.Thread(target=self._accept, daemon=True).start() + + def _accept(self) -> None: + while True: + try: + sock, _addr = self._listener.accept() + except OSError: + return # the listener was closed + threading.Thread(target=self._session, args=(sock,), daemon=True).start() + + def _session(self, sock: socket.socket) -> None: + with sock, contextlib.suppress(OSError): + sock.settimeout(10) + while (message := _read_message(sock)) is not None: + raw, body = message + message_id = body[: 2 + body[1]] # the messageID INTEGER, echoed verbatim + op = body[2 + body[1]] + if op == _BIND_REQUEST: + replies = [_done(self._bind_code, tag=_BIND_RESPONSE)] + elif op == _SEARCH_REQUEST: + self.searches.append(raw) + replies = self._answer(raw) + else: # unbind, or anything this server does not speak + return + for reply, controls in replies: + sock.sendall(_tlv(0x30, message_id + reply + controls)) + + def close(self) -> None: + # shutdown wakes a blocked accept() on Linux, where close() alone does not. + with contextlib.suppress(OSError): + self._listener.shutdown(socket.SHUT_RDWR) + self._listener.close() + + +@contextlib.contextmanager +def _scripted_dc( + answer: Callable[[bytes], list[_Reply]], *, bind_code: int = RESULT_SUCCESS +) -> Iterator[tuple[_ScriptedDc, LdapAuthenticator]]: + """A loopback server, and an authenticator pointed at it over plain ``ldap://``: the property + under test is the decode, not the hop's TLS.""" + dc = _ScriptedDc(answer, bind_code) + try: + settings = _ad_settings( + ad_server=f"ldap://127.0.0.1:{dc.port}", ad_allow_insecure_ldap=True + ) + yield dc, LdapAuthenticator(settings, enforcing=False) + finally: + dc.close() + + +@contextlib.contextmanager +def _service_conn(authenticator: LdapAuthenticator) -> Iterator[Any]: + conn = authenticator._service_conn() + try: + yield conn + finally: + conn.unbind() + + +def _two_pages(raw: bytes) -> list[_Reply]: + if b"synthetic-cookie-1" in raw: + return [_entry(_group_dn(3)), _done(RESULT_SUCCESS, b"")] + return [ + _entry(_group_dn(1)), + _entry(_group_dn(2)), + _done(RESULT_SUCCESS, b"synthetic-cookie-1"), + ] + + +def test_real_ldap3_pages_the_group_search_and_sends_the_cookie_back() -> None: + """ldap3 encodes the request control and decodes the cookie where ``_paged_cookie`` reads it.""" + with _scripted_dc(_two_pages) as (dc, authenticator), _service_conn(authenticator) as conn: + groups = authenticator._resolve_groups(conn, _USER_DN, []) + assert groups == {ldap_module.canonical_group_dn(_group_dn(n)) for n in (1, 2, 3)} + assert len(dc.searches) == 2 + assert all(ldap_module._PAGED_RESULTS_OID.encode() in raw for raw in dc.searches) + assert all(_IN_CHAIN in raw for raw in dc.searches) + assert b"synthetic-cookie-1" not in dc.searches[0] + assert b"synthetic-cookie-1" in dc.searches[1] + + +def test_real_ldap3_refuses_a_group_search_whose_second_page_is_busy() -> None: + def answer(raw: bytes) -> list[_Reply]: + if b"synthetic-cookie-1" in raw: + return [_done(RESULT_BUSY)] + return _two_pages(raw) + + with ( + _scripted_dc(answer) as (dc, authenticator), + _service_conn(authenticator) as conn, + pytest.raises(LdapUnansweredError, match=r"result code 51 \(busy\)"), + ): + authenticator._resolve_groups(conn, _USER_DN, []) + assert len(dc.searches) == 2 + + +@pytest.mark.parametrize("code", [99, 4242], ids=["99", "4242"]) +def test_real_ldap3_raises_key_error_on_an_unknown_code_and_the_engine_maps_it(code: int) -> None: + """Measured. THE PREMISE first: a bare ldap3 search, at ldap3's default ``fast_decoder=True``, + raises ``KeyError`` from inside ``conn.search``. Then the engine's own search on the same + server refuses it as unanswered, so the probe reads as an outage.""" + assert code not in RESULT_CODES + with _scripted_dc(lambda _raw: [_done(code)]) as (_dc, authenticator): + with _service_conn(authenticator) as conn, pytest.raises(KeyError): + conn.search(search_base="OU=Users,DC=example,DC=com", search_filter="(cn=x)") + with pytest.raises(LdapUnansweredError, match="an unknown result code") as refused: + authenticator.probe_principal("alice") + assert refused.value.code == code + assert f"user search with result code {code}" in str(refused.value) + + +def test_real_ldap3_maps_a_bind_that_ends_with_an_unknown_code() -> None: + """The service-account bind every probe and sign-in opens with: refused, not a ``KeyError``.""" + with ( + _scripted_dc(lambda _raw: [_done(RESULT_SUCCESS)], bind_code=4242) as (dc, authenticator), + pytest.raises(LdapUnansweredError, match="bind with result code 4242") as refused, + ): + authenticator.probe_principal("alice") + assert refused.value.code == 4242 + assert dc.searches == [] + + # --- sign-in: a directory error, not an unknown user --------------------------------------------- From c67c178bd0f5b4fd6ab25b187b3a746dd2c21fdc Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Fri, 16 Oct 2026 17:23:40 -0500 Subject: [PATCH 09/57] A self-service session revoke that ends nothing writes one audit row (BACKLOG #3260) Limb 2 of vault BACKLOG #3260, built on the owner ruling of 2026-10-16 (fleet wiki note 20261016T211331989Z-vfzmx7). AuthService.revoke_own_session writes auth.session_revoke_missed through write_audit_soft on a miss, with no defect re-raised, so the 404 and the console's revoke_missed notice stand whatever the audit write does. The detail is the id's first 12 characters and nothing else, so an unknown id and another account's id write the same row as well as getting the same answer. The method takes a required keyword client, which both planes pass (DELETE /me/sessions/{id} and /ui/account/sessions/{id}/revoke); the auth.session_revoked row now carries it too. The seam digest records names, not parameters, so it did not move; the _ui_seam history and the console changelog say so. SECURITY.md and a changelog fragment describe the row. Tests: tests/test_self_target_refusal_audit.py (JSON plane, unknown and foreign in one run, plus the success control) and packaging/messagefoundry-webconsole/tests/test_ui_session_revoke_miss_audit.py (console plane, unknown, foreign and an over-long plain-text id). --- .../3260-session-revoke-miss.security.md | 7 ++ docs/SECURITY.md | 9 ++- messagefoundry/api/_ui_seam.py | 5 ++ messagefoundry/api/auth_routes.py | 7 +- messagefoundry/auth/service.py | 32 ++++++++- messagefoundry_webconsole/routes/account.py | 7 +- .../messagefoundry-webconsole/CHANGELOG.md | 5 ++ .../test_ui_session_revoke_miss_audit.py | 67 +++++++++++++++++++ tests/test_self_target_refusal_audit.py | 61 +++++++++++++++++ 9 files changed, 192 insertions(+), 8 deletions(-) create mode 100644 changelog.d/3260-session-revoke-miss.security.md create mode 100644 packaging/messagefoundry-webconsole/tests/test_ui_session_revoke_miss_audit.py diff --git a/changelog.d/3260-session-revoke-miss.security.md b/changelog.d/3260-session-revoke-miss.security.md new file mode 100644 index 0000000000..d5c8cc27c9 --- /dev/null +++ b/changelog.d/3260-session-revoke-miss.security.md @@ -0,0 +1,7 @@ +- **A self-service revoke of one session that ends nothing is now audited.** `DELETE + /me/sessions/{session_id}` and the console's revoke button answer as before when the id names no + session or another account's. Each such miss now writes one `auth.session_revoke_missed` row. It + names the actor, the caller's address and the first 12 characters of the id, and nothing else, so + the row does not tell an unknown id from another account's any more than the answer does. Before, + the miss left no row of its own on either plane. The `auth.session_revoked` row now carries the + caller's address too. See [SECURITY.md](../docs/SECURITY.md). (`vault BACKLOG #3260`) diff --git a/docs/SECURITY.md b/docs/SECURITY.md index 9aa5de2ebc..880d7999c9 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -752,8 +752,11 @@ AD group-map and scope-map saves and the custom-role edit and delete can still e sessions on the login window. The row names the actor, the route as `op`, the caller's stored id and the caller's address. The guard refuses when the path's spelling or the stored id matches the caller (vault BACKLOG #3259); `_refuse_if_self` in `api/auth_routes.py` says why the -stored id matters on the console. The `404` that `DELETE /me/sessions/{session_id}` answers for -another account's session still writes no row of its own. +stored id matters on the console. A self-service revoke of one session that ends nothing, the +`404` of `DELETE /me/sessions/{session_id}` and the console's "Nothing was revoked" notice, writes +one `auth.session_revoke_missed` row (vault BACKLOG #3260, limb 2). It names the actor, the +caller's address and the first 12 characters of the id, and nothing else: an unknown id and +another account's id get the same answer and the same row. **Delegated identity & admin device posture (#193 sibling; ASVS 13.2.1 / 13.3.2 / 8.4.2 — the delegation boundary).** Three controls whose enforcement is largely the deploying organization's to provide. @@ -5170,7 +5173,7 @@ Every authentication and authorization event is written to the durable `audit_lo user: `auth.login_success` / `auth.login_failed` / `auth.login_locked` / `auth.logout` / `auth.login_new_ip` / `auth.login_address_unevaluated` (the first-seen sign-in address, BACKLOG #288; on a directory sign-in the row's `mech` names the leg, `kerberos` or `oidc`) / -`auth.permission_denied` / `auth.channel_denied` / `auth.self_target_refused`, the 6.3.5 events `auth.account_locked` / +`auth.permission_denied` / `auth.channel_denied` / `auth.self_target_refused` / `auth.session_revoke_missed`, the 6.3.5 events `auth.account_locked` / `auth.login_after_failures`, the re-proof rows `auth.reauth` / `auth.password_change_failed`, plus `user.created` / `user.roles_changed` / `user.channel_scope_changed` / `user.channel_scope_change_refused` / `user.deleted`, `ad_group_map.updated` / `ad_group_scope_map.updated`, and `auth.ad_scope_resynced`. PHI access (viewing a raw message or displaying patient summaries) is recorded diff --git a/messagefoundry/api/_ui_seam.py b/messagefoundry/api/_ui_seam.py index 68233ecc21..27901b2e09 100644 --- a/messagefoundry/api/_ui_seam.py +++ b/messagefoundry/api/_ui_seam.py @@ -404,6 +404,11 @@ #: surface grew. An older console on this engine is refused at mount; a newer console on an older #: engine would call a method that does not exist. #: +#: Vault BACKLOG #3260, limb 2: ``AuthService.revoke_own_session`` takes a required keyword +#: ``client``, the address on its new ``auth.session_revoke_missed`` row and on its +#: ``auth.session_revoked`` row, and ``/ui/account/sessions/{id}/revoke`` passes it. The digest did +#: not move, for the same reason as #3259, and the same TypeError skew applies to that call. +#: #: The digest below covers the surface DISCOVERED from the console's own imports and uses, which is #: strictly larger than the five hand-maintained tuples it replaced -- those had drifted, and the #: proof is that commit 40a4d5d9 added a REQUIRED ``UploadedFileList.scope`` field the console renders diff --git a/messagefoundry/api/auth_routes.py b/messagefoundry/api/auth_routes.py index adaac6a2d9..8f2348bcc7 100644 --- a/messagefoundry/api/auth_routes.py +++ b/messagefoundry/api/auth_routes.py @@ -962,6 +962,7 @@ async def count(pin: float | None) -> int: @app.delete("/me/sessions/{session_id}", response_model=SimpleMessage) async def revoke_my_session( session_id: DigestId, + request: Request, service: AuthService = Depends(_service), # 7.5.2 (ASVS): terminating a session needs a fresh PASSWORD re-proof BOUND TO THIS ACTION # (BACKLOG #1149) — single-use, so the login-seeded window no longer satisfies it. Still the @@ -971,7 +972,11 @@ async def revoke_my_session( identity: Identity = Depends(require_reauth_only_action(STEP_UP_ACTION_SESSION_TERMINATE)), ) -> SimpleMessage: # Ownership-checked in the service: a 404 (not 403) avoids confirming another user's session id. - if not await service.revoke_own_session(identity, session_id, actor=identity.username): + # A miss writes one audit row there, the same for an unknown and a foreign id (vault + # BACKLOG #3260, limb 2), and the answer here is unchanged. + if not await service.revoke_own_session( + identity, session_id, actor=identity.username, client=client_ip(request) + ): raise HTTPException(status.HTTP_404_NOT_FOUND, "no such session") return SimpleMessage(detail="session revoked") diff --git a/messagefoundry/auth/service.py b/messagefoundry/auth/service.py index 4b6eb9b71c..4335c5e8aa 100644 --- a/messagefoundry/auth/service.py +++ b/messagefoundry/auth/service.py @@ -2027,6 +2027,11 @@ def _idp_error_fields(exc: BaseException) -> dict[str, object]: #: current credential's is the newest, so a page this size is ample. _REMINDER_MARK_PAGE: Final = 50 +#: Vault BACKLOG #3260, limb 2 (owner ruling 2026-10-16): the row a self-service revoke of one +#: session writes when it ends nothing, because the id names no session or another account's. +#: One action for both, and the same detail, so the row cannot tell the two apart either. +SESSION_REVOKE_MISSED_ACTION: Final = "auth.session_revoke_missed" + #: Vault BACKLOG #3260: the row an admin route writes when it refuses the caller's own account. #: Its detail's ``op`` names the route, one of :data:`SelfTargetOp`. SELF_TARGET_REFUSED_ACTION: Final = "auth.self_target_refused" @@ -8479,18 +8484,41 @@ async def list_sessions(self, user_id: str) -> list[SessionRecord]: await self._session_seal_failed(row.token_hash) # see identity_for_token return kept - async def revoke_own_session(self, identity: Identity, session_id: str, *, actor: str) -> bool: + async def revoke_own_session( + self, identity: Identity, session_id: str, *, actor: str, client: str | None + ) -> bool: """Revoke one of ``identity``'s **own** sessions by id (its ``token_hash``). Returns ``False`` if the session doesn't exist or isn't the caller's — so the API answers 404 without revealing - or letting a user touch another's session. Audited.""" + or letting a user touch another's session. Audited. + + A miss writes :data:`SESSION_REVOKE_MISSED_ACTION` (vault BACKLOG #3260, limb 2; owner + ruling 2026-10-16). Its detail is the id's first 12 characters, as the success row's is, + and nothing else: an unknown id and another account's id write the same row, so the trail + records no ownership a caller could not already learn. The write is soft with no defect + re-raised, for the reason ``api.auth_routes._refuse_if_self`` gives: a raise would replace + the answer the caller is owed. ``client`` is required, as on + :meth:`audit_self_target_refused`, because both planes call this from a request.""" session = await self._load_session(session_id) if session is None or session.user_id != identity.user_id: + await write_audit_soft( + lambda: self._audit( + SESSION_REVOKE_MISSED_ACTION, + actor=actor, + detail=_json({"session": session_id[:12]}), + client=client, + ), + log=_log, + message="a missed session revoke by %s stands, but its audit row failed", + args=lambda: (scrub_log_argument(actor),), + defects=(), + ) return False await self._store.revoke_session(session_id) await self._audit( "auth.session_revoked", actor=actor, detail=_json({"scope": "self", "session": session_id[:12]}), + client=client, ) return True diff --git a/messagefoundry_webconsole/routes/account.py b/messagefoundry_webconsole/routes/account.py index 72a813fb50..b332980515 100644 --- a/messagefoundry_webconsole/routes/account.py +++ b/messagefoundry_webconsole/routes/account.py @@ -626,8 +626,11 @@ async def ui_revoke_session( # time the POST lands: the target may have completed MFA or a step-up on its own device in # between. Reporting "revoked" then would tell the operator a session had ended while it was # still live under its new id. An unknown id and a foreign id - # read the same, so the notice discloses nothing the JSON twin's 404 does not. - revoked = await service.revoke_own_session(identity, session_id, actor=identity.username) + # read the same, so the notice discloses nothing the JSON twin's 404 does not. A miss writes + # the same audit row the JSON twin's does, in the service (vault BACKLOG #3260, limb 2). + revoked = await service.revoke_own_session( + identity, session_id, actor=identity.username, client=client_ip(request) + ) outcome = "revoked" if revoked else "revoke_missed" return RedirectResponse(f"/ui/account/sessions?m={outcome}", status_code=303) diff --git a/packaging/messagefoundry-webconsole/CHANGELOG.md b/packaging/messagefoundry-webconsole/CHANGELOG.md index c1a4d85a85..04cf29289c 100644 --- a/packaging/messagefoundry-webconsole/CHANGELOG.md +++ b/packaging/messagefoundry-webconsole/CHANGELOG.md @@ -92,6 +92,11 @@ this line.** ### Security +- **A session revoke that ends nothing is audited, with your address.** The revoke button on + `/ui/account/sessions` now passes the request's address to the engine, which writes one + `auth.session_revoke_missed` row when the id names no session or another account's. The notice + is unchanged. `AuthService.revoke_own_session` takes a required keyword `client`, which the seam + digest does not record, so a skew raises TypeError on that call. (vault BACKLOG #3260) - **The audit page shows where each entry came from.** `/ui/audit` has a Client column, which shows the address the engine recorded for the entry, or a dash when no client was in scope. An operator tracing a run of refused sign-ins can now see their source on the page, not only in diff --git a/packaging/messagefoundry-webconsole/tests/test_ui_session_revoke_miss_audit.py b/packaging/messagefoundry-webconsole/tests/test_ui_session_revoke_miss_audit.py new file mode 100644 index 0000000000..2bbfad27e1 --- /dev/null +++ b/packaging/messagefoundry-webconsole/tests/test_ui_session_revoke_miss_audit.py @@ -0,0 +1,67 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +# Copyright (C) 2026 MessageFoundry Foundation, LLC and contributors +"""Vault BACKLOG #3260, limb 2, on the console plane: a session revoke that ends nothing is audited. + +Owner ruling 2026-10-16: one ``auth.session_revoke_missed`` row per miss, with the id cut to its +first 12 characters, and the answer unchanged and the same for an unknown id and another account's. +The console takes the id as plain text, so the cut also bounds what a caller can put in the row. +The JSON plane's twin is ``tests/test_self_target_refusal_audit.py``. +""" + +from __future__ import annotations + +import json + +import pytest +from _ui_clients import PW, SAME_ORIGIN, cookie_login, mint_bound_proof, provision, ui_client + +from messagefoundry.auth import Role +from messagefoundry.auth.service import AuthService +from messagefoundry.auth.tokens import hash_token +from messagefoundry.config.settings import AuthSettings +from messagefoundry.pipeline import Engine + + +@pytest.mark.parametrize( + "kind", + [ + "unknown", + "foreign", + # Plain text on this plane, so the row must cut it, not copy it. + "long", + ], +) +async def test_a_missed_console_revoke_writes_one_row_and_the_same_answer( + engine: Engine, kind: str +) -> None: + """RED when: the console's revoke of a session it cannot end writes no audit row of its own, or + its answer or row tells an unknown id from another account's (vault BACKLOG #3260, limb 2).""" + service = AuthService( + engine.store, AuthSettings(require_mfa=False, admin_write_min_interval_seconds=0) + ) + await service.initialize() + await provision(service, "a", [Role.VIEWER.value]) + await provision(service, "b", [Role.VIEWER.value]) + theirs = await service.login("b", PW) + assert theirs.token is not None + target = { + "unknown": "0" * 64, + "foreign": hash_token(theirs.token), + "long": "x" * 200, + }[kind] + path = f"/ui/account/sessions/{target}/revoke" + async with ui_client(engine, service) as c: + await cookie_login(c, "a") + await mint_bound_proof(c, path) + r = await c.post(path, headers=SAME_ORIGIN) + assert r.status_code == 303, (r.status_code, r.text[:300]) + assert r.headers["location"] == "/ui/account/sessions?m=revoke_missed" + assert await service.identity_for_token(theirs.token, activity=False) is not None, ( + "a missed revoke ended another account's session" + ) + rows = await engine.store.list_audit(action="auth.session_revoke_missed", limit=10) + assert len(rows) == 1, "one miss, one row" + assert rows[0]["actor"] == "a" + assert rows[0]["client"] == "127.0.0.1", "the miss row dropped the caller's address" + # The cut id and nothing else, so the row records no ownership either. + assert json.loads(str(rows[0]["detail"])) == {"session": target[:12]} diff --git a/tests/test_self_target_refusal_audit.py b/tests/test_self_target_refusal_audit.py index 8c41395e6b..02a9ba70a9 100644 --- a/tests/test_self_target_refusal_audit.py +++ b/tests/test_self_target_refusal_audit.py @@ -137,3 +137,64 @@ async def test_another_account_writes_no_refusal_row( r = await c.request(method, f"/users/{other.id}{suffix}", json=body, headers=_auth(tok)) assert r.status_code == 200, r.text assert await engine.store.list_audit(action="auth.self_target_refused", limit=10) == [] + + +# --- vault BACKLOG #3260, limb 2: a self-service revoke that ends nothing -------------------------- + +#: A well-formed session id that names no session. +_UNKNOWN_SESSION = "0" * 64 + + +async def test_a_missed_session_revoke_writes_one_row_and_the_same_answer(engine: Engine) -> None: + """RED when: ``DELETE /me/sessions/{id}`` answers its 404 with no audit row of its own (vault + BACKLOG #3260, limb 2; owner ruling 2026-10-16), or the answer or the row tells an unknown id + from another account's. + + One row per miss, with the id cut to 12 characters, and nothing else in its detail.""" + service = await _service(engine) + await _add(service, "a", Role.VIEWER) + await _add(service, "b", Role.VIEWER) + async with _client(engine, service) as c: + ta = (await _login(c, "a")).json()["token"] + tb = (await _login(c, "b")).json()["token"] + foreign = (await c.get("/me/sessions", headers=_auth(tb))).json()["sessions"][0]["id"] + answers = [] + # The grant is single-use, so each attempt takes its own re-proof, as a caller's would. + for target in (_UNKNOWN_SESSION, foreign): + _r, ta = await _reauth(c, ta, purpose="session_terminate") + assert _r.status_code == 200 + r = await c.delete(f"/me/sessions/{target}", headers=_auth(ta)) + answers.append((r.status_code, r.json())) + still = await c.get("/auth/me", headers=_auth(tb)) + assert answers == [(404, {"detail": "no such session"})] * 2, answers + assert still.status_code == 200, "a missed revoke ended another account's session" + rows = await engine.store.list_audit(action="auth.session_revoke_missed", limit=10) + assert len(rows) == 2, "one miss, one row" + # Newest first: the foreign id's row, then the unknown id's. + assert [json.loads(str(r["detail"])) for r in rows] == [ + {"session": foreign[:12]}, + {"session": _UNKNOWN_SESSION[:12]}, + ] + for row in rows: + assert row["actor"] == "a" + assert row["client"] == _DEFAULT_PEER[0], "the miss row dropped the caller's address" + assert await engine.store.list_audit(action="auth.session_revoked", limit=10) == [] + + +async def test_a_revoke_of_your_own_session_writes_no_miss_row(engine: Engine) -> None: + """The control: a revoke that ends a session writes ``auth.session_revoked``, not a miss.""" + service = await _service(engine) + await _add(service, "u", Role.VIEWER) + async with _client(engine, service) as c: + t1 = (await _login(c, "u")).json()["token"] + t2 = (await _login(c, "u")).json()["token"] + _r, t2 = await _reauth(c, t2, purpose="session_terminate") + sessions = (await c.get("/me/sessions", headers=_auth(t2))).json()["sessions"] + other = next(s for s in sessions if not s["current"]) + r = await c.delete(f"/me/sessions/{other['id']}", headers=_auth(t2)) + gone = await c.get("/auth/me", headers=_auth(t1)) + assert r.status_code == 200, r.text + assert gone.status_code == 401 + assert await engine.store.list_audit(action="auth.session_revoke_missed", limit=10) == [] + revoked = await engine.store.list_audit(action="auth.session_revoked", limit=10) + assert len(revoked) == 1 and revoked[0]["client"] == _DEFAULT_PEER[0] From c8fae10f3e1348df5aa979b71b96923168d76967 Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Fri, 16 Oct 2026 17:54:48 -0500 Subject: [PATCH 10/57] OIDC step-up wave 1 follow-ups from engine PR 2078 (BACKLOG #3145) Each item fixed or declined in one written line: 1. Declined. A gate-refused restart paging config_changed on every attempt is #2597's crash-loop contract, pinned by test_the_alert_fires_even_when_the_start_then_fails (comment in the app.py lifespan). 2. Declined the same way: the audit-chain walk runs inside engine.start(), and moving it ahead of the notice gate reorders that start (comment above auth.initialize()). 3. Built in part. build_idp_opener_from_settings in auth/oidc_http.py assembles the arguments for both call sites, AuthService.__init__ and verify/federation.py::_idp_opener; each keeps its own error mapping. The checks.py settings-preamble refactor is declined: it would collide with vault #3063 on the same helper. 4. Built. A step-up refusal made after the flow was redeemed carries start_client and client_moved through _step_up_address_fields, the helper the success row now uses too. They come from the staged flow, never the token; extra stays first. A refusal before the state check (or with no flow) carries neither. 5. Built. SECURITY.md documents the two fields; PHI.md's audit_log section row no longer says writers "only ever" store shapes, counts and ids, and the detail row names the client-address fields. 6. (a) Declined: the client raises IdpStepUpRequired before the registered handler because that handler can only re-prove a password, which POST /me/reauth refuses for this session. (b) Built as wording: the three IdP refusals say to repeat the action in the web console, which opens /ui/reauth for it; a bare /ui/reauth still bounces an unregistered next to /ui, unchanged. (c) Built: _step_up_refusal catches a store fault reading the mechanism (driver errors, RuntimeError, OSError, aiosqlite's ValueError; AUDIT_WRITE_DEFECTS re-raised) and keeps the 403 with the password wording instead of a 500. The one extra session read on a refusal is declined: it is the read that picks the wording. (d) Built: STEP_UP_VIA_HEADER / STEP_UP_VIA_IDP live in api/auth_models.py, which the FastAPI-free apiclient already imports. --- .../3145-oidc-step-up-follow-ups.security.md | 8 ++ docs/PHI.md | 4 +- docs/SECURITY.md | 8 +- messagefoundry/api/app.py | 9 +- messagefoundry/api/auth_models.py | 7 + messagefoundry/api/auth_routes.py | 16 ++- messagefoundry/api/security.py | 38 +++++- messagefoundry/apiclient/client.py | 17 ++- messagefoundry/auth/oidc_http.py | 28 +++- messagefoundry/auth/service.py | 128 ++++++++++++------ messagefoundry/verify/federation.py | 10 +- tests/test_auth_oidc_service.py | 3 +- tests/test_oidc_json_step_up_hint.py | 35 +++++ tests/test_oidc_step_up.py | 48 +++++++ 14 files changed, 291 insertions(+), 68 deletions(-) create mode 100644 changelog.d/3145-oidc-step-up-follow-ups.security.md diff --git a/changelog.d/3145-oidc-step-up-follow-ups.security.md b/changelog.d/3145-oidc-step-up-follow-ups.security.md new file mode 100644 index 0000000000..c40a970b03 --- /dev/null +++ b/changelog.d/3145-oidc-step-up-follow-ups.security.md @@ -0,0 +1,8 @@ +- **A refused federated step-up records where it started.** An `auth.reauth` row for a step-up + refused after its flow was redeemed now carries `start_client` and `client_moved`, as the row of + a completed step-up does. Both come from the flow the start leg staged, never from the token. A + refusal before the `state` check carries neither. The refusal a step-up gate sends an `oidc` + session now says to repeat the action in the web console, because a bare `/ui/reauth` names no + action and returns to `/ui`. A store fault while that refusal reads the session's sign-in + mechanism now keeps the `403`, with the password wording, rather than answering `500`. See + [SECURITY.md](../docs/SECURITY.md). (`vault BACKLOG #3145`) diff --git a/docs/PHI.md b/docs/PHI.md index 600fc3579f..d3db7efce9 100644 --- a/docs/PHI.md +++ b/docs/PHI.md @@ -194,7 +194,7 @@ destruction) are documented in [§3](#3-encryption-at-rest) under the matching h | `queue.handler_name` / `destination_name` / `channel_id` | all three | No — names, not bodies | No (metadata, deliberately not ciphered) | **PL-4** | The handler the transform worker runs; the destination the delivery worker drains | `n/a — not PHI` | | `messages.control_id`, `messages.message_type` | all three | Low (MSH-10/MSH-9) | **No** — plaintext by design | **PL-4** | Needed plaintext for dedup/routing/indexes (`ix_messages_control`). Covered only by the whole-DB / volume layer | `keep-forever by design` — dedup/routing keys that live and die with the message row | | `messages.origin`, `messages.origin_actor` (vault BACKLOG #2615) | all three | **No** — a fixed label and a username, never message content | **No** — plaintext by design | **PL-4** | How the row came to exist: `partner`, `operator_edit`, `operator_upload` or `reingress`, and for the two operator origins the acting user. Written at insert and never updated. Plain on purpose, because it must outlive the body and the ciphered `metadata`, which retention blanks. `NULL` on a row written before the column existed means not recorded. See [§6](#6-audit--accountability) | `keep-forever by design` — provenance that lives and dies with the message row | -| `audit_log.detail` | all three | Low — exposed IDs/counts, not bodies | **No** — plaintext by design | **PL-4** | JSON metadata about PHI *access* and other actions, not the PHI itself. Its writers store at least filter shapes, counts and ids. Some also store more: an edit-resend row holds **keyed** HMAC-SHA256 digests of the original and the edited body, and an upload-inject row one of the injected body. Without the store key a digest cannot be recovered or tested against a guess ([§6](#6-audit--accountability), vault BACKLOG #2615). And some store bounded engine-authored text: at least the `connection_test` and `connection_credential_test` rows carry the probe outcome, a fixed line or `safe_text()`/`safe_exc()`-scrubbed connector error text (BACKLOG #2372). None stores a message body. *Corrected 2026-10-07:* this said the writers "only ever store filter shapes, counts and ids" | `keep-forever by design` — 45 CFR 164.316(b)(2)(i) six-year documentation retention. **Not** chain-breakage: which rows a delete removes decides that, and the reasoning is stated once, in the `audit_days` row of [CONFIGURATION.md](CONFIGURATION.md#retention) | +| `audit_log.detail` | all three | Low — exposed IDs/counts, not bodies | **No** — plaintext by design | **PL-4** | JSON metadata about PHI *access* and other actions, not the PHI itself. Its writers store at least filter shapes, counts and ids. Some also store more: an edit-resend row holds **keyed** HMAC-SHA256 digests of the original and the edited body, and an upload-inject row one of the injected body. Without the store key a digest cannot be recovered or tested against a guess ([§6](#6-audit--accountability), vault BACKLOG #2615). And some store bounded engine-authored text: at least the `connection_test` and `connection_credential_test` rows carry the probe outcome, a fixed line or `safe_text()`/`safe_exc()`-scrubbed connector error text (BACKLOG #2372). Some store a client address beside the row's own `client`: at least the federated step-up's `auth.reauth` row, whose `start_client` is the address that started the step-up and whose `client_moved` says whether the callback came from another host (BACKLOG #2160, vault BACKLOG #3145). None stores a message body. *Corrected 2026-10-07:* this said the writers "only ever store filter shapes, counts and ids" | `keep-forever by design` — 45 CFR 164.316(b)(2)(i) six-year documentation retention. **Not** chain-breakage: which rows a delete removes decides that, and the reasoning is stated once, in the `audit_days` row of [CONFIGURATION.md](CONFIGURATION.md#retention) | | `audit_log.client` (ADR 0150) | all three | **No** — a network address; identifies a *host*, not a patient | No (metadata, deliberately not ciphered) | **PL-4** | The caller's client address — the "from where" of an audited action; `NULL` for engine-internal/`system` writes. **Personal data, but not PHI**, and exactly what HIPAA §164.312(b) audit controls exist to capture. Plaintext by decision: it must stay greppable/indexable for incident response, it already appears in the clear in `sessions.client`, and it is folded **inside** the tamper-evident hash chain — so it carries **integrity** protection even without confidentiality. Widens a store-file compromise from *who did what* to *who did what from where*; volume encryption + owner-only ACLs on whichever host owns the files — the engine's own `_secure_file` covers the **SQLite** store only ([§10](#10-secure-deployment--operations-checklist)) — are the control | `keep-forever by design` — same `audit_log` row lifetime as `detail`; the value is folded **inside** the hash chain | | `delivered_keys` (H2 idempotency ledger) | all three | **No** — hashes + ids only | No (deliberately not ciphered — nothing to protect) | **PL-4** | One row per completed outbound delivery: a SHA-256 `delivery_key` over non-PHI ids + a replay-stable seq, plus `outbox_id`/`message_id`/`destination_name`/`delivery_seq`. **Never a body or any PHI** — `control_id` is only *folded into the hash input*, never stored in the clear here. Lets the FIFO claim skip-and-complete a re-claimed already-delivered head without re-sending | `keep-forever by design` — the idempotency ledger a re-claimed already-delivered row checks instead of re-sending | | `state.namespace` / `state.key` | all three | **Possibly** — a Handler that keys correlation state on a raw MRN stores that identifier here in the clear | **No** — plaintext by construction: the pair is the composite primary key **and** the AAD input for `state.value`, so it cannot be ciphered without losing the lookup | **PL-4** | Authors must key state on a **surrogate, never a raw identifier**. Covered only by the whole-DB / volume layer. Rides `[retention].state_max_age_days` with its value | ``rides `[retention].state_max_age_days` `` | @@ -1805,7 +1805,7 @@ with materially different PHI profiles, so they get their own rows; stream 4 is | **2. `uvicorn` request/access log** (sub-stream of 1) | one line per HTTP request — method, **full request line including the query string**, status, timing | inherits stream 1's format | inherits stream 1's sink | request tracing, latency and error triage | inherits stream 1's | inherits stream 1's | **Can contain PHI.** `configure_logging` clears uvicorn's own handlers and propagates to the root, so the four filters apply; `serve` passes `log_config=None` and never disables `access_log`, so at the default `INFO` level every request is logged. OIDC `code`/`state` **are** scrubbed. **Not** scrubbed: a `?content=…` / `?field_value=…` that arrives anyway — no route has declared either since BACKLOG #1184, but the access line is built from the raw `query_string` rather than from what a route binds (the single-token residual above) | | **3. `messagefoundry.audit` off-box tee** (sub-stream of 1) | one JSON object per **committed** `audit_log` row: `event`/`ts`/`action`/`actor`/`channel_id`/`client`/`detail`, plus the `row_id`/`row_hash` anchor pair (BACKLOG #1198) that lets a collector tie the copy back to the row it came from and detect a gap in the chain | JSON | emitted after the row is durably committed and **outside** the store write lock; rides stream 1's handlers | shipping audit evidence to a SIEM so it survives a host compromise | inherits stream 1's | inherits stream 1's | `detail` is passed through the `safe_text` PHI chokepoint **before** it leaves the process; `client` is forwarded verbatim as a discrete field so a SIEM can index it. The `row_hash` anchor carries no PHI directly, but on a **keyless** store it is a plain SHA-256 whose other preimage members travel in the same record or the one before it, so a reader of the forwarded stream could test offline guesses at a span `safe_text` cut; it is a keyed MAC once the store cipher is active. Best-effort: a logging failure is caught, never raised into the audit write. **Pinned to `INFO`** — it is emitted even at `[logging].level = WARNING` | | **4. Off-box syslog/SIEM forwarder** — the shared **transport** for 1–3 | a copy of every record from 1–3 | `forward_format`, default **JSON** (independent of the stdout format) | the operator's collector (`forward_host`/`_port`) | off-box evidence retention / SIEM correlation | **default-on when a collector is named.** Transport: `udp` (default) / `tcp` / **`tls`** (RFC 5425, CA-anchored, verified by default). `serve` gates the hop on the shared posture gradient before the handler is installed: verified TLS ungated; otherwise loopback / attested ALLOW, non-enforcing WARN, **enforcing REFUSE (exit 2)** — there is no synthetic arm, and [ADR 0186](adr/0186-retire-the-synthetic-data-declaration-every-instance-carries-patient-data.md) left this cell with no per-hop way to accept the risk at all | the collector's, not the engine's | the identical four filters are installed on this handler. So the forwarded copy gets the same best-effort PHI redaction and credential scrub as stdout. A single-token identifier can survive it, as it can there. The copy also carries usernames, connection names, message ids, client addresses and the audit chain. That is the engine's own stated reason for gating the hop | -| **5. `audit_log` table** (SQLite, Postgres, SQL Server) | who / what / **where-from** / when of auth + PHI *access* and admin actions — plus, while `[security].audit_all_authorization_decisions` is on (**default `true`** since BACKLOG #1277, 2026-09-02, which reversed the ADR 0118 §5 `false`; the internal field it desugars to is `audit_all_authz`, whose old `[diagnostics]` TOML spelling is **refused at load** — ADR 0118), an `authz` row for **every** authorization decision including successes. That is the shipped volume of this stream, not an opt-in addition to it: one row per authenticated request on each `require()`-gated route, and this row's retention cell records that **nothing prunes the table** — `actor`, `action`, `channel_id`, `client`, `detail`, `row_hash` | JSON `detail`; **tamper-evident hash chain** over `prev_hash` + the row (the `client` address is **inside** the chained payload — ADR 0150) | the store database | HIPAA §164.312(b) audit controls; incident response; `verify_audit_chain` integrity checks | `GET /audit` requires **`audit:read`**; `GET /audit/export` requires the separate **`audit:export`** and streams CSV with formula-injection neutralisation, recording its own `audit.export` row *before* streaming; `GET /me/security-events` is a per-user view of the same table | **`[retention].audit_days` is reserved and NOT enforced — keep-forever by design** (the audit-retention requirement, ~6 years — **not** chain-breakage; [§8](#8-retention--purge) states the position and cites its source of record) | `detail` is stored **in the clear** (it is not a cipher-covered column): its protection is that writers only ever store filter shapes, counts and ids — never bodies or credentials — plus the store ACL and the volume layer | +| **5. `audit_log` table** (SQLite, Postgres, SQL Server) | who / what / **where-from** / when of auth + PHI *access* and admin actions — plus, while `[security].audit_all_authorization_decisions` is on (**default `true`** since BACKLOG #1277, 2026-09-02, which reversed the ADR 0118 §5 `false`; the internal field it desugars to is `audit_all_authz`, whose old `[diagnostics]` TOML spelling is **refused at load** — ADR 0118), an `authz` row for **every** authorization decision including successes. That is the shipped volume of this stream, not an opt-in addition to it: one row per authenticated request on each `require()`-gated route, and this row's retention cell records that **nothing prunes the table** — `actor`, `action`, `channel_id`, `client`, `detail`, `row_hash` | JSON `detail`; **tamper-evident hash chain** over `prev_hash` + the row (the `client` address is **inside** the chained payload — ADR 0150) | the store database | HIPAA §164.312(b) audit controls; incident response; `verify_audit_chain` integrity checks | `GET /audit` requires **`audit:read`**; `GET /audit/export` requires the separate **`audit:export`** and streams CSV with formula-injection neutralisation, recording its own `audit.export` row *before* streaming; `GET /me/security-events` is a per-user view of the same table | **`[retention].audit_days` is reserved and NOT enforced — keep-forever by design** (the audit-retention requirement, ~6 years — **not** chain-breakage; [§8](#8-retention--purge) states the position and cites its source of record) | `detail` is stored **in the clear** (it is not a cipher-covered column): its protection is that writers store at least filter shapes, counts, ids, keyed digests, bounded engine-authored text and client addresses, the `audit_log.detail` row above says which — never bodies or credentials — plus the store ACL and the volume layer. *Corrected 2026-10-16 (vault BACKLOG #3145):* this said writers "only ever store filter shapes, counts and ids" | | **6. `message_events` table** | the per-message disposition timeline — the **complete** vocabulary is `received`, `routed`, `unrouted`, `filtered`, `transformed`, `delivered`, `failed`, `dead`, `error`, `replayed`, `resent`, `reingressed`, `passthrough`, `passthrough_dropped`, `cancelled`, `edit_resend`, `edit_resubmit`, `viewed`, `not_deployed`, the ADR 0154 synchronous-reply pair `reply_returned` / `reply_timeout` (names, counts and `waited_ms` only — **never** a fragment of the partner's reply body), and `reply_dropped`, a delivery whose 2xx reply body was over the byte bound and dropped (the status and the bound only, vault BACKLOG #3040) (CI asserts this list against the engine's own `MESSAGE_EVENT_KINDS`). `[diagnostics].message_events` can thin the set, but never below the compliance floor `viewed` / `dead` / `error` / `failed` / `not_deployed` / `reply_timeout` / `reply_dropped` | rows: `message_id`, `ts`, `event`, `destination`, `detail` | the store database | operator timeline on the message-detail view; the `viewed` row is the HIPAA PHI-access record | `GET /messages/{id}` under **`messages:view_raw`** + `require_phi_read`; the read itself writes a `viewed` event **and** a `message_view` audit row | no dedicated window — `purge_message_bodies` sets `message_events.detail` to `NULL` in the same transaction that blanks the body, so it inherits `[retention].messages_days` | `detail` is `safe_text()`-scrubbed **then** cipher-encrypted (AAD `("message_events","detail",message_id,ts,event)`). Verbosity gate `[diagnostics].message_events` = `all` (default) / `errors` / `off`, with a **compliance floor that can never be thinned**: `viewed`, `dead`, `error`, `failed`, `not_deployed`, `reply_timeout`, `reply_dropped` are retained at every level (`reply_timeout` is the one row that explains a "we called you and got a 504" complaint, so an instance that thinned its logs would lose exactly the record it is later asked for; `reply_dropped` is the one row that answers "you ignored our reply", for the same reason) | | **7. `connection_event` table — DEFAULT ON** (`[diagnostics].connection_events = true`) | transport/lifecycle events per connection: `established`, `closed` (reason `eof`, `idle_timeout` or `frame_deadline` — a fixed vocabulary chosen by the listener, never free text), `idle_timeout`, `at_capacity`, `peer_not_allowlisted`, `frame_oversize`, `framing_error`, `peer_reset`, the MLLP listener's `handler_error` (BACKLOG #1619 -- the inbound handler faulted on a frame the listener read cleanly, such as a store outage at the ingress commit; the reason is scrubbed at the source like every other), the MLLP listener's `tls_handshake_failed` (vault BACKLOG #2613 -- a TLS handshake on a listener that runs its own handshakes (the standard asyncio loop, not uvloop) failed; the first at once, then one per listener per minute carrying the count since the last, and the reason is a count and an exception class name, never the peer's bytes), the DATABASE poll source's `row_undecodable` (BACKLOG #1662 — a polled row the source cannot turn into a body; the reason is scrubbed at the source and names the exception type or the operator's own configured column, never a row value), the FILE poll source's quarantine kinds `file_oversize` / `file_decompress_failed` / `file_content_mismatch` / `file_scan_rejected` (BACKLOG #1621 — a drop moved to `.error`; the reason is a size, a content type or a scrubbed codec or scanner message, never the file name), the inbound-HTTP intake-auth refusals `intake_auth_failed` / `auth_subject_denied` / `auth_rate_limited` (ADR 0154 D6 — peer address and mode only; **never** the credential, a prefix of it, or its length. Each of these also writes a tamper-evident audit-log row — the copy that survives an operator turning this diagnostics stream off), plus the runner's `connection_lost` / `connection_restored`. That is the whole vocabulary, asserted in CI against the literal emit call sites in `transports/` and the pipeline runner **and** cross-checked against the console's own filter tuple. The MLLP, raw-TCP, X12 and HTTP listeners emit these; the **DATABASE poll source** emits `row_undecodable` and nothing else (it dials out, so there is no lifecycle to report and its rows carry a **NULL peer column**); the **FILE poll source** emits only its four quarantine kinds, also with a NULL peer column; the **DICOM inbound C-STORE SCP** emits none — the runner injects the sink onto **every** source (`wiring_runner.py`, over the base-class `on_connection_event` field), so that connector *has* the wiring and simply never calls it — so this stream covers those four listeners plus those poll-source events and the runner's outbound-lane transitions — not literally every connection. A DICOM association's connects and refusals are therefore **absent** from this stream. The **`ISA`/`IEA`-framed X12 inbound** was equally absent until BACKLOG #1665 and is not any more: it emits the same seven kinds as the raw-TCP listener | rows: `ts`, `connection`, `transport`, `direction`, `kind`, `peer_host`, `message_id` (correlation hint), `reason` | the store database, **all three backends** | Corepoint-style transport diagnostics — "did the sender connect, and why did it drop" | `GET /events` and `GET /connections/{name}/events` under **`monitoring:read`** (**not** a PHI permission) with per-channel RBAC — an out-of-scope `connection=` is 403'd *and* audited — server-clamped to ≤1000 rows. `reason` alone is gated on `messages:view_summary` and masked as `****` until the audited per-event `reveal=` act (BACKLOG #2443; the `connection_event.reason` row of §3's PL-2 block says how) | `[retention].connection_event_retention_hours` (its own **hours** window); 0 inherits `[retention].messages_days`; both 0 = keep forever. Plain age `DELETE` (metadata-only) | **`reason` is free text that can carry sensitive fragments.** Defended twice — `safe_exc()` at the source, `safe_text(reason, limit=200)` at the store (the bound §3's PL-2 block states) — then cipher-encrypted (AAD `("connection_event","reason",connection,ts,kind)`). Every other column is config metadata; the table is documented **metadata-only** — never a frame, body or HL7 field value. Writes are a pure side observer: a bounded in-memory queue drained by a background task outside any handoff transaction, so a flood can never block a listener or pin a message disposition | | **8. `alert_instance` table — default on wherever an `[alerts]` notifier exists** | resolvable operator alerts: `connection_stopped`, `queue_buildup`, `lane_stuck`, `message_stall`, `saturation`, `connection_error`, `storage_threshold`, `cert_expiry`, `secret_rotation`, `initial_credential_expiring` (an UNCLAIMED admin-issued temporary password nearing the instant the login gate stops accepting it -- ASVS 6.4.5, BACKLOG #1141; keyed on `user:`, and the payload is the ISO deadline plus whole hours remaining, repeated in the reason column, never the password), `integrity_drift`, `update_available`, `backup_failed`, `store_privilege_warning` (the store privilege preflight found the store principal over-granted or could not read it -- BACKLOG #305; the payload is the finding, a count and the preflight's summary line, which names the login, the database and role names, never a password or message content), `leadership_acquired`, `dr_activated`, `log_write_failed` (an application-log sink was rolled after a write failure, or is UNWRITABLE and this process's connections were stopped -- BACKLOG #122, ADR 0162; the payload is the sink LABEL, the stage, a `safe_exc` reason and a count of connections stopped, never the record whose write failed), `log_forward_failed` (the off-box log forwarder is absent, losing records or not sending -- BACKLOG #2612; keyed on `forwarder:`, with `@node:` or `@shard:` appended when several engine processes share the store, and the payload is that key, fixed reason words and a count, never a record, the collector's address or an error text; an unpinned cluster node's `node_id` holds this engine's own host name and process id, so the key carries them), `gcm_invocations` (the per-key AES-GCM invocation bound crossing its 2^31 soft warn — ASVS 11.3.4; its payload carries a one-way `key_id` fingerprint plus counters, never key bytes), `approval_stale_requester` (a dual-control release refused because the requester no longer holds the authority the operation needs -- ASVS 8.3.2, BACKLOG #289; keyed on the approval id, and the payload is that id, the operation key and a closed-set reason slug, never the requester's name or the captured params), `approval_too_early` (a dual-control release refused because it arrived before `[approvals].min_dwell_seconds` -- ASVS 2.4.2, BACKLOG #287; keyed on `approval:`, and the payload is that key, the operation key and a fixed reason string, never the approver's or requester's name or the captured params), `audit_write_failed` (the approval gate could not write one of its audit rows -- vault BACKLOG #2255; keyed on `approval:`, or on `config_reload:inline` when an ungated config reload could not write its `config_reload_attempted` row -- vault BACKLOG #2254; the payload is that key and the lost row's audit action name, never the row's detail, a username or the captured params; the ERROR log line beside it holds the detail on some paths and only the exception on others), `ad_reconcile_aborted` (a directory reconciler pass left signed-in accounts unrevoked and needs an operator: the mass-revoke breaker tripped and nothing was revoked, or one or more probes were answered with an LDAP referral and those accounts were left unjudged while the rest of the pass was applied, BACKLOG #2538; the payload is a fixed source label, `directory-reconciler` for the breaker and `directory-reconciler-referral` for a referral so each is its own instance, the closed-set reason slug `mass_revoke_breaker` or `directory_referral`, the probed count and the latched operator explanation, never a username), `ad_reconcile_held` (the reconciler is holding accounts whose `userAccountControl` it cannot read, ADR 0195; the payload is the same fixed source label, a closed-set reason slug, the count of undetermined accounts and the latched operator explanation, never a username), `ad_session_revoked` (the reconciler revoked a directory principal's sessions; keyed on that operator account's username, with a `directory_absent`, `directory_disabled`, `directory_undetermined`, `roles_changed` or `scope_changed` reason), `approval_approver_provenance` (a dual-control release, or the resolution of an interrupted one, went ahead with an approver or resolver account created, re-passworded or TOTP-enrolled after the request -- BACKLOG #315, #2125; keyed on `approval:`, or `approval::resolve` for a resolution, and the payload is the operation key and the closed-set change slugs, never a username or the captured params), `administrator_granted` (the built-in Administrator role was granted by account creation, a role change, a directory group newly mapped to it -- BACKLOG #315 -- or a directory sign-in whose role sync newly gave the account the role -- vault BACKLOG #2610; keyed on `user:` or `ad-group:`, with the grant route and the granting administrator's username, or `` for a directory sign-in), `no_enabled_administrator` (a directory sign-in's role sync or the session reconciler took Administrator from its last enabled holder -- vault BACKLOG #3229; keyed on `user:`, and the payload is `via`, `ad-sync` or `ad-reconcile`, and a fixed reason sentence, never message content), `intake_paused` (the engine paused intake because the staged backlog went over `[inbound].max_staged_depth` or the SQLite volume fell below `[retention].min_free_disk_mb` -- BACKLOG #290; keyed on `intake:staged_depth` or `intake:disk_floor`, with `@node:` or `@shard:` appended when several engine processes share the store (BACKLOG #2272), and the payload is the reason, a count or a MiB figure, its limit and the store backend's name, never message content), `config_changed` (a start loaded a config whose fingerprint differs from the store's baseline -- vault BACKLOG #2597; keyed on `config:`, and the payload is the two digests, the node and engine shard labels, and the action, actor username and time of the row the older digest came from, never the config directory, a git commit or message content), `crl_reload_refused` (the CRL reload refused a replaced CRL file, so running TLS contexts keep the copy they hold -- vault BACKLOG #2904; keyed on `crl_reload:`, with `@node:` or `@shard:` appended when several engine processes share the store, and the payload is that key, the file's path, a short class token for the remedy, a count of contexts, the engine's own fixed remedy sentence and, when a start would refuse the file too, a `start_refuses` flag, never the refusal's reason, which quotes CRL issuer names and dates; that reason is in the application log, and in the `detail` of the CRL expiry alert once the held copy nears its `nextUpdate`), `remote_debug_refused` (the engine's remote-debugging guard refused scripts that another process injected into the engine process -- vault BACKLOG #2742; keyed on `remote_debug:refused`, with `@node:` or `@shard:` appended when several engine processes share the store, and the payload is that key, the count of scripts refused since the alert was last raised, the count since the process started and a fixed sentence built from the two, never a script's file name, which the injecting process chooses; that name is in the application log, in the guard's WARNING line), and the six security signals of the audit-stream rule layer -- vault BACKLOG #2613: `signin_failure_burst` (keyed on `signin:`, never the typed username), `access_denied_burst`, `body_view_burst` and `bulk_export` (each keyed on `account:`), `log_level_debug` (keyed on `logging:debug`) and `posture_loosened` (keyed on `posture:start`); the payload is a count and a fixed sentence holding a level name, switch names or a username, never a message body, a message id or an audit row's detail. The reachable **inverse** signals — `connection_restored`, `leadership_lost`, `dr_released`, `store_privilege_clean`, `intake_resumed`, `crl_reload_applied` (it resolves `crl_reload_refused` for the same file on the same engine process, once the reload's record no longer holds the refusal, and when the reload stops -- vault BACKLOG #2904), `ad_reconcile_breaker_cleared`, `ad_reconcile_hold_released` (the last two resolve `ad_reconcile_aborted`, under either source, and `ad_reconcile_held`; when they do, and where they never do, is stated once in [CONFIGURATION.md](CONFIGURATION.md#when-the-reconcilers-two-alerts-resolve-themselves) -- BACKLOG #2136) — are never rows here: `_record_state` routes an inverse through `_AUTO_RESOLVE` to `resolve_alert_instances_for`, never to `upsert_alert_instance`. (One more mapped key, `connection_started`, is emitted by no code path today.) | rows: `event_type`, `connection`, `severity`, `status`, `first_seen`, `last_seen`, `count`, `reason`, `acked_by`, `acked_at`, `resolved_at`, `suspended_until`, `escalation_tier` | the store database, **all three backends** | the operator alert list — acknowledge / resolve / suspend. Durable state is recorded **before** any suppression or throttle return, so a muted alert still leaves a record | `GET /alerts/active` under **`monitoring:diagnose`** (**not** a PHI permission) with the same per-channel scope, and `reason` alone gated on `messages:view_summary` and masked as `****` until the audited per-alert `reveal=` act (BACKLOG #2443; the `alert_instance.reason` row of §3's PL-2 block says how); ack/resolve/suspend/**resume** are POSTs on the same tier, and the separate read-only `GET /alerts/rules` view sits on its own gate | shares the connection-event window; **only RESOLVED instances are DELETEd**, by `resolved_at` — an open or acknowledged condition is never aged out from under an operator | **`reason` is free text** taken from the event's `detail`/`reason`: `safe_text(reason, limit=200)` (the bound §3's PL-2 block states) then cipher-encrypted (AAD `("alert_instance","reason",event_type,connection)` — the de-dup grain, so one AAD covers both the INSERT and the re-fire UPDATE). | diff --git a/docs/SECURITY.md b/docs/SECURITY.md index 880d7999c9..3d4c48e79c 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -3855,7 +3855,13 @@ token claim**. There is no new `auth_provider` value: a federated login resolves `/ui/login?e=rate_limited` and a WARNING log), a return the `Sec-Fetch-Mode` check refuses (audited `auth.login_failed`), and any refusal at the start leg `POST /ui/reauth/oidc`. A refusal those two methods decide changes nothing and re-renders the step-up page with a plain message, except one that finds the session gone, which sends the operator to sign in. A refusal after the `state` - check is filed under the staged session's account. A cancel at the IdP returns to the same page. + check is filed under the staged session's account. That row, and the row of a completed step-up, + also carry `start_client`, the address the start leg staged on the flow, and `client_moved`, + whether the callback came from another host by the new-address comparison, or null when either + address is unknown (BACKLOG #2160, vault BACKLOG #3145). Neither is ever read from the token. A + refusal before the `state` check names neither. A move is recorded, never refused. Behind a proxy + the engine does not trust, both legs carry the proxy's address, so `client_moved` reads false + there. A cancel at the IdP returns to the same page. A callback without the flow's own `state` consumes nothing, and the flow waits for the real return. The sign-in callback follows the same rule, an `error=` or malformed one included (BACKLOG #3198). A password is diff --git a/messagefoundry/api/app.py b/messagefoundry/api/app.py index 6370ef36ed..d406f33509 100644 --- a/messagefoundry/api/app.py +++ b/messagefoundry/api/app.py @@ -10276,7 +10276,9 @@ async def _audit_convergence_reload(outcome: ReloadOutcome) -> None: # This process cannot vouch for a config it never checked: its flag rows say so. engine.config_baseline_unchecked = start_outcome in _UNCHECKED_OUTCOMES # Raised here, before engine.start(), so new bytes that make the start itself fail - # still page on every attempt of a crash loop. Never raises. + # still page on every attempt of a crash loop. Never raises. Vault BACKLOG #3145 + # item 1 (a gate-refused restart pages on every attempt) is declined: that is this + # contract, pinned by test_the_alert_fires_even_when_the_start_then_fails (#2597). if start_config is not None and start_config.changed: _raise_config_changed(notifier or LoggingAlertSink(), engine, start_config) except BaseException: @@ -10367,7 +10369,10 @@ async def _audit_convergence_reload(outcome: ReloadOutcome) -> None: # any connection starts rather than after. Both touch only the store: initialize() # seeds the built-in roles and the gate reads the users. A refusal here also comes # before the start's opt-in audit-chain walk, so that walk cannot alert on such a - # start. Everything else that uses the service stays below the start. + # start. Vault BACKLOG #3145 item 2 is declined: the walk runs inside engine.start(), + # and moving it ahead of this gate reorders the start whose failure paging + # test_the_alert_fires_even_when_the_start_then_fails pins (#2597). Everything else + # that uses the service stays below the start. await auth.initialize() app.state.auth = auth await _assert_security_notice_is_deliverable( diff --git a/messagefoundry/api/auth_models.py b/messagefoundry/api/auth_models.py index 8dce0ff434..04e6b8f41d 100644 --- a/messagefoundry/api/auth_models.py +++ b/messagefoundry/api/auth_models.py @@ -34,6 +34,13 @@ # A session token is 43 URL-safe characters (auth/tokens.py); the cap only refuses absurd input. _TOKEN_MAX = 256 +#: The response header an ``oidc`` session's step-up refusal carries, set to +#: :data:`STEP_UP_VIA_IDP`, so a client can branch on a header rather than parse the detail +#: (BACKLOG #2158). Here, with the auth wire models, so the engine's raise sites and the +#: FastAPI-free ``apiclient`` share one spelling (vault BACKLOG #3145 item 6). +STEP_UP_VIA_HEADER = "X-Step-Up-Via" +STEP_UP_VIA_IDP = "idp" + class CredentialReply(BaseModel): """A response model whose body carries a live credential: a session token, a staged TOTP seed, diff --git a/messagefoundry/api/auth_routes.py b/messagefoundry/api/auth_routes.py index 8f2348bcc7..73a94cb303 100644 --- a/messagefoundry/api/auth_routes.py +++ b/messagefoundry/api/auth_routes.py @@ -29,6 +29,8 @@ from messagefoundry.api._ui_seam import AdminHandlers from messagefoundry.api.auth_models import ( _GROUP_MAX, + STEP_UP_VIA_HEADER, + STEP_UP_VIA_IDP, AdGroupMap, AdGroupMapEntry, AdGroupScopeEntry, @@ -749,13 +751,14 @@ async def reauth( # the message names the leg that works. The header is the one the step-up gates # send such a session (BACKLOG #2158), so a client branches on it here too rather # than parsing the detail. No X-Step-Up-Required: this is the step-up itself. - # Typed out because api/security.py keeps its constant private; a test pins the - # name and the value to what the gates send. + # The constants the gates send too (vault BACKLOG #3145); a test pins the two + # to each other. raise HTTPException( status.HTTP_403_FORBIDDEN, "this session was signed in through the identity provider; re-authenticate" - " there through the web console at /ui/reauth, not with a password", - headers={"X-Step-Up-Via": "idp"}, + " there, not with a password: repeat the action in the web console, which" + " opens /ui/reauth for it", + headers={STEP_UP_VIA_HEADER: STEP_UP_VIA_IDP}, ) if elevation.directory_unconfirmed: # BACKLOG #2027: the directory could not judge the password, so "failed" would @@ -802,8 +805,9 @@ async def mfa_verify( raise HTTPException( status.HTTP_403_FORBIDDEN, "this session was signed in through the identity provider and owes no code;" - " step up there through the web console at /ui/reauth", - headers={"X-Step-Up-Via": "idp"}, + " step up there by repeating the action in the web console, which opens" + " /ui/reauth for it", + headers={STEP_UP_VIA_HEADER: STEP_UP_VIA_IDP}, ) if elevation.directory_unconfirmed: # BACKLOG #2023: the code was never checked, and the token still authenticates, so a diff --git a/messagefoundry/api/security.py b/messagefoundry/api/security.py index 29e325f32d..5315f5a4b1 100644 --- a/messagefoundry/api/security.py +++ b/messagefoundry/api/security.py @@ -18,6 +18,7 @@ import time from collections.abc import Awaitable, Callable, Coroutine, Mapping from dataclasses import dataclass +from functools import cache from typing import Any, get_args from fastapi import ( @@ -34,11 +35,12 @@ from pydantic import BaseModel from starlette.requests import HTTPConnection -from messagefoundry.api.auth_models import CredentialReply +from messagefoundry.api.auth_models import STEP_UP_VIA_HEADER, STEP_UP_VIA_IDP, CredentialReply from messagefoundry.api.tls_client_cert import ( MF_CLIENT_PEERCERT_STATE_KEY, peercert_from_ssl_object, ) +from messagefoundry.audit_write import AUDIT_WRITE_DEFECTS from messagefoundry.auth import AuthProvider, Identity, Permission, Role from messagefoundry.auth.notifications import deadline_utc as deadline_utc # re-export from messagefoundry.auth.service import AuthService, LoginOutcome @@ -52,6 +54,7 @@ from messagefoundry.pipeline.alerts import AlertSink, LoggingAlertSink from messagefoundry.pipeline.cert_expiry import peer_cert_expiry from messagefoundry.redaction import safe_exc +from messagefoundry.store.base import store_driver_errors from messagefoundry.store.store import UserRecord log = logging.getLogger(__name__) @@ -1123,11 +1126,22 @@ async def dependency(request: Request) -> Identity: #: session (BACKLOG #2142, ADR 0142 Amendment F). _IDP_STEP_UP_DETAIL = ( "step-up re-verification required; this session steps up at the identity provider, in a" - " browser, through the web console at /ui/reauth, not with a password" + " browser, not with a password: repeat the action in the web console, which opens /ui/reauth" + " for it" ) #: Set to ``idp`` on an ``oidc`` session's step-up refusal, so a client can branch on a header #: rather than parse the detail. Absent on every other refusal. -_STEP_UP_VIA_HEADER = "X-Step-Up-Via" +_STEP_UP_VIA_HEADER = STEP_UP_VIA_HEADER + + +@cache +def _session_read_faults() -> tuple[type[Exception], ...]: + """What a refused session read raises: the drivers' errors, plus ``RuntimeError`` and + ``OSError`` as :func:`~messagefoundry.store.base.store_driver_errors` asks, and aiosqlite's + ``ValueError`` for a closed connection. A caller re-raises + :data:`~messagefoundry.audit_write.AUDIT_WRITE_DEFECTS` first, so a bug is not read as a fault. + Built on first use, because naming a server driver imports it.""" + return (RuntimeError, OSError, ValueError, *store_driver_errors()) async def _step_up_refusal( @@ -1142,8 +1156,22 @@ async def _step_up_refusal( headers = {"X-Step-Up-Required": "1"} if action is not None: headers["X-Step-Up-Action"] = action - if await auth.session_steps_up_at_idp(token): - headers[_STEP_UP_VIA_HEADER] = "idp" + try: + via_idp = await auth.session_steps_up_at_idp(token) + except AUDIT_WRITE_DEFECTS: + raise + except _session_read_faults() as exc: + # Vault BACKLOG #3145 item 6: the read only picks the wording, and the gate has already + # refused, so a store fault keeps the 403 rather than becoming a 500. It answers with the + # password wording; an oidc client's POST /me/reauth then asks again and is told the IdP. + log.warning( + "a step-up refusal could not read the session's sign-in mechanism (%s); it answers" + " with the password wording", + type(exc).__name__, + ) + via_idp = False + if via_idp: + headers[_STEP_UP_VIA_HEADER] = STEP_UP_VIA_IDP return HTTPException(status.HTTP_403_FORBIDDEN, _IDP_STEP_UP_DETAIL, headers=headers) return HTTPException(status.HTTP_403_FORBIDDEN, _STEP_UP_DETAIL, headers=headers) diff --git a/messagefoundry/apiclient/client.py b/messagefoundry/apiclient/client.py index ee4338daa1..c5eeeff6a9 100644 --- a/messagefoundry/apiclient/client.py +++ b/messagefoundry/apiclient/client.py @@ -29,6 +29,8 @@ from pydantic import BaseModel, ValidationError from messagefoundry.api.auth_models import ( + STEP_UP_VIA_HEADER, + STEP_UP_VIA_IDP, AdGroupMap, AdGroupMapEntry, AuditList, @@ -147,8 +149,10 @@ class IdpStepUpRequired(ApiError): Raised on a 403 carrying ``X-Step-Up-Via: idp``, which the engine sends for a session minted by the federated (``oidc``) sign-in. ``POST /me/reauth`` refuses such a session, so this client - raises instead of prompting for a password. Its step-up runs in the web console at - ``/ui/reauth``, through the IdP, and this client cannot perform it: that leg needs a browser and + raises instead of prompting for a password. Its step-up runs in the web console, through the + IdP: repeating the action there sends the browser to ``/ui/reauth`` with that action named + (a bare ``/ui/reauth`` names none and returns to ``/ui``; vault BACKLOG #3145). This client + cannot perform it: that leg needs a browser and re-keys the session it steps up. A subclass of :class:`ApiError`, so a caller that catches only that still sees the refusal.""" @@ -1034,8 +1038,13 @@ def _request( **kw, ) # BACKLOG #2158: an ``oidc`` session steps up at the IdP, in a browser. POST /me/reauth - # refuses it, so the password prompt below would only earn a second 403. - if response.status_code == 403 and response.headers.get("X-Step-Up-Via") == "idp": + # refuses it, so the password prompt below would only earn a second 403. Vault BACKLOG + # #3145 item 6 asked to run the registered handler first; declined, because that handler + # can only re-prove a password, which is the second 403. + if ( + response.status_code == 403 + and response.headers.get(STEP_UP_VIA_HEADER) == STEP_UP_VIA_IDP + ): raise IdpStepUpRequired(_error_detail(response), status=response.status_code) # Step-up re-verification (ASVS 7.5.3): the engine refuses a sensitive op with 403 + # X-Step-Up-Required when this session hasn't re-proved its credential recently. Prompt the diff --git a/messagefoundry/auth/oidc_http.py b/messagefoundry/auth/oidc_http.py index 6bfd7ef22a..3c5fe80a21 100644 --- a/messagefoundry/auth/oidc_http.py +++ b/messagefoundry/auth/oidc_http.py @@ -42,6 +42,7 @@ import ssl import urllib.request from collections.abc import Callable +from typing import TYPE_CHECKING from messagefoundry.auth.oidc.jwks import _MAX_JWKS_BYTES from messagefoundry.auth.trust_anchors import oidc_anchor_spec, verified_anchor_cadata @@ -59,7 +60,10 @@ reply_framing_fault, ) -__all__ = ["build_idp_opener", "jwks_fetcher"] +if TYPE_CHECKING: + from messagefoundry.config.settings import AuthSettings + +__all__ = ["build_idp_opener", "build_idp_opener_from_settings", "jwks_fetcher"] #: Wall-clock bound for a single IdP HTTP round trip. Deliberately short: both legs run inside a #: browser request, and a hung IdP must degrade the federated login, never pin a worker thread. @@ -143,6 +147,28 @@ def build_idp_opener( return build_strict_opener(_NoRedirectHandler, urllib.request.HTTPSHandler(context=ctx)) +def build_idp_opener_from_settings( + settings: AuthSettings, *, enforcing: bool, anchor: bool = True +) -> urllib.request.OpenerDirector: + """:func:`build_idp_opener` with its arguments read from ``[auth]``, so the engine and the + federation check cannot read the anchor, pin or CRL file differently (vault BACKLOG #3145 item + 3). Only the argument assembly is shared: each caller keeps its own answer to a failure, a + refused start in ``AuthService`` and a check row in ``verify.federation``. + + ``anchor=False`` leaves the anchor and its pin out, for the federation check's revocation row + alone. The CRL file is the hop's own knob (BACKLOG #299) and is passed either way. + + The settings preamble the check legs in ``messagefoundry/checks.py`` repeat is not shared here: + that refactor is declined, because it would collide with vault BACKLOG #3063 on the same + helper.""" + return build_idp_opener( + settings.oidc_tls_ca_cert_file if anchor else None, + pin=settings.oidc_tls_ca_cert_pin if anchor else None, + enforcing=enforcing, + crl_file=settings.oidc_tls_crl_file, + ) + + def jwks_fetcher( jwks_uri: str, opener: urllib.request.OpenerDirector, diff --git a/messagefoundry/auth/service.py b/messagefoundry/auth/service.py index 4335c5e8aa..95faa6c05a 100644 --- a/messagefoundry/auth/service.py +++ b/messagefoundry/auth/service.py @@ -91,7 +91,7 @@ notice_kind_log_label, ) from messagefoundry.auth.oidc import PendingFlow -from messagefoundry.auth.oidc_http import build_idp_opener, jwks_fetcher +from messagefoundry.auth.oidc_http import build_idp_opener_from_settings, jwks_fetcher from messagefoundry.auth.passwords import hash_password, needs_rehash, verify_password from messagefoundry.auth.permissions import ( CUSTOM_ROLE_ID_PREFIX, @@ -2622,13 +2622,10 @@ def __init__( # preflight honors [security].enforcement (warn ≠ refuse) — a pin mismatch always refuses, # a group/world-writable DACL refuses only at enforce. Without the dial this seam would # hardcode enforce and make warn-mode startup unreachable for the OIDC anchor. - self._oidc_opener = build_idp_opener( - settings.oidc_tls_ca_cert_file, - pin=settings.oidc_tls_ca_cert_pin, - enforcing=self._trust_anchors_enforcing, - # BACKLOG #299: revocation checking against the IdP certificate. This hop resolves no - # trust anchor, so it carries its own CRL setting rather than inheriting [tls].crl_file. - crl_file=settings.oidc_tls_crl_file, + # BACKLOG #299: the CRL file is this hop's own knob, not [tls].crl_file; the helper + # passes it. The federation check builds through the same helper (vault #3145). + self._oidc_opener = build_idp_opener_from_settings( + settings, enforcing=self._trust_anchors_enforcing ) # BACKLOG #1887: must follow the opener, whose finished context it reads. _refuse_idp_revocation(settings, self._oidc_opener, hop_posture) @@ -4848,9 +4845,11 @@ async def abandon_oidc_step_up( return flow actor = await self._step_up_actor(flow) if flow.step_up_session_hash is None: - return await self._step_up_refused(FLOW_PURPOSE_MISMATCH, actor=actor, client=client) + return await self._step_up_refused( + FLOW_PURPOSE_MISMATCH, actor=actor, client=client, flow=flow + ) return await self._step_up_refused( - reason, actor=actor, client=client, return_to=flow.return_to + reason, actor=actor, client=client, return_to=flow.return_to, flow=flow ) async def _redeem_step_up_flow( @@ -4910,8 +4909,9 @@ async def complete_oidc_step_up( The session is re-anchored to the callback's address when the callback has one. The ``auth.reauth`` row a completed step-up writes, including one whose rotation lost the session, also records the start leg's address as ``start_client`` and whether the two are - different hosts as ``client_moved`` (BACKLOG #2160). The refusal rows above carry neither. - A move is recorded, never refused. Behind a proxy the engine does not trust, both legs carry + different hosts as ``client_moved`` (BACKLOG #2160). So does a refusal made after the flow + was redeemed (vault BACKLOG #3145); one made before carries neither. A move is recorded, + never refused. Behind a proxy the engine does not trust, both legs carry the proxy's address, so ``client_moved`` reads false there. Not padded to a deadline, unlike the sign-in callback: the caller already holds a session, @@ -4926,7 +4926,9 @@ async def complete_oidc_step_up( actor = await self._step_up_actor(flow) token_hash = flow.step_up_session_hash if token_hash is None: - return await self._step_up_refused(FLOW_PURPOSE_MISMATCH, actor=actor, client=client) + return await self._step_up_refused( + FLOW_PURPOSE_MISMATCH, actor=actor, client=client, flow=flow + ) # BACKLOG #2301 (ASVS 2.4.2): max_age=0 and prompt=login make the person authenticate at the # IdP, so a step-up callback faster than a person can do that is refused. Before the code is # redeemed, so the refused flow spends nothing at the token endpoint. @@ -4936,7 +4938,7 @@ async def complete_oidc_step_up( arrived_too_early = self._oidc_callback_too_early(flow) if arrived_too_early and not self._oidc_callback_floor_exempt_values(): return await self._step_up_refused( - TOO_EARLY, actor=actor, client=client, return_to=return_to + TOO_EARLY, actor=actor, client=client, return_to=return_to, flow=flow ) try: principal_claims = await asyncio.to_thread( @@ -4945,7 +4947,7 @@ async def complete_oidc_step_up( except oidc.ClaimsError as exc: # exc.reason is closed-set, so nothing IdP-influenced reaches the audit row. return await self._step_up_refused( - exc.reason, actor=actor, client=client, return_to=return_to + exc.reason, actor=actor, client=client, return_to=return_to, flow=flow ) except oidc.TokenRefusedError as exc: # The endpoint answered. As on the sign-in leg, that is not an outage (#1948), and the @@ -4956,6 +4958,7 @@ async def complete_oidc_step_up( client=client, return_to=return_to, extra={"status": exc.status}, + flow=flow, ) except (OSError, ValueError, http.client.HTTPException) as exc: # The sign-in leg's outage arm, for the same reasons (#1948). Only the exception TYPE, @@ -4967,6 +4970,7 @@ async def complete_oidc_step_up( client=client, return_to=return_to, extra=_idp_error_fields(exc), + flow=flow, ) self.mark_oidc_unavailable(type(exc).__name__) return await self._step_up_refused( @@ -4975,6 +4979,7 @@ async def complete_oidc_step_up( client=client, return_to=return_to, extra=_idp_error_fields(exc), + flow=flow, ) floor_exempt_amr = ( self._oidc_callback_floor_exemption(principal_claims.amr) if arrived_too_early else () @@ -4982,7 +4987,7 @@ async def complete_oidc_step_up( if arrived_too_early and not floor_exempt_amr: # The opted-in site's too-early callback whose amr names no exempt value (BACKLOG #2388). return await self._step_up_refused( - TOO_EARLY, actor=actor, client=client, return_to=return_to + TOO_EARLY, actor=actor, client=client, return_to=return_to, flow=flow ) now = time.time() session = await self._load_session(token_hash) @@ -4996,12 +5001,22 @@ async def complete_oidc_step_up( or not session.is_live(now=now, idle_seconds=self.session_idle_seconds) ): return await self._step_up_refused( - "session_gone", actor=actor, client=client, return_to=return_to, lost=True + "session_gone", + actor=actor, + client=client, + return_to=return_to, + lost=True, + flow=flow, ) user = await self._load_user(session.user_id) if user is None or user.disabled or user.auth_provider != AuthProvider.AD.value: return await self._step_up_refused( - "session_gone", actor=actor, client=client, return_to=return_to, lost=True + "session_gone", + actor=actor, + client=client, + return_to=return_to, + lost=True, + flow=flow, ) if (user.oidc_issuer, user.oidc_subject) != ( principal_claims.issuer, @@ -5016,7 +5031,7 @@ async def complete_oidc_step_up( # the claims ladder is filed as a subject mismatch whatever its auth_time. Both tests # below judge this session's identity, so neither is the right name for someone else. return await self._step_up_refused( - STEP_UP_SUBJECT_MISMATCH, actor=actor, client=client, return_to=return_to + STEP_UP_SUBJECT_MISMATCH, actor=actor, client=client, return_to=return_to, flow=flow ) # FRESHNESS, two tests, and both must pass. max_age=0 and prompt=login ask the IdP to # authenticate the user afresh, so a conforming IdP's auth_time postdates this request. @@ -5025,7 +5040,7 @@ async def complete_oidc_step_up( skew = self._settings.oidc_clock_skew_seconds if flow.issued_at <= 0 or principal_claims.auth_time < flow.issued_at - skew: return await self._step_up_refused( - STEP_UP_NOT_FRESH, actor=actor, client=client, return_to=return_to + STEP_UP_NOT_FRESH, actor=actor, client=client, return_to=return_to, flow=flow ) # (b) Against the IdP's own clock (BACKLOG #2143): auth_time must be LATER than the one the # session holds, which is the sign-in's or the last step-up's. No skew applies, because @@ -5041,11 +5056,19 @@ async def complete_oidc_step_up( held_auth_time = session.idp_auth_time if held_auth_time is None: return await self._step_up_refused( - STEP_UP_IDP_AUTH_TIME_MISSING, actor=actor, client=client, return_to=return_to + STEP_UP_IDP_AUTH_TIME_MISSING, + actor=actor, + client=client, + return_to=return_to, + flow=flow, ) if principal_claims.auth_time <= held_auth_time: return await self._step_up_refused( - STEP_UP_IDP_AUTH_TIME_NOT_LATER, actor=actor, client=client, return_to=return_to + STEP_UP_IDP_AUTH_TIME_NOT_LATER, + actor=actor, + client=client, + return_to=return_to, + flow=flow, ) # The DIRECTORY still has the account. The password re-bind this leg replaces failed for a # disabled or deleted AD object, and the sign-in leg refuses one as not_in_directory. An IdP @@ -5053,13 +5076,17 @@ async def complete_oidc_step_up( # the directory itself, by the row's immutable id as the sign-in leg does (BACKLOG #1532). if self._ldap is None: return await self._step_up_refused( - "not_configured", actor=actor, client=client, return_to=return_to + "not_configured", actor=actor, client=client, return_to=return_to, flow=flow ) if not user.directory_object_id: # Never a name-only lookup: a directory may reissue a freed name to someone else. The # sign-in leg refuses such a row for the same reason (BACKLOG #2027). return await self._step_up_refused( - DIRECTORY_OBJECT_ID_MISSING, actor=actor, client=client, return_to=return_to + DIRECTORY_OBJECT_ID_MISSING, + actor=actor, + client=client, + return_to=return_to, + flow=flow, ) try: principal = await asyncio.to_thread( @@ -5070,18 +5097,18 @@ async def complete_oidc_step_up( "directory lookup failed during a federated step-up: %s", type(exc).__name__ ) return await self._step_up_refused( - "directory_unavailable", actor=actor, client=client, return_to=return_to + "directory_unavailable", actor=actor, client=client, return_to=return_to, flow=flow ) if principal is None: return await self._step_up_refused( - "not_in_directory", actor=actor, client=client, return_to=return_to + "not_in_directory", actor=actor, client=client, return_to=return_to, flow=flow ) # BACKLOG #3174: the answer must name the row's own directory object before its groups are # trusted, by the test the sign-in leg and the AD step-up re-bind apply. mismatch = _directory_answer_mismatch(principal, user.directory_object_id) if mismatch is not None: return await self._step_up_refused( - mismatch, actor=actor, client=client, return_to=return_to + mismatch, actor=actor, client=client, return_to=return_to, flow=flow ) # BACKLOG #2154: the account's stored roles must all be among the roles its current groups # map to, the same test _directory_step_up_refusal applies on the TOTP and passkey legs @@ -5092,11 +5119,16 @@ async def complete_oidc_step_up( fresh = await self._load_user(user.id) if fresh is None: return await self._step_up_refused( - "session_gone", actor=actor, client=client, return_to=return_to, lost=True + "session_gone", + actor=actor, + client=client, + return_to=return_to, + lost=True, + flow=flow, ) if not set(fresh.role_ids) <= await self._ad_roles_for_groups(principal.groups): return await self._step_up_refused( - DIRECTORY_ROLES_DEMOTED, actor=actor, client=client, return_to=return_to + DIRECTORY_ROLES_DEMOTED, actor=actor, client=client, return_to=return_to, flow=flow ) purpose = flow.step_up_purpose # The password leg's three ORDER-CRITICAL steps (see :meth:`reauth`), against the hash. @@ -5123,14 +5155,9 @@ async def complete_oidc_step_up( self._grant_action_step_up(hash_token(elevation.token), purpose) self.clear_oidc_unavailable() # BACKLOG #2160: the start leg staged its caller's address, and the session is re-anchored - # to the callback's above. Record whether the address moved mid-ceremony, compared as the - # new-address signal compares. Recorded only: refusing a move, or anchoring to the start - # address, would change ADR 0142 Amendment B and needs a ruling first. None when either - # address is unknown, since an empty string matches nothing and proves no move. - start_client = flow.client_ip or None - client_moved = ( - not self._same_host(start_client, client) if start_client and client else None - ) + # to the callback's above. Record whether the address moved mid-ceremony. Recorded only: + # refusing a move, or anchoring to the start address, would change ADR 0142 Amendment B + # and needs a ruling first. detail: dict[str, object] = { "ok": elevation.ok, "provider": AuthProvider.AD.value, @@ -5139,8 +5166,7 @@ async def complete_oidc_step_up( "session_lost": elevation.session_lost, "grant_refused": grant_refused, "session_revoked": False, - "start_client": start_client, - "client_moved": client_moved, + **self._step_up_address_fields(flow, client), } if floor_exempt_amr: # BACKLOG #2388: present only when the exemption let a too-early callback through, and @@ -5160,6 +5186,21 @@ async def _step_up_actor(self, flow: PendingFlow) -> str: user = await self._load_user(session.user_id) if session is not None else None return user.username if user is not None else "" + def _step_up_address_fields(self, flow: PendingFlow, client: str | None) -> dict[str, object]: + """``start_client`` and ``client_moved`` for a step-up row (BACKLOG #2160, vault #3145). + + ``start_client`` is the address the start leg staged on the flow, never a value from the + IdP's token. ``client_moved`` compares it with the callback's as the new-address signal + compares (:meth:`_same_host`), and is None when either address is unknown, since an empty + string matches nothing and proves no move.""" + start_client = flow.client_ip or None + return { + "start_client": start_client, + "client_moved": ( + not self._same_host(start_client, client) if start_client and client else None + ), + } + async def _step_up_refused( self, reason: str, @@ -5170,11 +5211,17 @@ async def _step_up_refused( lost: bool = False, extra: Mapping[str, object] | None = None, flow_consumed: bool = True, + flow: PendingFlow | None = None, ) -> OidcStepUp: """Audit and package one refusal of the federated step-up leg. ``reason`` is closed-set. ``extra`` adds fields to the audit row only, such as a received status or an exception - type (BACKLOG #1948). It never carries IdP text.""" + type (BACKLOG #1948). It never carries IdP text. + + ``flow`` adds ``start_client`` and ``client_moved``, as the success row carries them (vault + BACKLOG #3145). Only a refusal made after the flow was redeemed passes it: before ``state`` + matched, the callback is not shown to belong to the flow, and some refusals have no flow + at all, so those rows carry neither field.""" # `extra` first, so it can never overwrite a closed-set field. detail: dict[str, object] = { **(extra or {}), @@ -5182,6 +5229,7 @@ async def _step_up_refused( "provider": AuthProvider.AD.value, "mech": "oidc", "reason": reason, + **(self._step_up_address_fields(flow, client) if flow is not None else {}), } await self._audit("auth.reauth", actor=actor, detail=_json(detail), client=client) return OidcStepUp( diff --git a/messagefoundry/verify/federation.py b/messagefoundry/verify/federation.py index bd2e1d7ba1..6ee2094a01 100644 --- a/messagefoundry/verify/federation.py +++ b/messagefoundry/verify/federation.py @@ -280,17 +280,15 @@ def _idp_opener( return the ``fed.idp_tls`` FAIL or ERROR row that says why it did not build. Reads the anchor and CRL files; opens no socket. ``anchor=False`` leaves the anchor and its pin out, for :func:`idp_revocation_result` alone.""" - from messagefoundry.auth.oidc_http import build_idp_opener + from messagefoundry.auth.oidc_http import build_idp_opener_from_settings from messagefoundry.auth.trust_anchors import TrustAnchorError from messagefoundry.config.settings import SecurityEnforcement - auth = settings.auth try: - return build_idp_opener( - auth.oidc_tls_ca_cert_file if anchor else None, - pin=auth.oidc_tls_ca_cert_pin if anchor else None, + return build_idp_opener_from_settings( + settings.auth, enforcing=settings.security.enforcement is SecurityEnforcement.ENFORCE, - crl_file=auth.oidc_tls_crl_file, + anchor=anchor, ) except OSError as exc: return CheckResult(*_TLS_ROW, Status.FAIL, f"could not build the pinned TLS context: {exc}") diff --git a/tests/test_auth_oidc_service.py b/tests/test_auth_oidc_service.py index 1d74d94213..035cd29975 100644 --- a/tests/test_auth_oidc_service.py +++ b/tests/test_auth_oidc_service.py @@ -348,7 +348,8 @@ def spy( ) return urllib.request.build_opener() - monkeypatch.setattr("messagefoundry.auth.service.build_idp_opener", spy) + # The helper both build sites share (vault BACKLOG #3145) calls the module-level builder. + monkeypatch.setattr("messagefoundry.auth.oidc_http.build_idp_opener", spy) # The CRL path must name a real file: [auth].oidc_tls_crl_file refuses a missing one at load # (BACKLOG #1997). The spy never reads it, so its content does not matter. crl = tmp_path / "idp-crl.pem" diff --git a/tests/test_oidc_json_step_up_hint.py b/tests/test_oidc_json_step_up_hint.py index 5c65fbd963..248287cae4 100644 --- a/tests/test_oidc_json_step_up_hint.py +++ b/tests/test_oidc_json_step_up_hint.py @@ -172,6 +172,9 @@ def _assert_idp_hint(site: str, r: httpx.Response, action: str | None) -> None: assert r.headers.get("X-Step-Up-Action") == action, site detail = r.json()["detail"] assert "/ui/reauth" in detail, (site, detail) + # Vault BACKLOG #3145 item 6: a bare /ui/reauth names no action and bounces to /ui, so the + # hint sends the reader back through the action, which reaches /ui/reauth with it named. + assert "repeat the action in the web console" in detail, (site, detail) assert "/me/reauth" not in detail, (site, detail) # BACKLOG #2142: a code opens no window on an oidc session, so the hint must not name it. assert "mfa-verify" not in detail, (site, detail) @@ -402,3 +405,35 @@ def _prompt() -> bool: client.close() assert not isinstance(raised.value, IdpStepUpRequired) assert prompted == ["step-up"] + + +def test_the_step_up_header_is_one_constant_on_both_sides() -> None: + """Vault BACKLOG #3145 item 6: the gates, the two route refusals and the FastAPI-free client + read one spelling, from the auth wire models, rather than four copies of a literal.""" + from messagefoundry.api import auth_models + from messagefoundry.apiclient import client as apiclient + + assert _STEP_UP_VIA_HEADER is auth_models.STEP_UP_VIA_HEADER == "X-Step-Up-Via" + assert apiclient.STEP_UP_VIA_HEADER is auth_models.STEP_UP_VIA_HEADER + assert apiclient.STEP_UP_VIA_IDP is auth_models.STEP_UP_VIA_IDP == "idp" + package = Path(auth_models.__file__).parent.parent + for module in ("api/auth_routes.py", "apiclient/client.py", "api/security.py"): + text = (package / module).read_text(encoding="utf-8") + assert '"X-Step-Up-Via"' not in text, f"{module} spells the header out again" + + +async def test_a_store_fault_reading_the_mechanism_keeps_the_403( + engine: Engine, service: AuthService, monkeypatch: pytest.MonkeyPatch +) -> None: + """RED when: a store fault in the read that picks the refusal's wording turns the gate's 403 + into a 500 (vault BACKLOG #3145 item 6). It keeps the refusal and the password wording.""" + uid = await _add_admin(service) + token = await _session(service, uid, "tok-fault-oidc", mechanism="oidc") + + async def refused_read(token: str | None) -> bool: + raise OSError("the store connection dropped") + + monkeypatch.setattr(service, "session_steps_up_at_idp", refused_read) + async with _client_at(engine, service, HOME) as c: + r = await c.post("/me/mfa/enroll", headers=_auth(token)) + _assert_password_refusal("require_reauth_only_action", r, STEP_UP_ACTION_MFA_ENROLL) diff --git a/tests/test_oidc_step_up.py b/tests/test_oidc_step_up.py index fb47e46204..d953bb97ef 100644 --- a/tests/test_oidc_step_up.py +++ b/tests/test_oidc_step_up.py @@ -413,6 +413,51 @@ async def test_the_step_up_row_records_whether_the_address_moved_mid_ceremony( await store.close() +async def test_a_refusal_after_the_flow_is_redeemed_records_the_address_fields( + rsa_key: rsa.RSAPrivateKey, monkeypatch: pytest.MonkeyPatch +) -> None: + """RED when: a step-up refused after its flow was redeemed leaves out ``start_client`` and + ``client_moved``, which the success row carries (vault BACKLOG #3145 item 4). They come from the + flow the start leg staged, never from the token.""" + store = await MessageStore.open(":memory:") + try: + service = await _service(store, rsa_key) + token = await _oidc_session(service, monkeypatch, rsa_key) + flow_id, _url = await _begin(service, token, client="10.0.0.8") + stale = _staged(service, flow_id).issued_at - 2 * service._settings.oidc_clock_skew_seconds + out = await _return_from_idp( + service, monkeypatch, rsa_key, flow_id, client="10.0.0.9", auth_time=stale - 1 + ) + + assert not out.ok and out.reason == STEP_UP_NOT_FRESH + detail = json.loads(str((await _audit_rows(store, "auth.reauth"))[-1]["detail"])) + assert detail["reason"] == STEP_UP_NOT_FRESH + assert (detail["start_client"], detail["client_moved"]) == ("10.0.0.8", True) + finally: + await store.close() + + +async def test_a_refusal_before_the_state_matches_records_no_address_fields( + rsa_key: rsa.RSAPrivateKey, monkeypatch: pytest.MonkeyPatch +) -> None: + """The control for vault BACKLOG #3145 item 4: before ``state`` matches, the callback is not + shown to belong to the flow, so its row compares nothing with the flow's start address.""" + store = await MessageStore.open(":memory:") + try: + service = await _service(store, rsa_key) + token = await _oidc_session(service, monkeypatch, rsa_key) + flow_id, _url = await _begin(service, token, client="10.0.0.8") + out = await service.complete_oidc_step_up( + flow_id=flow_id, state="forged", code=AUTH_CODE, client="10.0.0.9", public_origin=ORIGIN + ) + + assert not out.ok and out.reason == "state_mismatch" + detail = json.loads(str((await _audit_rows(store, "auth.reauth"))[-1]["detail"])) + assert "start_client" not in detail and "client_moved" not in detail + finally: + await store.close() + + # --- the callback: every refusal elevates nothing ----------------------------------------------------- @@ -815,6 +860,9 @@ async def test_an_account_demoted_in_the_directory_since_sign_in_is_refused( "provider": "ad", "mech": "oidc", "reason": DIRECTORY_ROLES_DEMOTED, + # Both legs at the helpers' default address (vault BACKLOG #3145 item 4). + "start_client": "10.0.0.9", + "client_moved": False, } await _assert_untouched(service, token) assert set(await store.get_user_role_ids(user_id)) == {"operator"}, "roles were written" From 6512a27d0ae5ba99cd294d6d02f1dd10044f1297 Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Fri, 16 Oct 2026 17:55:15 -0500 Subject: [PATCH 11/57] Paging refuses a later page that drops its control; Who am I and binds share the unknown-code mapping (BACKLOG #3010) Review round one on the follow-up limb, applied: - A later page with no readable paged results control is refused, not read as the last page; only an empty cookie ends the paging. The first page may still lack it (the control is sent non-critical). - The unknown-code mapping is one context manager, _unknown_code_refused, used by _search, by _socket_faults_as_ldap_error (binds) and around the Who am I extended operation, which ldap3 decodes the same way. A failed Who am I is reported in whoami_error and the group read still runs. - The paging refusals carry their own code_name (too many pages, paging control lost); _MAX_PAGES is described as a count of pages. - The stale "ldap3 raises on none of these codes" docstring is corrected. Proposed PR title: An LDAP search that does not end with success is an outage, not "no such account" (BACKLOG #3010) Proposed vault banner, addition to the one in commit c6113d94: Follow-up limb (branch b257-3010b): the nested group search is paged at 500 per page, so an account in more groups than a directory's unpaged cap would no longer fail sign-in; a page that does not end with success, a later page that drops the paging control, or a search still paging after 20 pages refuses the whole search as unanswered, never a shorter group list. A result code ldap3 has no name for, which its default fast decoder raises KeyError on inside a search, a bind or Who am I, now reads as the directory not answering instead of an unhandled error. Measured through real ldap3 against a loopback server, not a domain controller. Shipped open: no overall deadline across pages, no RFC 2696 abandon of a refused paged search, the socket an unknown-code auto_bind leaves open, no static pin that every ldap3 call reads through the mapping. --- changelog.d/3010.security.md | 10 +-- messagefoundry/auth/ldap.py | 108 +++++++++++++++---------- tests/test_ldap_search_result_codes.py | 96 ++++++++++++++++++---- 3 files changed, 148 insertions(+), 66 deletions(-) diff --git a/changelog.d/3010.security.md b/changelog.d/3010.security.md index 369267a621..fda407722e 100644 --- a/changelog.d/3010.security.md +++ b/changelog.d/3010.security.md @@ -12,9 +12,9 @@ **The nested group search is paged** (the simple paged results control, 500 per page), so an account in more groups than a directory's unpaged cap (AD's MaxPageSize, 1000 by default) would not end it with sizeLimitExceeded and fail sign-in. A page that does not end with success refuses - the whole search, and so does a search still paging after 20 pages; the pages already read are - never used as the whole answer. Paging was measured against a loopback server through ldap3, not - against a domain controller. + the whole search, and so does a later page that drops the paging control, or a search still + paging after 20 pages; the pages already read are never used as the whole answer. Paging was + measured against a loopback server through ldap3, not against a domain controller. **A result code ldap3 has no name for is the same refusal.** ldap3's default decoder raises - `KeyError` on one from inside a search or a bind, which would have surfaced as an unhandled - error at sign-in; it now reads as the directory not answering. (vault `BACKLOG #3010`) + `KeyError` on one from inside a search, a bind or the Who am I read, which would have surfaced as + an unhandled error at sign-in; it now reads as the directory not answering. (vault `BACKLOG #3010`) diff --git a/messagefoundry/auth/ldap.py b/messagefoundry/auth/ldap.py index 4470584f20..9bea7029e9 100644 --- a/messagefoundry/auth/ldap.py +++ b/messagefoundry/auth/ldap.py @@ -123,7 +123,9 @@ class LdapUnansweredError(LdapError): A subclass, so every ``except LdapError`` handles it as before; :func:`_refuse_unanswered` says what that means for each caller. ``code`` is the result code, or ``None`` when the result held - none that is an integer; ``code_name`` is ldap3's name for it, never text the directory sent. + none that is an integer or a paged search was refused for its paging (:func:`_search_paged`); + ``code_name`` is ldap3's name for the code, or the engine's for the case, never text the + directory sent. Both default, so the exception survives a copy or a pickle, which rebuild it from its message and then restore both from its ``__dict__``. """ @@ -777,8 +779,10 @@ def _refuse_unanswered(conn: Any, operation: str, *, answers: frozenset[int] | N unless ``conn``'s last operation ended with a result code in ``answers``, which is success alone unless the caller names another set. - BACKLOG #3010. ldap3 raises on none of these codes here. Every ``Connection`` in this module - keeps ldap3's default ``raise_exceptions=False``, and even under ``True`` ldap3 2.9.1 passes + BACKLOG #3010. ldap3 raises no ``LDAPException`` for any of these codes here. (A code outside + ldap3's own table raises ``KeyError`` instead, which :func:`_unknown_code_refused` maps to this + same refusal.) Every ``Connection`` in this module keeps ldap3's default + ``raise_exceptions=False``, and even under ``True`` ldap3 2.9.1 passes sizeLimitExceeded (4) and timeLimitExceeded (3) through (``DO_NOT_RAISE_EXCEPTIONS``). So a search the directory did not finish -- busy (51), unavailable (52), unwillingToPerform (53), adminLimitExceeded (11), a time or size limit -- came back with no entries, or some. Read on, the @@ -854,34 +858,38 @@ def _search(conn: Any, operation: str, **kwargs: Any) -> None: (:func:`_refuse_unanswered`). Every search in this module goes through here, so none can read either as "no entries"; a test pins that. - A result code ldap3 has no name for is the same refusal; :func:`_unknown_result_code` says + A result code ldap3 has no name for is the same refusal; :func:`_unknown_code_refused` says why it never reaches :func:`_refuse_unanswered`. """ - try: + with _unknown_code_refused(operation): conn.search(**kwargs) - except KeyError as exc: - code = _unknown_result_code(exc) - if code is None: - raise - raise _unanswered(operation, code) from exc _refuse_unanswered(conn, operation) -def _unknown_result_code(exc: KeyError) -> int | None: - """The result code ``exc`` names when it is ldap3 failing to name one, else ``None``. +@contextlib.contextmanager +def _unknown_code_refused(operation: str) -> Iterator[None]: + """Turn ldap3's ``KeyError`` for a result code it has no name for into the refusal + :func:`_refuse_unanswered` raises for any other code that is not an answer. BACKLOG #3010. ldap3 2.9.1 decodes with its fast decoder by default (``Connection(fast_decoder=True)``), and that decoder looks a bind, search or extended result's - code up in ``RESULT_CODES`` (``utils/asn1.py``, ``ldap_result_to_dict_fast``, and - ``operation/bind.py``), so a code outside the table raises ``KeyError(code)`` from inside the - ldap3 call. Unmapped, that would escape every ``except LdapError`` and surface as an unhandled - error at sign-in. Only a ``KeyError`` whose sole argument is an integer missing from that table - is ldap3's; any other is the engine's own defect, and the caller lets it propagate. + code up in ``RESULT_CODES`` (``utils/asn1.py``, ``ldap_result_to_dict_fast``; + ``operation/bind.py``; ``operation/extended.py``), so a code outside the table raises + ``KeyError(code)`` from inside the ldap3 call. Unmapped, that would escape every + ``except LdapError`` and surface as an unhandled error at sign-in. It wraps the search in + :func:`_search`, every bind (through :func:`_socket_faults_as_ldap_error`), and the Who am I + extended operation. Only a ``KeyError`` whose sole argument is an integer missing from that + table is ldap3's; any other is the engine's own defect and propagates. """ from ldap3.core.results import RESULT_CODES # lazy, like every ldap3 import here - code = exc.args[0] if len(exc.args) == 1 else None - return code if type(code) is int and code not in RESULT_CODES else None + try: + yield + except KeyError as exc: + code = exc.args[0] if len(exc.args) == 1 else None + if type(code) is not int or code in RESULT_CODES: + raise + raise _unanswered(operation, code) from exc #: The simple paged results control (RFC 2696), which AD and ldap3 both name by this OID. @@ -892,25 +900,25 @@ def _unknown_result_code(exc: KeyError) -> int | None: #: returns smaller pages, which is still the whole answer. _PAGE_SIZE = 500 -#: Pages read before a paged search is refused as unfinished: 10,000 entries. Each page may take up -#: to ``[auth].ad_receive_timeout``, so a directory that keeps returning a cookie would otherwise -#: hold the sign-in or reconcile worker for as long as it keeps answering. +#: Pages read before a paged search is refused as unfinished. A count of pages, not of entries: at +#: :data:`_PAGE_SIZE` it is 10,000 entries, and fewer where a directory returns shorter pages. Each +#: page may take up to ``[auth].ad_receive_timeout``, so a directory that keeps returning a cookie +#: would otherwise hold the sign-in or reconcile worker for as long as it keeps answering. _MAX_PAGES = 20 def _paged_cookie(conn: Any) -> bytes | None: - """The cookie ``conn``'s last search returned for the next page, or ``None`` at the last page. + """The cookie ``conn``'s last search returned: empty at the last page (RFC 2696), and ``None`` + when the result carries no paged results control that can be read. - ldap3 decodes the control into ``result['controls'][oid]['value']['cookie']``. Absent, the - directory ignored the request for pages, which is allowed because it is sent non-critical; that - answer is then the whole answer or ends with a code :func:`_refuse_unanswered` refuses. + ldap3 decodes the control into ``result['controls'][oid]['value']['cookie']``. """ result = conn.result controls = result.get("controls") if isinstance(result, dict) else None control = controls.get(_PAGED_RESULTS_OID) if isinstance(controls, dict) else None value = control.get("value") if isinstance(control, dict) else None cookie = value.get("cookie") if isinstance(value, dict) else None - return cookie if isinstance(cookie, bytes) and cookie else None + return cookie if isinstance(cookie, bytes) else None def _search_paged(conn: Any, operation: str, **kwargs: Any) -> list[Any]: @@ -920,21 +928,34 @@ def _search_paged(conn: Any, operation: str, **kwargs: Any) -> list[Any]: refuses the whole search, and the pages already read are dropped with it: a part of the answer is never used as the whole. Unpaged, an account in more groups than the cap would end the nested group search with sizeLimitExceeded, which that refusal turns into a failed sign-in. + + **A first page with no readable control is the whole answer**: the control is sent + non-critical, so a directory may ignore it, and that answer either is complete or ends with a + code :func:`_refuse_unanswered` refuses. **A later page with none is refused**: the directory + was paging, and a page that does not say whether it is the last cannot be read as the last. """ entries: list[Any] = [] cookie: bytes | None = None - for _ in range(_MAX_PAGES): + for page in range(_MAX_PAGES): _search(conn, operation, paged_size=_PAGE_SIZE, paged_cookie=cookie, **kwargs) entries.extend(conn.entries) cookie = _paged_cookie(conn) - if cookie is None: + if cookie is None and page: + raise _paging_refused(operation, "paging control lost", "dropped the paging control") + if not cookie: return entries + raise _paging_refused(operation, "too many pages", f"was still paging after {_MAX_PAGES} pages") + + +def _paging_refused(operation: str, case: str, what: str) -> LdapUnansweredError: + """The refusal for a paged search whose pages each ended with success but whose paging did not + end, warned once per operation and case.""" message = ( - f"AD returned more than {_MAX_PAGES} pages of {_PAGE_SIZE} for the {operation}; the engine " - "reads it as the directory not answering (BACKLOG #3010)." + f"AD {what} on the {operation}; the engine reads it as the directory not answering " + "(BACKLOG #3010)." ) - _warn_once(f"unanswered: {operation}: pages", "%s Reported once.", message) - raise LdapUnansweredError(message) + _warn_once(f"unanswered: {operation}: {case}", "%s Reported once.", message) + return LdapUnansweredError(message, code_name=case) #: What opening an ldap3 socket can raise that is not an ldap3 error (BACKLOG #2566). ldap3's @@ -972,7 +993,7 @@ def _socket_faults_as_ldap_error() -> Iterator[None]: ``authenticate`` wraps the whole decoy bind, which must neither map nor swallow by itself. It also maps a bind that ended with a result code ldap3 has no name for - (:func:`_unknown_result_code`) to the refusal a search gets for one, so it reads as an outage + (:func:`_unknown_code_refused`) to the refusal a search gets for one, so it reads as an outage and not as an unhandled error. Any other ``KeyError`` propagates. An ``LDAPException`` passes through untouched, even one that is also an ``OSError``, @@ -984,7 +1005,8 @@ def _socket_faults_as_ldap_error() -> Iterator[None]: import ldap3 try: - yield + with _unknown_code_refused("bind"): + yield except ldap3.core.exceptions.LDAPException: raise except _SOCKET_FAULTS as exc: @@ -997,11 +1019,6 @@ def _socket_faults_as_ldap_error() -> Iterator[None]: else f"{kind.__module__}.{kind.__qualname__}" ) raise LdapError(f"AD directory call failed: {name}") from exc - except KeyError as exc: - code = _unknown_result_code(exc) - if code is None: - raise - raise _unanswered("bind", code) from exc class LdapAuthenticator: @@ -1612,11 +1629,14 @@ def read_bind_account(self) -> BindAccountReading: # failure, and the group read still runs. whoami_error: str | None = None try: - who = _authzid_text(svc.extend.standard.who_am_i()) - except ldap3.core.exceptions.LDAPException as exc: + with _unknown_code_refused("Who am I"): + who = _authzid_text(svc.extend.standard.who_am_i()) + except (ldap3.core.exceptions.LDAPException, LdapError) as exc: # The type name only, as the group read below does: ldap3's text for an - # extended-operation error carries the directory's own diagnostic message. - who, whoami_error = None, f"Who am I failed: {type(exc).__name__}" + # extended-operation error carries the directory's own diagnostic message. An + # LdapError is a result code ldap3 has no name for (BACKLOG #3010), in engine text. + why = str(exc) if isinstance(exc, LdapError) else type(exc).__name__ + who, whoami_error = None, f"Who am I failed: {why}" try: _search( svc, diff --git a/tests/test_ldap_search_result_codes.py b/tests/test_ldap_search_result_codes.py index cd1b2899d7..0bb9d3f182 100644 --- a/tests/test_ldap_search_result_codes.py +++ b/tests/test_ldap_search_result_codes.py @@ -507,26 +507,51 @@ def test_a_later_page_that_did_not_end_with_success_refuses_the_whole_search( def test_a_directory_that_never_stops_paging_is_refused(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(ldap_module, "_MAX_PAGES", 3) conn = _PagedConn([_page(RESULT_SUCCESS, [object()], b"synthetic-again")]) - with pytest.raises(LdapUnansweredError, match="more than 3 pages"): + with pytest.raises(LdapUnansweredError, match="still paging after 3 pages") as refused: ldap_module._search_paged(conn, "group search") assert len(conn.calls) == 3 + assert (refused.value.code, refused.value.code_name) == (None, "too many pages") -@pytest.mark.parametrize( - "result", - [ - None, - {"result": 0}, - {"result": 0, "controls": None}, - {"result": 0, "controls": {ldap_module._PAGED_RESULTS_OID: None}}, +_UNREADABLE_CONTROLS = [ + pytest.param(None, id="no-result"), + pytest.param({"result": 0}, id="no-controls"), + pytest.param({"result": 0, "controls": None}, id="controls-none"), + pytest.param( + {"result": 0, "controls": {ldap_module._PAGED_RESULTS_OID: None}}, id="control-none" + ), + pytest.param( {"result": 0, "controls": {ldap_module._PAGED_RESULTS_OID: {"value": {"cookie": "x"}}}}, - ], - ids=["no-result", "no-controls", "controls-none", "control-none", "cookie-not-bytes"], -) -def test_a_cookie_that_is_absent_or_unreadable_ends_the_paging(result: object) -> None: + id="cookie-not-bytes", + ), +] + + +@pytest.mark.parametrize("result", _UNREADABLE_CONTROLS) +def test_a_control_that_is_absent_or_unreadable_reads_as_none(result: object) -> None: assert ldap_module._paged_cookie(SimpleNamespace(result=result)) is None +def test_an_empty_cookie_reads_as_the_last_page_not_as_no_control() -> None: + conn = _PagedConn([_page(RESULT_SUCCESS, [], b"")]) + conn.search() + assert ldap_module._paged_cookie(conn) == b"" + + +@pytest.mark.parametrize("result", _UNREADABLE_CONTROLS[1:]) +def test_a_later_page_that_drops_the_paging_control_refuses_the_whole_search( + result: dict[str, Any], +) -> None: + """The directory was paging, so a page that does not say whether it is the last cannot be + read as the last: the pages read so far are not the whole answer.""" + later = {**result, "_entries": [object()]} + conn = _PagedConn([_page(RESULT_SUCCESS, [object()], b"synthetic-cookie-1"), later]) + with pytest.raises(LdapUnansweredError, match="dropped the paging control") as refused: + ldap_module._search_paged(conn, "group search") + assert (refused.value.code, refused.value.code_name) == (None, "paging control lost") + assert len(conn.calls) == 2 + + def _paged_groups(monkeypatch: pytest.MonkeyPatch, pages: list[dict[str, Any]]) -> list[object]: """Over the ``_install_fakes`` doubles, answer the n-th group search with ``pages[n]``. Returns the cookies each group search sent.""" @@ -639,6 +664,9 @@ def test_any_other_key_error_is_the_engines_own_and_propagates(args: tuple[objec _USER_DN = "CN=alice,OU=Users,DC=example,DC=com" +#: RFC 4511 ExtendedRequest and ExtendedResponse, [APPLICATION 23] and [APPLICATION 24]. +_EXTENDED_REQUEST, _EXTENDED_RESPONSE = 0x77, 0x78 + #: One message a loopback server sends: its protocolOp, and its controls (empty for none). _Reply = tuple[bytes, bytes] @@ -665,11 +693,14 @@ def _entry(dn: str) -> _Reply: class _ScriptedDc: """A loopback LDAP server, after ``tests/test_ldap_referrals.py``'s ``_FakeDc``. Binds end with - ``bind_code``; each search is answered by ``answer(raw request)``. ``searches`` holds every + ``bind_code``, an extended operation such as Who am I with ``extended_code``; each search is + answered by ``answer(raw request)``. ``searches`` holds every search request's bytes.""" - def __init__(self, answer: Callable[[bytes], list[_Reply]], bind_code: int) -> None: - self._answer, self._bind_code = answer, bind_code + def __init__( + self, answer: Callable[[bytes], list[_Reply]], bind_code: int, extended_code: int + ) -> None: + self._answer, self._bind_code, self._extended_code = answer, bind_code, extended_code self.searches: list[bytes] = [] self._listener = socket.create_server(("127.0.0.1", 0)) self.port: int = self._listener.getsockname()[1] @@ -692,6 +723,8 @@ def _session(self, sock: socket.socket) -> None: op = body[2 + body[1]] if op == _BIND_REQUEST: replies = [_done(self._bind_code, tag=_BIND_RESPONSE)] + elif op == _EXTENDED_REQUEST: + replies = [_done(self._extended_code, tag=_EXTENDED_RESPONSE)] elif op == _SEARCH_REQUEST: self.searches.append(raw) replies = self._answer(raw) @@ -709,11 +742,14 @@ def close(self) -> None: @contextlib.contextmanager def _scripted_dc( - answer: Callable[[bytes], list[_Reply]], *, bind_code: int = RESULT_SUCCESS + answer: Callable[[bytes], list[_Reply]], + *, + bind_code: int = RESULT_SUCCESS, + extended_code: int = RESULT_SUCCESS, ) -> Iterator[tuple[_ScriptedDc, LdapAuthenticator]]: """A loopback server, and an authenticator pointed at it over plain ``ldap://``: the property under test is the decode, not the hop's TLS.""" - dc = _ScriptedDc(answer, bind_code) + dc = _ScriptedDc(answer, bind_code, extended_code) try: settings = _ad_settings( ad_server=f"ldap://127.0.0.1:{dc.port}", ad_allow_insecure_ldap=True @@ -858,3 +894,29 @@ def test_the_bind_account_read_names_the_code_and_keeps_the_identity( assert "distinguished name" not in problem and "may not read" not in problem else: assert hint in problem + + +def test_a_who_am_i_ending_with_an_unknown_code_is_reported_and_the_group_read_still_runs() -> None: + """ldap3's decoder raises the same ``KeyError`` on an extended response. A failed Who am I is + not a bind failure, so it is reported in ``whoami_error`` and the group read still runs.""" + conn = _conn(_result(RESULT_SUCCESS), entries=[]) + conn.extend = SimpleNamespace(standard=SimpleNamespace(who_am_i=_raising(4242))) + conn.unbind = lambda: True + authenticator = LdapAuthenticator(_ad_settings()) + authenticator._service_conn = lambda: conn # type: ignore[method-assign] + reading = authenticator.read_bind_account() + assert reading.bound and reading.authzid is None + assert reading.whoami_error is not None + assert "Who am I with result code 4242 (an unknown result code)" in reading.whoami_error + assert reading.problem is not None and "returned no entry" in reading.problem + + +def test_real_ldap3_maps_a_who_am_i_that_ends_with_an_unknown_code() -> None: + with _scripted_dc(lambda _raw: [_done(RESULT_SUCCESS)], extended_code=4242) as ( + dc, + authenticator, + ): + reading = authenticator.read_bind_account() + assert reading.bound and reading.authzid is None + assert reading.whoami_error is not None and "result code 4242" in reading.whoami_error + assert len(dc.searches) == 1, "the group read did not run after the failed Who am I" From 7a6961afd0c8b32c04ed41e9d1160b696d20a803 Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Fri, 16 Oct 2026 18:12:57 -0500 Subject: [PATCH 12/57] The step-up header test reads the shared constant through auth_models only (BACKLOG #3145) mypy strict on tests refused reading the constant off apiclient.client, which does not re-export it. The test now pins the constant's value in auth_models and that each of the three modules names it and no longer spells the literal. --- tests/test_oidc_json_step_up_hint.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_oidc_json_step_up_hint.py b/tests/test_oidc_json_step_up_hint.py index 248287cae4..6ce49a6c7c 100644 --- a/tests/test_oidc_json_step_up_hint.py +++ b/tests/test_oidc_json_step_up_hint.py @@ -411,15 +411,16 @@ def test_the_step_up_header_is_one_constant_on_both_sides() -> None: """Vault BACKLOG #3145 item 6: the gates, the two route refusals and the FastAPI-free client read one spelling, from the auth wire models, rather than four copies of a literal.""" from messagefoundry.api import auth_models - from messagefoundry.apiclient import client as apiclient assert _STEP_UP_VIA_HEADER is auth_models.STEP_UP_VIA_HEADER == "X-Step-Up-Via" - assert apiclient.STEP_UP_VIA_HEADER is auth_models.STEP_UP_VIA_HEADER - assert apiclient.STEP_UP_VIA_IDP is auth_models.STEP_UP_VIA_IDP == "idp" + assert auth_models.STEP_UP_VIA_IDP == "idp" + # The client's use of the constant is behavioural: the replay tests above raise + # IdpStepUpRequired on the gate's own header. This pins that no copy of the literal is back. package = Path(auth_models.__file__).parent.parent for module in ("api/auth_routes.py", "apiclient/client.py", "api/security.py"): text = (package / module).read_text(encoding="utf-8") assert '"X-Step-Up-Via"' not in text, f"{module} spells the header out again" + assert "STEP_UP_VIA_HEADER" in text, f"{module} no longer reads the shared constant" async def test_a_store_fault_reading_the_mechanism_keeps_the_403( From 6fcd43b48f593e3738cc9dd74fce222dc9133226 Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Fri, 16 Oct 2026 18:14:49 -0500 Subject: [PATCH 13/57] The API client's sign-in and queued elevations wait 30 seconds (BACKLOG #3260 #3145 #2950) Item #2950. EngineClient.login, reauth and verify_mfa pass timeout=httpx.Timeout(SIGN_IN_TIMEOUT_SECONDS), a named 30-second constant matching the IDE's SIGN_IN_TIMEOUT_MS, the way integrity_check passes its own. reauth queues behind the per-account re-proof lock and verify_mfa behind the credential lock, and both rotate the token, so they get the same timeout. The engine's queue (keyed_lock.hold_keyed_lock) has no bound, so no value covers every wait: the login docstring says a lost reply is rarer, still possible, and why. Every other request keeps the 5-second default. apiclient stays FastAPI-free. New test tests/test_apiclient_sign_in_timeout.py reads request.extensions["timeout"] on an httpx.MockTransport: the three requests carry 30 s on every phase and /auth/me keeps 5 s; a second test reads the IDE's constant from ide/src/signInSession.ts. Proposed PR title: b257 D1+D2 auth follow-ups: TOTP refusal row, directory notices, revoke-miss audit, OIDC step-up follow-ups, sign-in timeout Proposed vault banners: #3113 (D1 carry-over, round-two findings 1, 3, 5, 6): BUILT on engine branch b257-d2. The auth.mfa_enroll_refused row is written in a finally around the revoke, so no exception the revoke raises skips it, and aiosqlite's closed-connection ValueError is logged as a store refusal. The directory role notice says the open sessions were ended; the stale notifications.py comment and two SECURITY.md wording points are fixed. Round-two findings 2, 4 and 7 stay open. #3260: limb 2 BUILT on engine branch b257-d2, closing the row with limb 1 (engine PR 2190). revoke_own_session writes one auth.session_revoke_missed row per miss on both planes, through write_audit_soft, with the id cut to 12 characters and nothing else, so an unknown id and a foreign id get the same answer and the same row (owner ruling 2026-10-16, wiki note 20261016T211331989Z-vfzmx7). Register 1af question 4 is answered by this build, and the 1af entry needs a re-draft. #3145: BUILT on engine branch b257-d2. Items 4, 5 and 6(b)(c)(d) built, item 3 built for the opener's argument assembly; items 1, 2, 6(a), the checks.py preamble half of 3, and 6's extra session read declined in one written line each (see the #3145 commit body). #2950: BUILT on engine branch b257-d2. login, reauth and verify_mfa wait 30 s (SIGN_IN_TIMEOUT_SECONDS, the IDE's value); every other request keeps 5 s. The per-account queue has no bound, so a lost reply is rarer, not impossible. --- changelog.d/2950-sign-in-timeout.changed.md | 6 ++ messagefoundry/apiclient/client.py | 48 ++++++++-- tests/test_apiclient_sign_in_timeout.py | 99 +++++++++++++++++++++ 3 files changed, 146 insertions(+), 7 deletions(-) create mode 100644 changelog.d/2950-sign-in-timeout.changed.md create mode 100644 tests/test_apiclient_sign_in_timeout.py diff --git a/changelog.d/2950-sign-in-timeout.changed.md b/changelog.d/2950-sign-in-timeout.changed.md new file mode 100644 index 0000000000..9bc2244f17 --- /dev/null +++ b/changelog.d/2950-sign-in-timeout.changed.md @@ -0,0 +1,6 @@ +- **The API client waits longer for a sign-in.** `EngineClient.login`, `reauth` and `verify_mfa` + now wait up to 30 seconds per phase, matching the IDE's sign-in timeout, instead of the client's + 5 second default. Each can wait in the engine's per-account queue, and a sign-in the client gave + up on could still end the session it replaced, leaving the client signed out. The longer wait + makes that rarer, not impossible, because the queue has no bound. Every other request keeps the + client's timeout. (`vault BACKLOG #2950`) diff --git a/messagefoundry/apiclient/client.py b/messagefoundry/apiclient/client.py index c5eeeff6a9..7c78e88710 100644 --- a/messagefoundry/apiclient/client.py +++ b/messagefoundry/apiclient/client.py @@ -135,6 +135,15 @@ # handlers by test_apiclient_approval_hold.py, which reads the code back rather than trusting 202. _HTTP_PENDING_APPROVAL = 202 +#: How long a request that can wait in the engine's per-account queue may go silent, in seconds +#: (vault BACKLOG #2950): the sign-in, and the two elevations that also rotate the token and queue +#: on that account, ``reauth`` behind the re-proof lock and ``verify_mfa`` behind the credential +#: lock. Matches the IDE's ``SIGN_IN_TIMEOUT_MS`` (``ide/src/signInSession.ts``). The queue has no +#: bound (``keyed_lock.hold_keyed_lock`` takes no timeout), so no value covers every wait: this +#: makes a lost reply rarer than the 5 s default did, and does not rule one out. Every other +#: request keeps the client's own timeout. +SIGN_IN_TIMEOUT_SECONDS = 30.0 + class ApiError(RuntimeError): """An API call failed (transport error, a non-2xx response, or an undecodable 2xx body).""" @@ -1124,7 +1133,17 @@ def reauth(self, password: str, *, purpose: str | None = None) -> None: body: dict[str, str] = {"password": password} if action is not None: body["purpose"] = action - self._adopt_rotated(self._request("POST", "/me/reauth", json=body, _allow_step_up=False)) + # Vault BACKLOG #2950: it rotates the token and queues behind the account's re-proof lock, + # so it gets the sign-in's timeout (see SIGN_IN_TIMEOUT_SECONDS). + self._adopt_rotated( + self._request( + "POST", + "/me/reauth", + json=body, + _allow_step_up=False, + timeout=httpx.Timeout(SIGN_IN_TIMEOUT_SECONDS), + ) + ) def set_mfa_handler(self, handler: Callable[[], bool] | None) -> None: """Register the callback invoked when the engine demands a second factor (403 + @@ -1166,8 +1185,16 @@ def verify_mfa(self, code: str) -> None: for an ``oidc`` session that already met its factor, whose code was not checked either (BACKLOG #2142). Does not itself trigger the MFA handler.""" self._refuse_credential_on_cleartext("a second factor") + # Vault BACKLOG #2950: it rotates the token and queues behind the account's credential + # lock, so it gets the sign-in's timeout (see SIGN_IN_TIMEOUT_SECONDS). self._adopt_rotated( - self._request("POST", "/auth/mfa-verify", json={"code": code}, _allow_mfa=False) + self._request( + "POST", + "/auth/mfa-verify", + json={"code": code}, + _allow_mfa=False, + timeout=httpx.Timeout(SIGN_IN_TIMEOUT_SECONDS), + ) ) def disable_mfa(self) -> None: @@ -1648,10 +1675,12 @@ def login( ends the old one, and the reply then fails to arrive, times out or will not decode, this client still holds the old token, which is dead, and never learns the new one. ``login`` raises :class:`ApiError` and every later call on the held token is refused - with a 401 until the caller signs in again. A timeout is the likely way in: the client - timeout (5 s per phase by default) also bounds this request, and a sign-in waiting in the - engine's per-account queue can take longer, while the engine goes on to finish it. - Before, the old token stayed live in that case. The new session nobody holds stays valid + with a 401 until the caller signs in again. A timeout is the likely way in: a sign-in + waiting in the engine's per-account queue can outlast the client, while the engine goes + on to finish it. This request therefore gets :data:`SIGN_IN_TIMEOUT_SECONDS` (30 s per + phase) rather than the client's 5 s default (vault BACKLOG #2950). That makes a lost + reply rarer, not impossible: the queue has no bound, so a long enough queue still + outlasts it. Before, the old token stayed live in that case. The new session nobody holds stays valid until it idles out or expires, as it did before. This is the accepted price of ending the old session ahead of the cap. It is not harmless. @@ -1663,7 +1692,12 @@ def login( prior = self._token if prior and self._token_cell.issued_here: body["supersedes"] = prior - result = _decode(self._request("POST", "/auth/login", json=body), LoginResponse) + result = _decode( + self._request( + "POST", "/auth/login", json=body, timeout=httpx.Timeout(SIGN_IN_TIMEOUT_SECONDS) + ), + LoginResponse, + ) self._hold_issued(result.token) self._user = result.user # A step-up purpose stashed for the old session must not bind the new one's next reauth(). diff --git a/tests/test_apiclient_sign_in_timeout.py b/tests/test_apiclient_sign_in_timeout.py new file mode 100644 index 0000000000..599bfeb376 --- /dev/null +++ b/tests/test_apiclient_sign_in_timeout.py @@ -0,0 +1,99 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +# Copyright (C) 2026 MessageFoundry Foundation, LLC and contributors +"""The sign-in and the two elevations that queue per account get a longer timeout (vault BACKLOG +#2950). + +``EngineClient.login`` names the token it replaces, and the engine ends that session inside the +sign-in (BACKLOG #2281). A sign-in waiting in the engine's per-account queue could outlast the +client's 5 s default, and a lost reply then leaves the client holding a token the engine ended. +``login``, ``reauth`` and ``verify_mfa`` now carry ``SIGN_IN_TIMEOUT_SECONDS``, which matches the +IDE's ``SIGN_IN_TIMEOUT_MS``; every other request keeps the client's own timeout. The queue has no +bound, so this makes the lost reply rarer, not impossible. + +The transport is an ``httpx.MockTransport``: what is pinned is the timeout each request carries, +read from ``request.extensions["timeout"]``, not what an engine does with it. +""" + +from __future__ import annotations + +from pathlib import Path + +import httpx + +from messagefoundry.apiclient import EngineClient +from messagefoundry.apiclient.client import SIGN_IN_TIMEOUT_SECONDS + +_BASE = "http://127.0.0.1:8765" +_USER = { + "user_id": "u1", + "username": "jdoe", + "auth_provider": "local", + "roles": ["viewer"], + "permissions": [], +} +#: Synthetic, low-entropy stand-ins for a session token. +_TOKEN = "t" * 43 +_ROTATED = "r" * 43 + + +def _reply(request: httpx.Request) -> httpx.Response: + path = request.url.path + if path == "/auth/login": + body: object = {"token": _TOKEN, "user": _USER} + elif path in ("/me/reauth", "/auth/mfa-verify"): + body = {"detail": "re-verified", "token": _ROTATED} + else: + body = _USER + return httpx.Response(200, json=body) + + +def _recording_client() -> tuple[EngineClient, dict[str, dict[str, float | None]]]: + """An ``EngineClient`` over a mock transport, and the timeout each path's request carried.""" + seen: dict[str, dict[str, float | None]] = {} + + def handler(request: httpx.Request) -> httpx.Response: + seen[request.url.path] = dict(request.extensions["timeout"]) + return _reply(request) + + client = EngineClient(_BASE) + default = client._http.timeout + client._http.close() + # The client's own timeout, as _open_transport sets it, so the default is the one under test. + client._http = httpx.Client( + base_url=_BASE, timeout=default, transport=httpx.MockTransport(handler) + ) + return client, seen + + +def _every_phase(seconds: float) -> dict[str, float | None]: + return {"connect": seconds, "read": seconds, "write": seconds, "pool": seconds} + + +def test_the_sign_in_and_the_queued_elevations_carry_the_longer_timeout() -> None: + """RED when: ``/auth/login``, ``/me/reauth`` or ``/auth/mfa-verify`` runs under the client's + 5 s default (vault BACKLOG #2950), or ``/auth/me`` loses that default.""" + client, seen = _recording_client() + try: + client.login("jdoe", "a-strong-test-passphrase") + client.reauth("a-strong-test-passphrase") + client.verify_mfa("123456") + client.me() + finally: + client.close() + longer = _every_phase(SIGN_IN_TIMEOUT_SECONDS) + assert seen["/auth/login"] == longer + assert seen["/me/reauth"] == longer + assert seen["/auth/mfa-verify"] == longer + # The control: any other request keeps the client's default. + assert seen["/auth/me"] == _every_phase(5.0) + + +def test_the_timeout_matches_the_ides_sign_in_timeout() -> None: + """The two clients of the same queue wait the same time. The IDE's constant is read from its + source, so a change on either side reds here.""" + source = Path(__file__).resolve().parents[1] / "ide" / "src" / "signInSession.ts" + line = next( + ln for ln in source.read_text(encoding="utf-8").splitlines() if "SIGN_IN_TIMEOUT_MS =" in ln + ) + milliseconds = int(line.split("=", 1)[1].strip().rstrip(";").replace("_", "")) + assert milliseconds == SIGN_IN_TIMEOUT_SECONDS * 1000 From aaa9f56ebdf70ae18b33c53ae6849463bf505d73 Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Fri, 16 Oct 2026 18:35:10 -0500 Subject: [PATCH 14/57] Simplify pass: one store-fault set, one step-up header name, one bound refusal (BACKLOG #3145 #3113 #2950) - store/base.py gains a cached store_call_errors() (driver errors, RuntimeError, OSError, aiosqlite's closed-connection ValueError). api/security.py's step-up refusal catches it in place of its own copy, the refused enrolment's revoke catches it plus CipherError, and AuthService builds it at startup so the first fault does not import the server drivers on the event loop. - api/security.py drops its private _STEP_UP_VIA_HEADER alias for the shared auth_models constant; the test imports that, and its source scan is replaced by a pin of the wire spelling. - complete_oidc_step_up binds actor, client, return_to and flow once in a partial, so a new refusal arm cannot leave the address fields out. - apiclient builds its sign-in httpx.Timeout once at module level. Skipped: a shared test helper for swapping EngineClient's transport (four copies, three outside this diff); moving the sign-in timeout into _adopt_rotated (confirm_mfa rotates but holds no per-account lock on the engine, so it keeps the default). --- messagefoundry/api/security.py | 22 ++---- messagefoundry/apiclient/client.py | 9 +-- messagefoundry/auth/service.py | 108 ++++++++------------------- messagefoundry/store/base.py | 13 +++- tests/test_oidc_json_step_up_hint.py | 17 ++--- 5 files changed, 57 insertions(+), 112 deletions(-) diff --git a/messagefoundry/api/security.py b/messagefoundry/api/security.py index 5315f5a4b1..ba46bac395 100644 --- a/messagefoundry/api/security.py +++ b/messagefoundry/api/security.py @@ -18,7 +18,6 @@ import time from collections.abc import Awaitable, Callable, Coroutine, Mapping from dataclasses import dataclass -from functools import cache from typing import Any, get_args from fastapi import ( @@ -54,7 +53,7 @@ from messagefoundry.pipeline.alerts import AlertSink, LoggingAlertSink from messagefoundry.pipeline.cert_expiry import peer_cert_expiry from messagefoundry.redaction import safe_exc -from messagefoundry.store.base import store_driver_errors +from messagefoundry.store.base import store_call_errors from messagefoundry.store.store import UserRecord log = logging.getLogger(__name__) @@ -1129,19 +1128,6 @@ async def dependency(request: Request) -> Identity: " browser, not with a password: repeat the action in the web console, which opens /ui/reauth" " for it" ) -#: Set to ``idp`` on an ``oidc`` session's step-up refusal, so a client can branch on a header -#: rather than parse the detail. Absent on every other refusal. -_STEP_UP_VIA_HEADER = STEP_UP_VIA_HEADER - - -@cache -def _session_read_faults() -> tuple[type[Exception], ...]: - """What a refused session read raises: the drivers' errors, plus ``RuntimeError`` and - ``OSError`` as :func:`~messagefoundry.store.base.store_driver_errors` asks, and aiosqlite's - ``ValueError`` for a closed connection. A caller re-raises - :data:`~messagefoundry.audit_write.AUDIT_WRITE_DEFECTS` first, so a bug is not read as a fault. - Built on first use, because naming a server driver imports it.""" - return (RuntimeError, OSError, ValueError, *store_driver_errors()) async def _step_up_refusal( @@ -1160,7 +1146,7 @@ async def _step_up_refusal( via_idp = await auth.session_steps_up_at_idp(token) except AUDIT_WRITE_DEFECTS: raise - except _session_read_faults() as exc: + except store_call_errors() as exc: # Vault BACKLOG #3145 item 6: the read only picks the wording, and the gate has already # refused, so a store fault keeps the 403 rather than becoming a 500. It answers with the # password wording; an oidc client's POST /me/reauth then asks again and is told the IdP. @@ -1171,7 +1157,9 @@ async def _step_up_refusal( ) via_idp = False if via_idp: - headers[_STEP_UP_VIA_HEADER] = STEP_UP_VIA_IDP + # Set to ``idp`` on an ``oidc`` session's step-up refusal, so a client can branch on a + # header rather than parse the detail. Absent on every other refusal. + headers[STEP_UP_VIA_HEADER] = STEP_UP_VIA_IDP return HTTPException(status.HTTP_403_FORBIDDEN, _IDP_STEP_UP_DETAIL, headers=headers) return HTTPException(status.HTTP_403_FORBIDDEN, _STEP_UP_DETAIL, headers=headers) diff --git a/messagefoundry/apiclient/client.py b/messagefoundry/apiclient/client.py index 7c78e88710..726d37ef48 100644 --- a/messagefoundry/apiclient/client.py +++ b/messagefoundry/apiclient/client.py @@ -143,6 +143,7 @@ #: makes a lost reply rarer than the 5 s default did, and does not rule one out. Every other #: request keeps the client's own timeout. SIGN_IN_TIMEOUT_SECONDS = 30.0 +_SIGN_IN_TIMEOUT = httpx.Timeout(SIGN_IN_TIMEOUT_SECONDS) class ApiError(RuntimeError): @@ -1141,7 +1142,7 @@ def reauth(self, password: str, *, purpose: str | None = None) -> None: "/me/reauth", json=body, _allow_step_up=False, - timeout=httpx.Timeout(SIGN_IN_TIMEOUT_SECONDS), + timeout=_SIGN_IN_TIMEOUT, ) ) @@ -1193,7 +1194,7 @@ def verify_mfa(self, code: str) -> None: "/auth/mfa-verify", json={"code": code}, _allow_mfa=False, - timeout=httpx.Timeout(SIGN_IN_TIMEOUT_SECONDS), + timeout=_SIGN_IN_TIMEOUT, ) ) @@ -1693,9 +1694,7 @@ def login( if prior and self._token_cell.issued_here: body["supersedes"] = prior result = _decode( - self._request( - "POST", "/auth/login", json=body, timeout=httpx.Timeout(SIGN_IN_TIMEOUT_SECONDS) - ), + self._request("POST", "/auth/login", json=body, timeout=_SIGN_IN_TIMEOUT), LoginResponse, ) self._hold_issued(result.token) diff --git a/messagefoundry/auth/service.py b/messagefoundry/auth/service.py index 95faa6c05a..597f60571b 100644 --- a/messagefoundry/auth/service.py +++ b/messagefoundry/auth/service.py @@ -37,7 +37,7 @@ from contextvars import ContextVar from dataclasses import dataclass, replace from enum import Enum -from functools import cache +from functools import cache, partial from types import MappingProxyType from typing import Any, Final, Literal, TypeVar from uuid import uuid4 @@ -127,7 +127,7 @@ from messagefoundry.credential import constant_time_equal from messagefoundry.keyed_lock import KeyedLock as _KeyedLock from messagefoundry.keyed_lock import hold_keyed_lock as _hold_keyed_lock -from messagefoundry.store.base import AdminStore, store_driver_errors +from messagefoundry.store.base import AdminStore, store_call_errors, store_driver_errors from messagefoundry.store.crypto import MARKER_PREFIX, CipherError from messagefoundry.store.row_seal import ( ADSET_KIND, @@ -2602,6 +2602,9 @@ def __init__( # Built now, at startup, so a reconciler's first refused audit write does not import # the server drivers on the event loop (see `_audit_write_errors`). _audit_write_errors() + # The same for the soft catches that read store_call_errors (a refused enrolment's revoke, + # the step-up refusal's session read), whichever plane meets a store fault first. + store_call_errors() # Advisory, NON-STICKY federated-IdP health (ADR 0142 AC-8) — see the oidc_available docstring. self._oidc_unavailable_reason: str | None = None self._oidc_client_auth: oidc.ClientAuthentication | None = None @@ -4929,6 +4932,11 @@ async def complete_oidc_step_up( return await self._step_up_refused( FLOW_PURPOSE_MISMATCH, actor=actor, client=client, flow=flow ) + # Every refusal below is made after the flow was redeemed, so each carries the address + # fields (vault BACKLOG #3145): bound once, so a new arm cannot leave them out. + refuse = partial( + self._step_up_refused, actor=actor, client=client, return_to=return_to, flow=flow + ) # BACKLOG #2301 (ASVS 2.4.2): max_age=0 and prompt=login make the person authenticate at the # IdP, so a step-up callback faster than a person can do that is refused. Before the code is # redeemed, so the refused flow spends nothing at the token endpoint. @@ -4937,58 +4945,40 @@ async def complete_oidc_step_up( # refusal waits for it below. With the list empty (the default) nothing changes here. arrived_too_early = self._oidc_callback_too_early(flow) if arrived_too_early and not self._oidc_callback_floor_exempt_values(): - return await self._step_up_refused( - TOO_EARLY, actor=actor, client=client, return_to=return_to, flow=flow - ) + return await refuse(TOO_EARLY) try: principal_claims = await asyncio.to_thread( self._exchange_and_validate, code, flow, self._oidc_redirect_uri(public_origin) ) except oidc.ClaimsError as exc: # exc.reason is closed-set, so nothing IdP-influenced reaches the audit row. - return await self._step_up_refused( - exc.reason, actor=actor, client=client, return_to=return_to, flow=flow - ) + return await refuse(exc.reason) except oidc.TokenRefusedError as exc: # The endpoint answered. As on the sign-in leg, that is not an outage (#1948), and the # status is what tells a junk code from a misconfiguration or a failing IdP. - return await self._step_up_refused( + return await refuse( "token_refused", - actor=actor, - client=client, - return_to=return_to, extra={"status": exc.status}, - flow=flow, ) except (OSError, ValueError, http.client.HTTPException) as exc: # The sign-in leg's outage arm, for the same reasons (#1948). Only the exception TYPE, # and a received status, are kept. if not _is_idp_transport_failure(exc): - return await self._step_up_refused( + return await refuse( IDP_ANSWER_UNUSABLE, - actor=actor, - client=client, - return_to=return_to, extra=_idp_error_fields(exc), - flow=flow, ) self.mark_oidc_unavailable(type(exc).__name__) - return await self._step_up_refused( + return await refuse( "idp_unavailable", - actor=actor, - client=client, - return_to=return_to, extra=_idp_error_fields(exc), - flow=flow, ) floor_exempt_amr = ( self._oidc_callback_floor_exemption(principal_claims.amr) if arrived_too_early else () ) if arrived_too_early and not floor_exempt_amr: # The opted-in site's too-early callback whose amr names no exempt value (BACKLOG #2388). - return await self._step_up_refused( - TOO_EARLY, actor=actor, client=client, return_to=return_to, flow=flow - ) + return await refuse(TOO_EARLY) now = time.time() session = await self._load_session(token_hash) if ( @@ -5000,23 +4990,15 @@ async def complete_oidc_step_up( # stepped up, so the operator is not told "verified" and then signed out. or not session.is_live(now=now, idle_seconds=self.session_idle_seconds) ): - return await self._step_up_refused( + return await refuse( "session_gone", - actor=actor, - client=client, - return_to=return_to, lost=True, - flow=flow, ) user = await self._load_user(session.user_id) if user is None or user.disabled or user.auth_provider != AuthProvider.AD.value: - return await self._step_up_refused( + return await refuse( "session_gone", - actor=actor, - client=client, - return_to=return_to, lost=True, - flow=flow, ) if (user.oidc_issuer, user.oidc_subject) != ( principal_claims.issuer, @@ -5030,18 +5012,14 @@ async def complete_oidc_step_up( # BEFORE both freshness tests (BACKLOG #2143), so another person's answer that passed # the claims ladder is filed as a subject mismatch whatever its auth_time. Both tests # below judge this session's identity, so neither is the right name for someone else. - return await self._step_up_refused( - STEP_UP_SUBJECT_MISMATCH, actor=actor, client=client, return_to=return_to, flow=flow - ) + return await refuse(STEP_UP_SUBJECT_MISMATCH) # FRESHNESS, two tests, and both must pass. max_age=0 and prompt=login ask the IdP to # authenticate the user afresh, so a conforming IdP's auth_time postdates this request. # (a) Against our clock: auth_time is IdP clock and issued_at is ours, so the floor allows # the configured skew for an IdP clock that runs behind. skew = self._settings.oidc_clock_skew_seconds if flow.issued_at <= 0 or principal_claims.auth_time < flow.issued_at - skew: - return await self._step_up_refused( - STEP_UP_NOT_FRESH, actor=actor, client=client, return_to=return_to, flow=flow - ) + return await refuse(STEP_UP_NOT_FRESH) # (b) Against the IdP's own clock (BACKLOG #2143): auth_time must be LATER than the one the # session holds, which is the sign-in's or the last step-up's. No skew applies, because # both values come from the IdP. This closes most of what (a) alone left: an IdP that @@ -5055,38 +5033,24 @@ async def complete_oidc_step_up( # row and the operator's text say which one refused. held_auth_time = session.idp_auth_time if held_auth_time is None: - return await self._step_up_refused( + return await refuse( STEP_UP_IDP_AUTH_TIME_MISSING, - actor=actor, - client=client, - return_to=return_to, - flow=flow, ) if principal_claims.auth_time <= held_auth_time: - return await self._step_up_refused( + return await refuse( STEP_UP_IDP_AUTH_TIME_NOT_LATER, - actor=actor, - client=client, - return_to=return_to, - flow=flow, ) # The DIRECTORY still has the account. The password re-bind this leg replaces failed for a # disabled or deleted AD object, and the sign-in leg refuses one as not_in_directory. An IdP # can still sign such a user in (sync lag, or its own credential store), so the engine asks # the directory itself, by the row's immutable id as the sign-in leg does (BACKLOG #1532). if self._ldap is None: - return await self._step_up_refused( - "not_configured", actor=actor, client=client, return_to=return_to, flow=flow - ) + return await refuse("not_configured") if not user.directory_object_id: # Never a name-only lookup: a directory may reissue a freed name to someone else. The # sign-in leg refuses such a row for the same reason (BACKLOG #2027). - return await self._step_up_refused( + return await refuse( DIRECTORY_OBJECT_ID_MISSING, - actor=actor, - client=client, - return_to=return_to, - flow=flow, ) try: principal = await asyncio.to_thread( @@ -5096,20 +5060,14 @@ async def complete_oidc_step_up( _log.warning( "directory lookup failed during a federated step-up: %s", type(exc).__name__ ) - return await self._step_up_refused( - "directory_unavailable", actor=actor, client=client, return_to=return_to, flow=flow - ) + return await refuse("directory_unavailable") if principal is None: - return await self._step_up_refused( - "not_in_directory", actor=actor, client=client, return_to=return_to, flow=flow - ) + return await refuse("not_in_directory") # BACKLOG #3174: the answer must name the row's own directory object before its groups are # trusted, by the test the sign-in leg and the AD step-up re-bind apply. mismatch = _directory_answer_mismatch(principal, user.directory_object_id) if mismatch is not None: - return await self._step_up_refused( - mismatch, actor=actor, client=client, return_to=return_to, flow=flow - ) + return await refuse(mismatch) # BACKLOG #2154: the account's stored roles must all be among the roles its current groups # map to, the same test _directory_step_up_refusal applies on the TOTP and passkey legs # (#2240). Inline rather than through that helper, because its probe would change this @@ -5118,18 +5076,12 @@ async def complete_oidc_step_up( # as before, through the loader (BACKLOG #2592). fresh = await self._load_user(user.id) if fresh is None: - return await self._step_up_refused( + return await refuse( "session_gone", - actor=actor, - client=client, - return_to=return_to, lost=True, - flow=flow, ) if not set(fresh.role_ids) <= await self._ad_roles_for_groups(principal.groups): - return await self._step_up_refused( - DIRECTORY_ROLES_DEMOTED, actor=actor, client=client, return_to=return_to, flow=flow - ) + return await refuse(DIRECTORY_ROLES_DEMOTED) purpose = flow.step_up_purpose # The password leg's three ORDER-CRITICAL steps (see :meth:`reauth`), against the hash. # (1) Every stamp against the OLD hash, re-anchoring the session to this client address. @@ -10249,7 +10201,7 @@ async def confirm_mfa_enrollment( log=_log, message="a refused TOTP enrolment for %s could not end the session it rotated", args=lambda: (scrub_log_argument(identity.username),), - errors=(*_audit_write_errors(), ValueError), + errors=(*store_call_errors(), CipherError), defects=AUDIT_WRITE_DEFECTS, ) finally: diff --git a/messagefoundry/store/base.py b/messagefoundry/store/base.py index a089041088..e0fd0c470a 100644 --- a/messagefoundry/store/base.py +++ b/messagefoundry/store/base.py @@ -27,7 +27,7 @@ import logging from collections.abc import AsyncIterator, Callable, Collection, Iterable, Mapping, Sequence from dataclasses import dataclass -from functools import partial +from functools import cache, partial from pathlib import Path from typing import TYPE_CHECKING, Any, Protocol, runtime_checkable @@ -3260,6 +3260,17 @@ def store_connect_errors() -> tuple[type[Exception], ...]: return tuple(errors) +@cache +def store_call_errors() -> tuple[type[Exception], ...]: + """What a refused store call can raise, on any backend: :func:`store_driver_errors`, plus + ``RuntimeError`` and ``OSError`` as that function asks a caller to add, and ``ValueError``, + which aiosqlite raises for a closed connection before sqlite3 is reached. A caller that must not + read a bug as a refusal re-raises :data:`~messagefoundry.audit_write.AUDIT_WRITE_DEFECTS` + first. Cached, because naming a server driver imports it: build it at startup, so the first + refusal does not run that import on the event loop (vault BACKLOG #3145, #3113).""" + return (RuntimeError, OSError, ValueError, *store_driver_errors()) + + def store_open_errors() -> tuple[type[Exception], ...]: """:func:`store_driver_errors` and :func:`store_connect_errors`: at least the driver and connection errors a store open can raise, on any backend.""" diff --git a/tests/test_oidc_json_step_up_hint.py b/tests/test_oidc_json_step_up_hint.py index 6ce49a6c7c..00b65e3777 100644 --- a/tests/test_oidc_json_step_up_hint.py +++ b/tests/test_oidc_json_step_up_hint.py @@ -31,7 +31,8 @@ from fastapi import Depends, FastAPI from messagefoundry.api import create_app -from messagefoundry.api.security import _STEP_UP_VIA_HEADER, require_reauth_only +from messagefoundry.api.auth_models import STEP_UP_VIA_HEADER as _STEP_UP_VIA_HEADER +from messagefoundry.api.security import require_reauth_only from messagefoundry.apiclient import ApiError, EngineClient, IdpStepUpRequired from messagefoundry.auth import Role, totp from messagefoundry.auth.identity import ALL_CHANNELS, Identity @@ -407,20 +408,14 @@ def _prompt() -> bool: assert prompted == ["step-up"] -def test_the_step_up_header_is_one_constant_on_both_sides() -> None: +def test_the_shared_step_up_header_keeps_its_wire_spelling() -> None: """Vault BACKLOG #3145 item 6: the gates, the two route refusals and the FastAPI-free client - read one spelling, from the auth wire models, rather than four copies of a literal.""" + read one constant from the auth wire models. Its spelling is the wire contract, so a rename + there would break every older client; the replay tests above pin that each side uses it.""" from messagefoundry.api import auth_models - assert _STEP_UP_VIA_HEADER is auth_models.STEP_UP_VIA_HEADER == "X-Step-Up-Via" + assert auth_models.STEP_UP_VIA_HEADER == "X-Step-Up-Via" assert auth_models.STEP_UP_VIA_IDP == "idp" - # The client's use of the constant is behavioural: the replay tests above raise - # IdpStepUpRequired on the gate's own header. This pins that no copy of the literal is back. - package = Path(auth_models.__file__).parent.parent - for module in ("api/auth_routes.py", "apiclient/client.py", "api/security.py"): - text = (package / module).read_text(encoding="utf-8") - assert '"X-Step-Up-Via"' not in text, f"{module} spells the header out again" - assert "STEP_UP_VIA_HEADER" in text, f"{module} no longer reads the shared constant" async def test_a_store_fault_reading_the_mechanism_keeps_the_403( From 9ed8226addd354236b9836bfdcd24e4edab4ec0f Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Fri, 16 Oct 2026 19:13:47 -0500 Subject: [PATCH 15/57] One ldap3 error boundary decides a decode fault by origin; refused paged searches are abandoned (BACKLOG #3010) _ldap3_call replaces _unknown_code_refused. Every ldap3 call that reads a response runs inside it: _search (each page, and the abandon), the service account's auto_bind, the user and decoy binds, and Who am I. A KeyError, TypeError, IndexError, ValueError or AttributeError that ldap3 raised while reading a response (innermost frame in ldap3, an outer frame in a strategy's get_response) becomes LdapUndecodableError, an outage, labelled with the operation and the fault type and no result code. Anything else propagates. Fixes the two defects measured on b257-3010b: a bind response carrying a referral (KeyError(3)) escaped bare, and a malformed paged results control (TypeError, IndexError) escaped _resolve_groups. A stray tag [22] is no longer called "result code 22". The service bind's socket is closed when its response cannot be decoded. A refused paged search is abandoned (RFC 2696). Coverage removed: the double-based tests that mapped an engine-raised KeyError(4242) are replaced, since that mapping was the defect; the real-ldap3 unknown-code tests moved to tests/test_ldap_decode_boundary.py. --- changelog.d/3010.security.md | 30 +- messagefoundry/auth/ldap.py | 230 +++++++++--- tests/test_ldap_decode_boundary.py | 461 +++++++++++++++++++++++++ tests/test_ldap_search_result_codes.py | 287 ++++++++------- 4 files changed, 793 insertions(+), 215 deletions(-) create mode 100644 tests/test_ldap_decode_boundary.py diff --git a/changelog.d/3010.security.md b/changelog.d/3010.security.md index fda407722e..cbcae0e211 100644 --- a/changelog.d/3010.security.md +++ b/changelog.d/3010.security.md @@ -1,20 +1,26 @@ - **A directory search that does not end with success is an outage, never "no such account".** - Every LDAP search now refuses any result code but success, with or without entries: busy, + Every LDAP search refuses any result code but success, with or without entries: busy, unavailable, unwillingToPerform, adminLimitExceeded, a time or size limit, and noSuchObject on a missing search base. Before, such a search read as no entries, so the session reconciler would have struck a present account toward revocation, and a busy nested group search read as fewer - groups. The reconciler now reads these as unavailable, which never revokes, and a sign-in writes + groups. The reconciler reads these as unavailable, which never revokes, and a sign-in writes `auth.login_error` naming the code. `check-privileges` names the code for the bind-account read. - No directory diagnostic text reaches the error, the log or the audit row. - **The cost:** a code that recurs on every pass, such as a deleted search base, now stops - revocation and the role re-diff for as long as it lasts, with one WARNING per code per process - and no latched alert. + The text of these refusals names the operation, the code and ldap3's name for it, never the + directory's diagnostic message; other directory errors keep the text they had. + **The cost:** a code that recurs on every pass, such as a deleted search base, stops revocation + and the role re-diff for as long as it lasts, with one WARNING per code per process and no + latched alert. **The nested group search is paged** (the simple paged results control, 500 per page), so an - account in more groups than a directory's unpaged cap (AD's MaxPageSize, 1000 by default) would + account in more groups than a directory's unpaged cap (AD's MaxPageSize, 1000 by default) does not end it with sizeLimitExceeded and fail sign-in. A page that does not end with success refuses the whole search, and so does a later page that drops the paging control, or a search still - paging after 20 pages; the pages already read are never used as the whole answer. Paging was - measured against a loopback server through ldap3, not against a domain controller. - **A result code ldap3 has no name for is the same refusal.** ldap3's default decoder raises - `KeyError` on one from inside a search, a bind or the Who am I read, which would have surfaced as - an unhandled error at sign-in; it now reads as the directory not answering. (vault `BACKLOG #3010`) + paging after 20 pages; the pages already read are never used as the whole answer. A refused paged + search is abandoned (RFC 2696) once the directory has sent a cookie. Paging was measured against + a loopback server through ldap3, not against a domain controller. + **A response ldap3 cannot decode is an outage, not an unhandled error.** ldap3's default decoder + raises `KeyError`, `TypeError`, `IndexError` or `AttributeError` on at least a bind response + carrying a referral, a result code outside its table, and a malformed paged results control. A + sign-in that met one would have failed with an unhandled error and no audit row. Each search, + bind and Who am I read now runs inside one boundary that maps such a fault to an outage when ldap3 + raised it while reading the response, and lets any other exception propagate. The service + account's socket is closed when its bind cannot be decoded. (vault `BACKLOG #3010`) diff --git a/messagefoundry/auth/ldap.py b/messagefoundry/auth/ldap.py index 9bea7029e9..a64b7379a8 100644 --- a/messagefoundry/auth/ldap.py +++ b/messagefoundry/auth/ldap.py @@ -138,6 +138,16 @@ def __init__( self.code_name = code_name +class LdapUndecodableError(LdapError): + """ldap3 could not decode the directory's response to an operation (BACKLOG #3010). + + :func:`_ldap3_call` raises it. A subclass, so every ``except LdapError`` reads it as an outage; + its own class keeps a paged search that met one from being abandoned (:func:`_search_paged`). No + result code was read, so none is claimed: the text names the type of the exception ldap3 raised, + never its text or a value the directory sent. + """ + + class DirectoryAnswer(Enum): """What one password-free lookup of one account established (ADR 0195 rule item 2). @@ -779,9 +789,9 @@ def _refuse_unanswered(conn: Any, operation: str, *, answers: frozenset[int] | N unless ``conn``'s last operation ended with a result code in ``answers``, which is success alone unless the caller names another set. - BACKLOG #3010. ldap3 raises no ``LDAPException`` for any of these codes here. (A code outside - ldap3's own table raises ``KeyError`` instead, which :func:`_unknown_code_refused` maps to this - same refusal.) Every ``Connection`` in this module keeps ldap3's default + BACKLOG #3010. ldap3 raises no ``LDAPException`` for any of these codes here. (ldap3's decoder + raises ``KeyError`` on a code outside its own table, which :func:`_ldap3_call` refuses as a + response it could not decode.) Every ``Connection`` in this module keeps ldap3's default ``raise_exceptions=False``, and even under ``True`` ldap3 2.9.1 passes sizeLimitExceeded (4) and timeLimitExceeded (3) through (``DO_NOT_RAISE_EXCEPTIONS``). So a search the directory did not finish -- busy (51), unavailable (52), unwillingToPerform (53), @@ -856,40 +866,112 @@ def _unanswered(operation: str, code: int | None) -> LdapUnansweredError: def _search(conn: Any, operation: str, **kwargs: Any) -> None: """``conn.search(**kwargs)``, refusing a referral result and any other result but success (:func:`_refuse_unanswered`). Every search in this module goes through here, so none can read - either as "no entries"; a test pins that. - - A result code ldap3 has no name for is the same refusal; :func:`_unknown_code_refused` says - why it never reaches :func:`_refuse_unanswered`. + either as "no entries"; a test pins that. The search runs inside :func:`_ldap3_call`. """ - with _unknown_code_refused(operation): + with _ldap3_call(operation): conn.search(**kwargs) _refuse_unanswered(conn, operation) +#: What ldap3 raises, other than its own ``LDAPException``, when a response it reads is malformed. +#: ldap3 2.9.1's default fast decoder indexes the BER it reads without checking it: a context tag a +#: response may not carry is ``KeyError(tag)`` (``utils/asn1.py``, ``decode_sequence``), so a bind +#: response carrying a referral raises ``KeyError(3)``; a result code outside its table is +#: ``KeyError(code)`` (``ldap_result_to_dict_fast``, ``operation/bind.py``, +#: ``operation/extended.py``); a paged results control with no value is ``TypeError`` and one with no +#: cookie is ``IndexError`` (``strategy/base.py``, ``decode_control_fast``); a bind response whose +#: matchedDN is not a string is ``AttributeError`` (``decode_response_fast``). At least these; the +#: decoder trusts every length and tag it reads, so others are reachable. +_DECODE_FAULTS: tuple[type[Exception], ...] = (LookupError, TypeError, ValueError, AttributeError) + +#: The ldap3 strategy methods that read and decode a response. A fault raised under one of them was +#: raised reading what the directory sent, not building what the engine asked for. +_RESPONSE_READERS = frozenset({"get_response", "_get_response"}) + + +def _type_name(exc: BaseException) -> str: + """An exception's type, never its text. ``struct.error``'s own name is just "error", so a type + from outside builtins carries its module.""" + kind = type(exc) + if kind.__module__ == "builtins": + return kind.__qualname__ + return f"{kind.__module__}.{kind.__qualname__}" + + +def _response_reader(exc: BaseException) -> Any: + """The ldap3 strategy that was reading a response when ``exc`` was raised, or ``None`` when + ``exc`` was not raised by ldap3 while reading one. + + Decided by ORIGIN, never by the exception's value: the innermost frame belongs to the ldap3 + package, and an outer frame is one of :data:`_RESPONSE_READERS` in ``ldap3.strategy``. A + ``KeyError(3)`` the decoder raises and a ``KeyError(3)`` engine code raises carry the same + value, and only the first is the directory's. The second condition keeps out what ldap3 raises + while building a request from the engine's own arguments, which is the engine's defect. + """ + reader: Any = None + module = "" + tb = exc.__traceback__ + while tb is not None: + frame = tb.tb_frame + module = str(frame.f_globals.get("__name__", "")) + if ( + reader is None + and module.startswith("ldap3.strategy.") + and frame.f_code.co_name in _RESPONSE_READERS + ): + reader = frame.f_locals.get("self") + tb = tb.tb_next + in_ldap3 = module == "ldap3" or module.startswith("ldap3.") + return reader if in_ldap3 else None + + @contextlib.contextmanager -def _unknown_code_refused(operation: str) -> Iterator[None]: - """Turn ldap3's ``KeyError`` for a result code it has no name for into the refusal - :func:`_refuse_unanswered` raises for any other code that is not an answer. - - BACKLOG #3010. ldap3 2.9.1 decodes with its fast decoder by default - (``Connection(fast_decoder=True)``), and that decoder looks a bind, search or extended result's - code up in ``RESULT_CODES`` (``utils/asn1.py``, ``ldap_result_to_dict_fast``; - ``operation/bind.py``; ``operation/extended.py``), so a code outside the table raises - ``KeyError(code)`` from inside the ldap3 call. Unmapped, that would escape every - ``except LdapError`` and surface as an unhandled error at sign-in. It wraps the search in - :func:`_search`, every bind (through :func:`_socket_faults_as_ldap_error`), and the Who am I - extended operation. Only a ``KeyError`` whose sole argument is an integer missing from that - table is ldap3's; any other is the engine's own defect and propagates. +def _ldap3_call(operation: str, *, close: bool = False) -> Iterator[None]: + """The boundary every ldap3 call that reads a response runs inside: each search and each page + of one, every bind including the service account's ``auto_bind``, and the Who am I extended + operation. A static test pins that. An unbind reads no response, so it runs outside. + + BACKLOG #3010. A :data:`_DECODE_FAULTS` exception that ldap3 raised while reading the + directory's response becomes :class:`LdapUndecodableError`, which every caller reads as an + outage. Unmapped, it would escape every ``except LdapError``, and a sign-in would fail as an + unhandled error with no audit row. An exception from anywhere else propagates unchanged, since + mapped it would hide an engine defect as an outage, which the session reconciler logs only at + debug level (:func:`_response_reader` says how the two are told apart). An ``LDAPException`` + passes through untouched, for its own handler. + + ``close`` closes the socket the failed read was on. The service account's ``auto_bind`` needs + it: that bind runs inside ldap3's ``Connection`` constructor, so when it raises no caller holds + the connection to unbind, and the socket would stay open for as long as the error is kept. """ - from ldap3.core.results import RESULT_CODES # lazy, like every ldap3 import here + import ldap3 try: yield - except KeyError as exc: - code = exc.args[0] if len(exc.args) == 1 else None - if type(code) is not int or code in RESULT_CODES: + except ldap3.core.exceptions.LDAPException: + raise + except _DECODE_FAULTS as exc: + reader = _response_reader(exc) + if reader is None: raise - raise _unanswered(operation, code) from exc + if close: + # BaseStrategy.close shuts the socket without sending anything on a stream that may + # be out of step. Best effort: the refusal below is the answer either way. + with contextlib.suppress(ldap3.core.exceptions.LDAPException, OSError): + reader.close() + raise _undecodable(operation, exc) from exc + + +def _undecodable(operation: str, exc: BaseException) -> LdapUndecodableError: + """The refusal for ``operation`` whose response ldap3 could not decode, warned once per + operation and fault type. The text names the type only: neither a value the directory sent + nor ldap3's text, and no result code, since none was read.""" + fault = _type_name(exc) + message = ( + f"AD answered the {operation} with a response ldap3 could not decode ({fault}); the " + "engine reads it as the directory not answering (BACKLOG #3010)." + ) + _warn_once(f"undecodable: {operation}: {fault}", "%s Reported once.", message) + return LdapUndecodableError(message) #: The simple paged results control (RFC 2696), which AD and ldap3 both name by this OID. @@ -933,18 +1015,53 @@ def _search_paged(conn: Any, operation: str, **kwargs: Any) -> list[Any]: non-critical, so a directory may ignore it, and that answer either is complete or ends with a code :func:`_refuse_unanswered` refuses. **A later page with none is refused**: the directory was paging, and a page that does not say whether it is the last cannot be read as the last. + + **A refused search is abandoned** (:func:`_abandon_paging`) once the directory has sent a + cookie, so it need not hold the paging state until its own timeout. Not after a response ldap3 + could not decode: the cookie in it could not be read either. """ entries: list[Any] = [] - cookie: bytes | None = None - for page in range(_MAX_PAGES): - _search(conn, operation, paged_size=_PAGE_SIZE, paged_cookie=cookie, **kwargs) - entries.extend(conn.entries) - cookie = _paged_cookie(conn) - if cookie is None and page: - raise _paging_refused(operation, "paging control lost", "dropped the paging control") - if not cookie: - return entries - raise _paging_refused(operation, "too many pages", f"was still paging after {_MAX_PAGES} pages") + cookie: bytes | None = None # what the next page is asked with + newest: bytes | None = None # the newest non-empty cookie the directory sent + try: + for page in range(_MAX_PAGES): + _search(conn, operation, paged_size=_PAGE_SIZE, paged_cookie=cookie, **kwargs) + entries.extend(conn.entries) + cookie = _paged_cookie(conn) + if cookie is None and page: + raise _paging_refused( + operation, "paging control lost", "dropped the paging control" + ) + if not cookie: + return entries + newest = cookie + raise _paging_refused( + operation, "too many pages", f"was still paging after {_MAX_PAGES} pages" + ) + except (LdapUnansweredError, LdapReferralError): + # The refusing page was read, so the stream is in step and a cookie in it is readable. + _abandon_paging(conn, operation, _paged_cookie(conn) or newest, kwargs) + raise + + +def _abandon_paging( + conn: Any, operation: str, cookie: bytes | None, kwargs: dict[str, Any] +) -> None: + """Tell the directory to drop a refused paged search's state: the same search again with a page + size of zero and ``cookie``, which RFC 2696 section 3 defines as abandoning it. ldap3 sends a + zero size as given (``Connection.search`` adds the control for any integer ``paged_size``). + + A finding of BACKLOG #3010's review. ``cookie`` is the newest non-empty one the directory sent; + with none, it never held state for this search and nothing is sent. Best effort: the search is + refused whatever this answers, so a refusal of the abandon is warned once by :func:`_search` and + not raised. It costs one more round trip, which can take up to ``[auth].ad_receive_timeout``. + """ + import ldap3 + + if not cookie: + return + with contextlib.suppress(LdapError, ldap3.core.exceptions.LDAPException): + _search(conn, f"{operation} abandon", paged_size=0, paged_cookie=cookie, **kwargs) def _paging_refused(operation: str, case: str, what: str) -> LdapUnansweredError: @@ -992,9 +1109,8 @@ def _socket_faults_as_ldap_error() -> Iterator[None]: of ``tests/test_ldap_referrals.py`` build that connection, so one would fail there first. And ``authenticate`` wraps the whole decoy bind, which must neither map nor swallow by itself. - It also maps a bind that ended with a result code ldap3 has no name for - (:func:`_unknown_code_refused`) to the refusal a search gets for one, so it reads as an outage - and not as an unhandled error. Any other ``KeyError`` propagates. + A response ldap3 could not decode is not a socket fault: each bind also runs inside + :func:`_ldap3_call`, nested inside this wrap, which refuses it first. An ``LDAPException`` passes through untouched, even one that is also an ``OSError``, ``TypeError`` or ``ValueError``, so its own handler keeps its own text. The new text is fixed @@ -1005,20 +1121,12 @@ def _socket_faults_as_ldap_error() -> Iterator[None]: import ldap3 try: - with _unknown_code_refused("bind"): - yield + yield except ldap3.core.exceptions.LDAPException: raise except _SOCKET_FAULTS as exc: - kind = type(exc) - # The type alone, never str(exc). struct.error's own name is just "error", so a type from - # outside builtins carries its module. - name = ( - kind.__qualname__ - if kind.__module__ == "builtins" - else f"{kind.__module__}.{kind.__qualname__}" - ) - raise LdapError(f"AD directory call failed: {name}") from exc + # The type alone, never str(exc). + raise LdapError(f"AD directory call failed: {_type_name(exc)}") from exc class LdapAuthenticator: @@ -1147,7 +1255,10 @@ def _service_conn(self) -> Any: # Server is built outside the wrap to keep engine code out of it. The timeout conversion # stays inline, because tests/test_ldap_timeouts.py checks for that call at this site. server = self._server() - with _socket_faults_as_ldap_error(): + with ( + _socket_faults_as_ldap_error(), + _ldap3_call("service-account bind", close=True), + ): return ldap3.Connection( server, user=self._s.ad_bind_dn, @@ -1176,7 +1287,8 @@ def _equalizing_bind(self, password: str) -> None: and ``authenticate`` maps it to :class:`LdapError` exactly as it maps the real bind's. Both branches open a socket to the same directory, so such a fault fails them alike; swallowing it here alone would answer "wrong password" for an absent account and "directory error" for - a present one. + a present one. The same holds for a bind response ldap3 could not decode (BACKLOG #3010): + both binds refuse it through :func:`_ldap3_call` as the "user bind". **What #2566 does not settle:** ldap3's OWN socket errors still differ between the branches. This method swallows an ``LDAPSocketOpenError`` while the real bind's handler maps it, so a @@ -1202,7 +1314,10 @@ def _equalizing_bind(self, password: str) -> None: auto_referrals=False, # BACKLOG #2530: see _refuse_referral ) try: - conn.bind() # result deliberately ignored — this branch always fails the login + # Labelled as the real bind is, so an undecodable answer fails both branches with + # the same text (_ldap3_call). + with _ldap3_call("user bind"): + conn.bind() # result deliberately ignored — this branch always fails the login finally: conn.unbind() except ldap3.core.exceptions.LDAPException: @@ -1516,7 +1631,7 @@ def authenticate( try: # BACKLOG #2566: the bind opens the socket, so only the bind is wrapped. The # build above does no I/O, and the release runs on an open socket. - with _socket_faults_as_ldap_error(): + with _socket_faults_as_ldap_error(), _ldap3_call("user bind"): bound = user_conn.bind() if not bound: _refuse_referral(user_conn, "user bind") @@ -1629,12 +1744,13 @@ def read_bind_account(self) -> BindAccountReading: # failure, and the group read still runs. whoami_error: str | None = None try: - with _unknown_code_refused("Who am I"): - who = _authzid_text(svc.extend.standard.who_am_i()) + with _ldap3_call("Who am I"): + answered = svc.extend.standard.who_am_i() + who = _authzid_text(answered) except (ldap3.core.exceptions.LDAPException, LdapError) as exc: # The type name only, as the group read below does: ldap3's text for an # extended-operation error carries the directory's own diagnostic message. An - # LdapError is a result code ldap3 has no name for (BACKLOG #3010), in engine text. + # LdapError is a response ldap3 could not decode (BACKLOG #3010), in engine text. why = str(exc) if isinstance(exc, LdapError) else type(exc).__name__ who, whoami_error = None, f"Who am I failed: {why}" try: diff --git a/tests/test_ldap_decode_boundary.py b/tests/test_ldap_decode_boundary.py new file mode 100644 index 0000000000..0925493fad --- /dev/null +++ b/tests/test_ldap_decode_boundary.py @@ -0,0 +1,461 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +# Copyright (C) 2026 MessageFoundry Foundation, LLC and contributors +"""BACKLOG #3010: a directory response ldap3 cannot decode is an outage, never an unhandled error. + +ldap3 2.9.1's default fast decoder indexes the BER it reads without checking it, so a malformed +response raises ``KeyError``, ``TypeError``, ``IndexError`` or ``AttributeError`` from inside the +ldap3 call. Unmapped, any of those would escape every ``except LdapError``, and a sign-in would fail +as an unhandled error with no audit row. ``_ldap3_call`` is the one boundary every ldap3 call that +reads a response runs inside, and it maps such a fault to ``LdapUndecodableError``. + +It decides by ORIGIN, never by the exception's value. The mapping it replaced matched a ``KeyError`` +whose argument was an integer outside ldap3's result-code table. That let ``KeyError(3)`` through +bare, which is what the decoder raises for a bind response carrying a referral, and it named a stray +context tag [22] "result code 22". Every malformed response here is driven through a REAL ldap3 stack +against a loopback server, and each test states its premise first: the bare ldap3 call raises the +fault. An exception the engine raises propagates, whatever its value. + +**Not exercised:** a real domain controller. Synthetic data only; nothing here touches PHI. +""" + +from __future__ import annotations + +import ast +import inspect +from collections.abc import Callable +from types import SimpleNamespace +from typing import Any + +import ldap3 +import pytest +from ldap3.core.results import RESULT_CODES, RESULT_SUCCESS +from ldap3.utils.asn1 import decode_sequence + +from messagefoundry.auth import ldap as ldap_module +from messagefoundry.auth.ldap import LdapError, LdapUndecodableError +from tests._ast_sites import callee_name, parse_source +from tests.test_ldap_referrals import _BIND_PASSWORD, _BIND_RESPONSE, _SEARCH_DONE, _tlv +from tests.test_ldap_search_result_codes import ( + _IN_CHAIN, + _USER_DN, + _done, + _entry, + _enumerated, + _group_dn, + _Reply, + _scripted_dc, + _service_conn, +) + +#: Stands for text the directory sends: it must never reach an error, a log line or an audit row. +_REFERRED = "ldap://dc9.synthetic-leak-3010.invalid/DC=other" + +#: Long enough for the loopback server to see the client close. Its own read times out at 10s. +_CLOSE_WAIT = 5.0 + + +@pytest.fixture(autouse=True) +def _fresh_warnings(monkeypatch: pytest.MonkeyPatch) -> None: + """Each test sees its own once-per-process warnings.""" + monkeypatch.setattr(ldap_module, "_object_guid_shapes_warned", set()) + + +def _result(tag: int, code: int, *, matched: bytes | None = None, extra: bytes = b"") -> bytes: + """An LDAPResult under ``tag``. ``matched`` replaces the matchedDN element; ``extra`` is appended + after the diagnostic message, where a referral would go.""" + matched_dn = _tlv(0x04, b"") if matched is None else matched + return _tlv(tag, _tlv(0x0A, _enumerated(code)) + matched_dn + _tlv(0x04, b"") + extra) + + +#: Malformed bind responses: the reply, and what ldap3's decoder raises on it. +_BAD_BINDS = [ + pytest.param( + _result(_BIND_RESPONSE, 10, extra=_tlv(0xA3, _tlv(0x04, _REFERRED.encode()))), + KeyError, + id="referral-KeyError-3", + ), + pytest.param( + _result(_BIND_RESPONSE, RESULT_SUCCESS, extra=_tlv(0x96, b"synthetic")), + KeyError, + id="context-tag-22", + ), + pytest.param(_result(_BIND_RESPONSE, 4242), KeyError, id="unknown-code-4242"), + pytest.param( + _result(_BIND_RESPONSE, RESULT_SUCCESS, matched=_tlv(0x02, b"\x05")), + AttributeError, + id="matched-dn-not-a-string", + ), +] + + +def _assert_undecodable(exc: BaseException, operation: str, fault: type[BaseException]) -> None: + """The refusal names the operation and the fault's type, and claims no result code.""" + assert isinstance(exc, LdapUndecodableError), repr(exc) + text = str(exc) + assert f"{operation} with a response ldap3 could not decode ({fault.__name__})" in text + assert "result code" not in text, "no result code was read, so none may be claimed" + assert "synthetic" not in text and "4242" not in text, "a value the directory sent leaked" + assert isinstance(exc.__cause__, fault) + + +def _bare_bind(port: int) -> None: + ldap3.Connection( + ldap3.Server(f"ldap://127.0.0.1:{port}", get_info=ldap3.NONE), + user="CN=svc,DC=example,DC=com", + password=_BIND_PASSWORD, + auto_bind=True, + receive_timeout=5, + ) + + +def _reading_connection(exc: BaseException) -> Any: + """The ldap3 connection whose response read raised ``exc``, found the way the boundary finds + it. A premise check closes it, so the loopback server's thread does not wait out its timeout.""" + reader = ldap_module._response_reader(exc) + assert reader is not None, "the premise: ldap3 raised this while reading a response" + return reader.connection + + +# --- the service-account bind, and the socket it leaves open ------------------------------------- + + +@pytest.mark.parametrize(("reply", "fault"), _BAD_BINDS) +def test_a_service_bind_ldap3_cannot_decode_is_an_outage( + reply: bytes, fault: type[BaseException] +) -> None: + """Every probe and sign-in opens with this bind. THE PREMISE first: the bare ldap3 bind raises + ``fault`` from inside the ``Connection`` constructor.""" + with _scripted_dc( + lambda _raw: [_done(RESULT_SUCCESS)], bind_reply=lambda _raw: (reply, b"") + ) as ( + dc, + authenticator, + ): + with pytest.raises(fault) as bare: + _bare_bind(dc.port) + _reading_connection(bare.value).strategy.close() + with pytest.raises(LdapError) as refused: + authenticator.probe_principal("alice") + _assert_undecodable(refused.value, "service-account bind", fault) + assert dc.searches == [] + + +def test_the_service_bind_socket_is_closed_when_its_response_cannot_be_decoded() -> None: + """Review finding 10. The bind runs inside ldap3's constructor, so no caller holds the + connection to unbind; the refusal closes it. The control: the bare ldap3 bind leaves the socket + open while its error is held.""" + reply = _result(_BIND_RESPONSE, 4242) + + def bind(_raw: bytes) -> _Reply: + return reply, b"" + + with _scripted_dc(lambda _raw: [], bind_reply=bind) as (dc, authenticator): + with pytest.raises(KeyError) as bare: + _bare_bind(dc.port) + left = _reading_connection(bare.value) + assert not left.closed, "the control: ldap3 alone leaves the socket open" + left.strategy.close() + assert dc.ended.wait(_CLOSE_WAIT) + dc.ended.clear() + with pytest.raises(LdapUndecodableError) as refused: + authenticator.probe_principal("alice") + assert refused.value.__cause__ is not None + assert _reading_connection(refused.value.__cause__).closed + assert dc.ended.wait(_CLOSE_WAIT), "the refused bind's socket was left open" + + +# --- the user bind, and the decoy bind that must fail the same way -------------------------------- + + +def _alice(raw: bytes) -> list[_Reply]: + """The user search finds alice, enabled, and nobody else.""" + if b"alice" not in raw: + return [_done(RESULT_SUCCESS)] + attributes = _tlv( + 0x30, + b"".join( + _tlv(0x30, _tlv(0x04, name) + _tlv(0x31, _tlv(0x04, value))) + for name, value in ((b"sAMAccountName", b"alice"), (b"userAccountControl", b"512")) + ), + ) + return [(_tlv(0x64, _tlv(0x04, _USER_DN.encode()) + attributes), b""), _done(RESULT_SUCCESS)] + + +@pytest.mark.parametrize(("reply", "fault"), _BAD_BINDS) +def test_a_user_bind_ldap3_cannot_decode_fails_a_present_and_an_absent_account_alike( + reply: bytes, fault: type[BaseException] +) -> None: + """The present account's password bind and the absent account's decoy bind (ASVS 6.3.8) both + refuse with the same text, so the answer does not tell the two apart.""" + + def bind(raw: bytes) -> _Reply | None: + is_user = b"CN=alice" in raw or b"mf-nonexistent-timing-equalizer" in raw + return (reply, b"") if is_user else None + + texts = [] + with _scripted_dc(_alice, bind_reply=bind) as (_dc, authenticator): + for username in ("alice", "nobody"): + with pytest.raises(LdapError) as refused: + authenticator.authenticate(username, "synthetic-user-pw") + _assert_undecodable(refused.value, "user bind", fault) + texts.append(str(refused.value)) + assert texts[0] == texts[1] + + +# --- the nested group search: a malformed paged results control --------------------------------- + +_OID = ldap_module._PAGED_RESULTS_OID.encode() + +#: Malformed paged results controls on a searchResDone, and what ldap3's decoder raises on each. +_BAD_CONTROLS = [ + pytest.param(_tlv(0x30, _tlv(0x04, _OID)), TypeError, id="no-value-TypeError"), + pytest.param( + _tlv(0x30, _tlv(0x04, _OID) + _tlv(0x04, _tlv(0x30, _tlv(0x02, b"\x00")))), + IndexError, + id="no-cookie-IndexError", + ), +] + + +def _bad_page(control: bytes) -> _Reply: + return _result(_SEARCH_DONE, RESULT_SUCCESS), _tlv(0xA0, control) + + +@pytest.mark.parametrize(("control", "fault"), _BAD_CONTROLS) +@pytest.mark.parametrize("later", [False, True], ids=["first-page", "later-page"]) +def test_a_paged_control_ldap3_cannot_decode_is_an_outage_and_is_not_abandoned( + control: bytes, fault: type[BaseException], later: bool +) -> None: + """New with paging: asking for the control is what lets a malformed one reach the decoder. + No abandon follows, since the cookie it would send is the part that could not be read.""" + + def answer(raw: bytes) -> list[_Reply]: + if later and b"synthetic-cookie-1" not in raw: + return [_entry(_group_dn(1)), _done(RESULT_SUCCESS, b"synthetic-cookie-1")] + return [_entry(_group_dn(2)), _bad_page(control)] + + with _scripted_dc(answer) as (dc, authenticator): + if not later: + with _service_conn(authenticator) as conn, pytest.raises(fault): + conn.search(search_base="OU=Groups", search_filter="(cn=*)", paged_size=5) + dc.searches.clear() + with _service_conn(authenticator) as conn, pytest.raises(LdapError) as refused: + authenticator._resolve_groups(conn, _USER_DN, []) + _assert_undecodable(refused.value, "group search", fault) + assert len(dc.searches) == (2 if later else 1), "a search was sent after the refusal" + assert all(_IN_CHAIN in raw for raw in dc.searches) + + +@pytest.mark.parametrize(("control", "fault"), _BAD_CONTROLS) +def test_a_sign_in_probe_whose_group_page_cannot_be_decoded_is_an_outage( + control: bytes, fault: type[BaseException] +) -> None: + """The reconciler's probe, through the real authenticator: an outage, not an unhandled error.""" + + def answer(raw: bytes) -> list[_Reply]: + return [_bad_page(control)] if _IN_CHAIN in raw else _alice(raw) + + with _scripted_dc(answer) as (_dc, authenticator), pytest.raises(LdapError) as refused: + authenticator.probe_principal("alice") + _assert_undecodable(refused.value, "group search", fault) + + +# --- a result code outside ldap3's table --------------------------------------------------------- + + +@pytest.mark.parametrize("code", [99, 4242], ids=["99", "4242"]) +def test_a_search_ending_with_a_code_ldap3_has_no_name_for_is_an_outage(code: int) -> None: + """THE PREMISE first: a bare ldap3 search raises ``KeyError`` from inside ``conn.search``.""" + assert code not in RESULT_CODES + with _scripted_dc(lambda _raw: [_done(code)]) as (_dc, authenticator): + with _service_conn(authenticator) as conn, pytest.raises(KeyError): + conn.search(search_base="OU=Users,DC=example,DC=com", search_filter="(cn=x)") + with pytest.raises(LdapError) as refused: + authenticator.probe_principal("alice") + _assert_undecodable(refused.value, "user search", KeyError) + + +def test_a_who_am_i_ldap3_cannot_decode_is_reported_and_the_group_read_still_runs() -> None: + """A failed Who am I is not a bind failure: ``whoami_error`` says why, in engine text.""" + with _scripted_dc(lambda _raw: [_done(RESULT_SUCCESS)], extended_code=4242) as ( + dc, + authenticator, + ): + reading = authenticator.read_bind_account() + assert reading.bound and reading.authzid is None + assert reading.whoami_error == ( + "Who am I failed: AD answered the Who am I with a response ldap3 could not decode " + "(KeyError); the engine reads it as the directory not answering (BACKLOG #3010)." + ) + assert len(dc.searches) == 1, "the group read did not run after the failed Who am I" + + +# --- an exception from anywhere but ldap3's response reading propagates --------------------------- + + +def _raise(exc: BaseException) -> Callable[..., Any]: + def call(*_a: object, **_kw: object) -> Any: + raise exc + + return call + + +_ENGINE_FAULTS = pytest.mark.parametrize( + "fault", + [ + KeyError(3), + KeyError(22), + KeyError(4242), + KeyError("objectGUID"), + KeyError((0, 5)), + TypeError("synthetic"), + IndexError("synthetic"), + AttributeError("synthetic"), + ValueError("synthetic"), + ], + ids=repr, +) + + +@_ENGINE_FAULTS +def test_an_engine_fault_propagates_whatever_its_value(fault: Exception) -> None: + """The mapping this replaced turned an engine ``KeyError(4242)`` into an outage and let ldap3's + ``KeyError(3)`` through. Mapped, an engine defect reads as an outage, which the session + reconciler logs only at debug level.""" + double = SimpleNamespace(search=_raise(fault), result=None) + with pytest.raises(type(fault)) as raised: + ldap_module._search(double, "user search") + assert raised.value is fault + with pytest.raises(type(fault)) as raised, ldap_module._ldap3_call("user bind", close=True): + _raise(fault)() + assert raised.value is fault + + +def test_a_fault_ldap3_raises_outside_response_reading_propagates() -> None: + """The innermost frame is ldap3's, but nothing was being read from a directory: the same + ``KeyError(3)`` ldap3 raises on a referral in a bind response, from its decoder called + directly. Origin is both conditions, not the package alone.""" + referral_in_a_bind = _tlv(0xA3, _tlv(0x04, b"synthetic")) + with pytest.raises(KeyError) as bare: + decode_sequence(referral_in_a_bind, 0, len(referral_in_a_bind), {7: None}) + assert bare.value.args == (3,) + with pytest.raises(KeyError), ldap_module._ldap3_call("user bind"): + decode_sequence(referral_in_a_bind, 0, len(referral_in_a_bind), {7: None}) + + +def test_an_ldap3_error_passes_through_for_its_own_handler() -> None: + fault = ldap3.core.exceptions.LDAPSocketReceiveError("synthetic") + with ( + pytest.raises(ldap3.core.exceptions.LDAPSocketReceiveError) as raised, + ldap_module._ldap3_call("user search"), + ): + raise fault + assert raised.value is fault + + +def test_the_ldap3_methods_the_origin_check_names_still_exist() -> None: + """A rename in ldap3 would send every decode fault back to an unhandled error at sign-in; this + names the cause in one place, where the loopback tests would each fail on it.""" + from ldap3.strategy.base import BaseStrategy + from ldap3.strategy.sync import SyncStrategy + + assert {"get_response", "_get_response"} == set(ldap_module._RESPONSE_READERS) + assert "get_response" in vars(BaseStrategy) + assert "_get_response" in vars(SyncStrategy) + + +def test_the_refusal_warns_once_per_operation_and_fault(caplog: pytest.LogCaptureFixture) -> None: + with _scripted_dc(lambda _raw: [_done(4242)]) as (_dc, authenticator): + for _ in range(3): + with pytest.raises(LdapUndecodableError): + authenticator.probe_principal("alice") + warned = [r for r in caplog.records if "could not decode" in r.getMessage()] + assert len(warned) == 1 and warned[0].levelname == "WARNING" + + +# --- every ldap3 call that reads a response runs inside the boundary ------------------------------ + +#: The ldap3 methods that send a request and read its response, as this module calls them. +_READING_CALLS = frozenset({"search", "bind", "who_am_i"}) + + +def _context_names(node: ast.With) -> list[str | None]: + """What each item of a ``with`` calls, by bare name, or ``None`` for an item that is no call.""" + return [ + callee_name(item.context_expr, bare_only=True) + if isinstance(item.context_expr, ast.Call) + else None + for item in node.items + ] + + +def _outside_the_boundary(source: str) -> tuple[list[str], int]: + """Each ldap3 call in ``source`` that reads a response and is not lexically inside a ``with`` + naming ``_ldap3_call``, and how many such calls there are in all. A ``Connection`` built with + ``auto_bind`` counts, bare or as ``ldap3.Connection``: its constructor binds.""" + tree = parse_source(source) + parents: dict[ast.AST, ast.AST] = {} + for node in ast.walk(tree): + for child in ast.iter_child_nodes(node): + parents[child] = node + + def guarded(node: ast.AST) -> bool: + while node in parents: + node = parents[node] + if isinstance(node, ast.With) and "_ldap3_call" in _context_names(node): + return True + return False + + outside, total = [], 0 + for node in ast.walk(tree): + if not isinstance(node, ast.Call): + continue + name = callee_name(node) + reads = (name in _READING_CALLS and isinstance(node.func, ast.Attribute)) or ( + name == "Connection" and any(k.arg == "auto_bind" for k in node.keywords) + ) + if reads: + total += 1 + if not guarded(node): + outside.append(f"line {node.lineno}: {ast.unparse(node.func)}") + return outside, total + + +def test_every_ldap3_call_that_reads_a_response_is_inside_the_boundary() -> None: + """Review finding 12. A new search, bind or extended operation added outside ``_ldap3_call`` + would bring back the unhandled error at sign-in for whatever it decodes.""" + outside, total = _outside_the_boundary(inspect.getsource(ldap_module)) + assert outside == [] + # The service bind, the user bind, the decoy bind, _search, and Who am I. + assert total >= 5, f"the census found {total} calls; the instrument has lost them" + + +def test_a_bind_maps_a_decode_fault_before_a_socket_fault() -> None: + """``TypeError`` and ``ValueError`` are both a socket fault and a decode fault. Where a ``with`` + names both wraps, ``_ldap3_call`` must be the inner one, so a decode fault is refused as one, + and the service socket closed, before ``_socket_faults_as_ldap_error`` can claim it.""" + tree = parse_source(inspect.getsource(ldap_module)) + paired = [ + names + for node in ast.walk(tree) + if isinstance(node, ast.With) + and {"_ldap3_call", "_socket_faults_as_ldap_error"} <= set(names := _context_names(node)) + ] + assert len(paired) >= 2, "the service bind and the user bind" + for names in paired: + assert names.index("_socket_faults_as_ldap_error") < names.index("_ldap3_call"), names + + +def test_the_static_check_catches_a_call_outside_the_boundary() -> None: + """The control: a planted unguarded search, and a guarded one, in the same text.""" + planted = ( + "def a(conn):\n" + " with _ldap3_call('x'):\n" + " conn.search(search_base='b')\n" + "def b(conn):\n" + " with contextlib.suppress(KeyError):\n" + " conn.extend.standard.who_am_i()\n" + ) + outside, total = _outside_the_boundary(planted) + assert total == 2 + assert outside == ["line 6: conn.extend.standard.who_am_i"] diff --git a/tests/test_ldap_search_result_codes.py b/tests/test_ldap_search_result_codes.py index 0bb9d3f182..5fe3e31817 100644 --- a/tests/test_ldap_search_result_codes.py +++ b/tests/test_ldap_search_result_codes.py @@ -10,17 +10,17 @@ search would read the same answer as fewer groups. ``_search`` now refuses every code but success with :class:`LdapUnansweredError`, which every caller already reads as an outage. -Two follow-ups ride here. The nested group search is paged, so an account in more groups than a -directory's unpaged cap would not end it with sizeLimitExceeded, and a page that does not end with -success refuses the whole search. And a result code ldap3 has no name for, which its default fast -decoder raises ``KeyError`` on from inside ``conn.search``, is mapped to the same refusal. +A follow-up rides here. The nested group search is paged, so an account in more groups than a +directory's unpaged cap would not end it with sizeLimitExceeded, a page that does not end with +success refuses the whole search, and a refused search is abandoned. A response ldap3 cannot decode, +such as a result code outside its table, is ``tests/test_ldap_decode_boundary.py``'s subject, and +the loopback server it uses lives here. Driven at four depths: ``_search`` itself over a bare double; the REAL ``LdapAuthenticator`` over the ldap3 doubles of ``tests/test_ad_user_account_control.py`` and ``tests/test_ldap_timeouts.py``; the REAL ``AuthService`` reconciler and Kerberos sign-in over those; and a REAL ldap3 stack against -a loopback server, for the paging control and the unknown code, which no double can reproduce. Each -refusal has a control on the same doubles: success with no entries still reads absent, and still -revokes. +a loopback server, for the paging control, which no double can reproduce. Each refusal has a +control on the same doubles: success with no entries still reads absent, and still revokes. **Not exercised:** a real domain controller. The doubles set the result dict the way ldap3 2.9.1's synchronous strategy leaves it. Synthetic data only; nothing here touches PHI. @@ -43,7 +43,6 @@ from ldap3.core.results import ( RESULT_ADMIN_LIMIT_EXCEEDED, RESULT_BUSY, - RESULT_CODES, RESULT_INSUFFICIENT_ACCESS_RIGHTS, RESULT_INVALID_DN_SYNTAX, RESULT_NO_SUCH_OBJECT, @@ -441,16 +440,29 @@ def _replay(conn: Any, page: dict[str, Any], wrap: Callable[[Any], object] = lam return bool(result["result"] == RESULT_SUCCESS) +_ABANDONED = _page(RESULT_SUCCESS, [], b"") +"""What a directory answers the RFC 2696 abandon with: success, no entries, an empty cookie.""" + + class _PagedConn: - """A bare double that answers its n-th search with ``pages[n]``, recording each search.""" + """A bare double that answers its n-th search with ``pages[n]``, recording each search. A search + with a page size of zero is the abandon: it is recorded in ``abandons`` instead.""" - def __init__(self, pages: list[dict[str, Any]]) -> None: - self._pages = pages + def __init__( + self, pages: list[dict[str, Any]], abandon_raises: Exception | None = None + ) -> None: + self._pages, self._abandon_raises = pages, abandon_raises self.calls: list[dict[str, Any]] = [] + self.abandons: list[dict[str, Any]] = [] self.result: dict[str, Any] | None = None self.entries: list[object] = [] def search(self, **kwargs: Any) -> bool: + if kwargs.get("paged_size") == 0: + self.abandons.append(kwargs) + if self._abandon_raises is not None: + raise self._abandon_raises + return _replay(self, _ABANDONED) self.calls.append(kwargs) return _replay(self, self._pages[min(len(self.calls), len(self._pages)) - 1]) @@ -472,6 +484,7 @@ def test_a_paged_search_reads_every_page_and_sends_each_cookie_back() -> None: ] assert {call["paged_size"] for call in conn.calls} == {ldap_module._PAGE_SIZE} assert all(call["search_base"] == _GROUPS for call in conn.calls) + assert conn.abandons == [], "a search that ended is not abandoned" def test_a_directory_that_ignores_the_page_request_is_read_once() -> None: @@ -481,6 +494,18 @@ def test_a_directory_that_ignores_the_page_request_is_read_once() -> None: conn = _PagedConn([_page(RESULT_SUCCESS, [entry], None)]) assert ldap_module._search_paged(conn, "group search") == [entry] assert len(conn.calls) == 1 + assert conn.abandons == [] + + +def _abandoned_with(conn: _PagedConn) -> list[object]: + """The cookie each abandon sent, after checking it repeats the paged search's own request.""" + asked = {k: v for k, v in conn.calls[0].items() if k not in ("paged_size", "paged_cookie")} + for abandon in conn.abandons: + assert {k: v for k, v in abandon.items() if k != "paged_cookie"} == { + **asked, + "paged_size": 0, + }, "RFC 2696: the abandon is the same search with a page size of zero" + return [abandon["paged_cookie"] for abandon in conn.abandons] @pytest.mark.parametrize( @@ -488,29 +513,76 @@ def test_a_directory_that_ignores_the_page_request_is_read_once() -> None: [(RESULT_BUSY, False), (RESULT_SIZE_LIMIT_EXCEEDED, True), (4242, False)], ids=["51-busy", "4-sizeLimitExceeded-partial", "4242-unknown"], ) +@pytest.mark.parametrize( + ("refused_cookie", "abandoned_with"), + [(b"synthetic-cookie-2", b"synthetic-cookie-2"), (None, b"synthetic-cookie-1")], + ids=["refused-page-has-a-cookie", "refused-page-has-none"], +) def test_a_later_page_that_did_not_end_with_success_refuses_the_whole_search( - code: int, partial: bool + code: int, partial: bool, refused_cookie: bytes | None, abandoned_with: bytes ) -> None: - """No silent truncation: the first page's entries are not returned as the whole answer.""" + """No silent truncation: the first page's entries are not returned as the whole answer. The + search is abandoned with the newest cookie the directory sent (review finding 10).""" conn = _PagedConn( [ _page(RESULT_SUCCESS, [object()], b"synthetic-cookie-1"), - _page(code, [object()] if partial else [], b"synthetic-cookie-2"), + _page(code, [object()] if partial else [], refused_cookie), ] ) with pytest.raises(LdapUnansweredError, match="group search") as refused: - ldap_module._search_paged(conn, "group search") + ldap_module._search_paged(conn, "group search", search_base=_GROUPS) assert refused.value.code == code assert len(conn.calls) == 2 + assert _abandoned_with(conn) == [abandoned_with] + + +def test_a_refused_first_page_is_not_abandoned() -> None: + """The directory sent no cookie, so it holds no paging state to drop.""" + conn = _PagedConn([_page(RESULT_BUSY, [], None)]) + with pytest.raises(LdapUnansweredError, match=r"result code 51 \(busy\)"): + ldap_module._search_paged(conn, "group search") + assert conn.abandons == [] + + +def test_a_later_page_answered_with_a_referral_is_refused_and_abandoned() -> None: + referred = {**_page(10, [], None), "referrals": ["ldaps://dc9.other.invalid/DC=other"]} + conn = _PagedConn([_page(RESULT_SUCCESS, [object()], b"synthetic-cookie-1"), referred]) + with pytest.raises(LdapReferralError): + ldap_module._search_paged(conn, "group search") + assert _abandoned_with(conn) == [b"synthetic-cookie-1"] + + +@pytest.mark.parametrize( + "fault", + [ + ldap3.core.exceptions.LDAPSocketReceiveError("synthetic"), + LdapUnansweredError("synthetic", code=RESULT_BUSY, code_name="busy"), + ], + ids=["ldap3-error", "refused"], +) +def test_an_abandon_that_fails_leaves_the_original_refusal(fault: Exception) -> None: + """Best effort: what the abandon answers, or raises, never replaces why the search failed.""" + conn = _PagedConn( + [ + _page(RESULT_SUCCESS, [object()], b"synthetic-cookie-1"), + _page(RESULT_BUSY, [], None), + ], + abandon_raises=fault, + ) + with pytest.raises(LdapUnansweredError, match=r"result code 51 \(busy\)") as refused: + ldap_module._search_paged(conn, "group search") + assert refused.value is not fault + assert len(conn.abandons) == 1 def test_a_directory_that_never_stops_paging_is_refused(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(ldap_module, "_MAX_PAGES", 3) conn = _PagedConn([_page(RESULT_SUCCESS, [object()], b"synthetic-again")]) with pytest.raises(LdapUnansweredError, match="still paging after 3 pages") as refused: - ldap_module._search_paged(conn, "group search") + ldap_module._search_paged(conn, "group search", search_base=_GROUPS) assert len(conn.calls) == 3 assert (refused.value.code, refused.value.code_name) == (None, "too many pages") + assert _abandoned_with(conn) == [b"synthetic-again"] _UNREADABLE_CONTROLS = [ @@ -550,11 +622,17 @@ def test_a_later_page_that_drops_the_paging_control_refuses_the_whole_search( ldap_module._search_paged(conn, "group search") assert (refused.value.code, refused.value.code_name) == (None, "paging control lost") assert len(conn.calls) == 2 + assert _abandoned_with(conn) == [b"synthetic-cookie-1"] -def _paged_groups(monkeypatch: pytest.MonkeyPatch, pages: list[dict[str, Any]]) -> list[object]: +def _paged_groups( + monkeypatch: pytest.MonkeyPatch, + pages: list[dict[str, Any]], + abandons: list[object] | None = None, +) -> list[object]: """Over the ``_install_fakes`` doubles, answer the n-th group search with ``pages[n]``. Returns - the cookies each group search sent.""" + the cookies each group search sent. An abandon (a page size of zero) is answered with success, + and its cookie goes to ``abandons`` instead.""" base = ldap3.Connection sent: list[object] = [] @@ -563,6 +641,10 @@ def search(self, **kwargs: Any) -> bool: answered: bool = super().search(**kwargs) if not _group_search(kwargs): return answered + if kwargs.get("paged_size") == 0: + if abandons is not None: + abandons.append(kwargs.get("paged_cookie")) + return _replay(self, _ABANDONED) sent.append(kwargs.get("paged_cookie")) return _replay(self, pages[len(sent) - 1], lambda dn: SimpleNamespace(entry_dn=dn)) @@ -598,67 +680,21 @@ def test_a_busy_later_page_refuses_the_sign_in_rather_than_dropping_its_groups( monkeypatch: pytest.MonkeyPatch, ) -> None: _install_fakes(monkeypatch) + abandons: list[object] = [] _paged_groups( monkeypatch, [ _page(RESULT_SUCCESS, [_group_dn(1)], b"synthetic-cookie-1"), _page(RESULT_BUSY, [], None), ], + abandons, ) with pytest.raises(LdapUnansweredError, match=r"group search with result code 51 \(busy\)"): LdapAuthenticator(_ad_settings()).authenticate("alice", "synthetic-pw") + assert abandons == [b"synthetic-cookie-1"] -# --- a result code ldap3 has no name for ----------------------------------------------------------- - - -def _raising(*args: object) -> Callable[..., bool]: - def call(*_a: object, **_kw: object) -> bool: - raise KeyError(*args) - - return call - - -def test_a_key_error_naming_an_unknown_code_is_the_same_refusal() -> None: - """ldap3's fast decoder raises ``KeyError(code)`` for a code outside ``RESULT_CODES``.""" - conn = SimpleNamespace(search=_raising(4242), result=None, entries=[]) - with pytest.raises(LdapUnansweredError, match="an unknown result code") as refused: - ldap_module._search(conn, "user search") - assert refused.value.code == 4242 - assert isinstance(refused.value.__cause__, KeyError) - - -def test_a_bind_ending_with_an_unknown_code_is_the_same_refusal() -> None: - """The decoder raises the same ``KeyError`` on a bind response, so the wrap every bind runs in - maps it too.""" - with ( - pytest.raises(LdapUnansweredError, match="bind with result code 4242") as refused, - ldap_module._socket_faults_as_ldap_error(), - ): - _raising(4242)() - assert refused.value.code == 4242 - - -_OTHER_KEY_ERRORS = pytest.mark.parametrize( - "args", - [("objectGUID",), (RESULT_BUSY,), (True,), (), (4242, "x")], - ids=["a-name", "a-known-code", "a-bool", "no-args", "two-args"], -) - - -@_OTHER_KEY_ERRORS -def test_any_other_key_error_is_the_engines_own_and_propagates(args: tuple[object, ...]) -> None: - """Mapped, an engine defect would read as an outage, which the reconciler logs at debug.""" - conn = SimpleNamespace(search=_raising(*args), result=None, entries=[]) - with pytest.raises(KeyError) as raised: - ldap_module._search(conn, "user search") - assert not isinstance(raised.value, LdapError) - with pytest.raises(KeyError) as raised, ldap_module._socket_faults_as_ldap_error(): - _raising(*args)() - assert not isinstance(raised.value, LdapError) - - -# --- through a real ldap3 stack: the paging control and the unknown code -------------------------- +# --- through a real ldap3 stack: the paging control ---------------------------------------------- _IN_CHAIN = b"1.2.840.113556.1.4.1941" # the nested group search's matching rule, on the wire @@ -692,16 +728,21 @@ def _entry(dn: str) -> _Reply: class _ScriptedDc: - """A loopback LDAP server, after ``tests/test_ldap_referrals.py``'s ``_FakeDc``. Binds end with - ``bind_code``, an extended operation such as Who am I with ``extended_code``; each search is - answered by ``answer(raw request)``. ``searches`` holds every - search request's bytes.""" + """A loopback LDAP server, after ``tests/test_ldap_referrals.py``'s ``_FakeDc``. A bind is + answered by ``bind_reply(raw request)``, or with success where that is absent or returns + ``None``; an extended operation such as Who am I ends with ``extended_code``; each search is + answered by ``answer(raw request)``. ``searches`` holds every search request's bytes, and + ``ended`` is set when a client's socket closes.""" def __init__( - self, answer: Callable[[bytes], list[_Reply]], bind_code: int, extended_code: int + self, + answer: Callable[[bytes], list[_Reply]], + extended_code: int, + bind_reply: Callable[[bytes], _Reply | None] | None = None, ) -> None: - self._answer, self._bind_code, self._extended_code = answer, bind_code, extended_code + self._answer, self._extended_code, self._bind_reply = answer, extended_code, bind_reply self.searches: list[bytes] = [] + self.ended = threading.Event() self._listener = socket.create_server(("127.0.0.1", 0)) self.port: int = self._listener.getsockname()[1] threading.Thread(target=self._accept, daemon=True).start() @@ -715,23 +756,27 @@ def _accept(self) -> None: threading.Thread(target=self._session, args=(sock,), daemon=True).start() def _session(self, sock: socket.socket) -> None: - with sock, contextlib.suppress(OSError): - sock.settimeout(10) - while (message := _read_message(sock)) is not None: - raw, body = message - message_id = body[: 2 + body[1]] # the messageID INTEGER, echoed verbatim - op = body[2 + body[1]] - if op == _BIND_REQUEST: - replies = [_done(self._bind_code, tag=_BIND_RESPONSE)] - elif op == _EXTENDED_REQUEST: - replies = [_done(self._extended_code, tag=_EXTENDED_RESPONSE)] - elif op == _SEARCH_REQUEST: - self.searches.append(raw) - replies = self._answer(raw) - else: # unbind, or anything this server does not speak - return - for reply, controls in replies: - sock.sendall(_tlv(0x30, message_id + reply + controls)) + try: + with sock, contextlib.suppress(OSError): + sock.settimeout(10) + while (message := _read_message(sock)) is not None: + raw, body = message + message_id = body[: 2 + body[1]] # the messageID INTEGER, echoed verbatim + op = body[2 + body[1]] + if op == _BIND_REQUEST: + scripted = self._bind_reply(raw) if self._bind_reply else None + replies = [scripted or _done(RESULT_SUCCESS, tag=_BIND_RESPONSE)] + elif op == _EXTENDED_REQUEST: + replies = [_done(self._extended_code, tag=_EXTENDED_RESPONSE)] + elif op == _SEARCH_REQUEST: + self.searches.append(raw) + replies = self._answer(raw) + else: # unbind, or anything this server does not speak + return + for reply, controls in replies: + sock.sendall(_tlv(0x30, message_id + reply + controls)) + finally: + self.ended.set() def close(self) -> None: # shutdown wakes a blocked accept() on Linux, where close() alone does not. @@ -744,12 +789,12 @@ def close(self) -> None: def _scripted_dc( answer: Callable[[bytes], list[_Reply]], *, - bind_code: int = RESULT_SUCCESS, extended_code: int = RESULT_SUCCESS, + bind_reply: Callable[[bytes], _Reply | None] | None = None, ) -> Iterator[tuple[_ScriptedDc, LdapAuthenticator]]: """A loopback server, and an authenticator pointed at it over plain ``ldap://``: the property under test is the decode, not the hop's TLS.""" - dc = _ScriptedDc(answer, bind_code, extended_code) + dc = _ScriptedDc(answer, extended_code, bind_reply) try: settings = _ad_settings( ad_server=f"ldap://127.0.0.1:{dc.port}", ad_allow_insecure_ldap=True @@ -802,33 +847,9 @@ def answer(raw: bytes) -> list[_Reply]: pytest.raises(LdapUnansweredError, match=r"result code 51 \(busy\)"), ): authenticator._resolve_groups(conn, _USER_DN, []) - assert len(dc.searches) == 2 - - -@pytest.mark.parametrize("code", [99, 4242], ids=["99", "4242"]) -def test_real_ldap3_raises_key_error_on_an_unknown_code_and_the_engine_maps_it(code: int) -> None: - """Measured. THE PREMISE first: a bare ldap3 search, at ldap3's default ``fast_decoder=True``, - raises ``KeyError`` from inside ``conn.search``. Then the engine's own search on the same - server refuses it as unanswered, so the probe reads as an outage.""" - assert code not in RESULT_CODES - with _scripted_dc(lambda _raw: [_done(code)]) as (_dc, authenticator): - with _service_conn(authenticator) as conn, pytest.raises(KeyError): - conn.search(search_base="OU=Users,DC=example,DC=com", search_filter="(cn=x)") - with pytest.raises(LdapUnansweredError, match="an unknown result code") as refused: - authenticator.probe_principal("alice") - assert refused.value.code == code - assert f"user search with result code {code}" in str(refused.value) - - -def test_real_ldap3_maps_a_bind_that_ends_with_an_unknown_code() -> None: - """The service-account bind every probe and sign-in opens with: refused, not a ``KeyError``.""" - with ( - _scripted_dc(lambda _raw: [_done(RESULT_SUCCESS)], bind_code=4242) as (dc, authenticator), - pytest.raises(LdapUnansweredError, match="bind with result code 4242") as refused, - ): - authenticator.probe_principal("alice") - assert refused.value.code == 4242 - assert dc.searches == [] + # The third is the abandon, sent through ldap3 with a page size of zero (RFC 2696). + assert len(dc.searches) == 3 + assert b"synthetic-cookie-1" in dc.searches[2] # --- sign-in: a directory error, not an unknown user --------------------------------------------- @@ -894,29 +915,3 @@ def test_the_bind_account_read_names_the_code_and_keeps_the_identity( assert "distinguished name" not in problem and "may not read" not in problem else: assert hint in problem - - -def test_a_who_am_i_ending_with_an_unknown_code_is_reported_and_the_group_read_still_runs() -> None: - """ldap3's decoder raises the same ``KeyError`` on an extended response. A failed Who am I is - not a bind failure, so it is reported in ``whoami_error`` and the group read still runs.""" - conn = _conn(_result(RESULT_SUCCESS), entries=[]) - conn.extend = SimpleNamespace(standard=SimpleNamespace(who_am_i=_raising(4242))) - conn.unbind = lambda: True - authenticator = LdapAuthenticator(_ad_settings()) - authenticator._service_conn = lambda: conn # type: ignore[method-assign] - reading = authenticator.read_bind_account() - assert reading.bound and reading.authzid is None - assert reading.whoami_error is not None - assert "Who am I with result code 4242 (an unknown result code)" in reading.whoami_error - assert reading.problem is not None and "returned no entry" in reading.problem - - -def test_real_ldap3_maps_a_who_am_i_that_ends_with_an_unknown_code() -> None: - with _scripted_dc(lambda _raw: [_done(RESULT_SUCCESS)], extended_code=4242) as ( - dc, - authenticator, - ): - reading = authenticator.read_bind_account() - assert reading.bound and reading.authzid is None - assert reading.whoami_error is not None and "result code 4242" in reading.whoami_error - assert len(dc.searches) == 1, "the group read did not run after the failed Who am I" From 834697ee3cc123f49e7f9ff4438e2b4fdbe6ae31 Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Fri, 16 Oct 2026 19:20:31 -0500 Subject: [PATCH 16/57] Review round one on the D2 auth follow-ups (BACKLOG #3260 #3145 #2950) Confirmed and fixed: - The seam digest DOES record AuthService method signatures, so the required client keyword on revoke_own_session moved it. Regenerated ENGINE_UI_SEAM and the golden (0971e2440e4237c7), set the console's SUPPORTED_ENGINE_SEAMS, and corrected the _ui_seam note and the console changelog, which said the digest did not move. - confirm_mfa also rotates under the account's re-proof lock, and a lost reply there loses the recovery codes: it gets the sign-in timeout too. - The sign-in timeout raises the read phase only (httpx.Timeout(self._timeout, read=30)), so a GUI-thread caller still learns of an unreachable engine within the client's own bound. - The step-up refusal's store-fault catch logs with its traceback, so a defect raising ValueError or RuntimeError can be told from a fault. - The refused enrolment's row write is soft as well, so a store fault there cannot replace the answer or a propagating defect; the cancellation-after-commit limit is stated in the comment. - A sign-in flow refused as flow_purpose_mismatch no longer records a start_client from another ceremony. - The console's plain-text session id is copied into the miss row only when it is lower-case hex; anything else is recorded as malformed. - SECURITY.md says the miss row is soft-written; the IDE's step-up text says to repeat the action, as the engine's now does. - The refuse(...) calls lost their magic trailing commas. Open, out of this brief's scope: _audit_write_errors() still lacks aiosqlite's closed-connection ValueError for the reconciler's soft writes; that is the reconcile alert path, left to the D3 Builder. Proposed PR title: b257 D1+D2 auth follow-ups: TOTP refusal row, directory notices, revoke-miss audit, OIDC step-up follow-ups, sign-in timeout Proposed vault banners: #3113 (D1 carry-over, round-two findings 1, 3, 5, 6): BUILT on engine branch b257-d2. The auth.mfa_enroll_refused row is written in a finally around the revoke, soft-written itself, so no exception the revoke raises skips it, and aiosqlite's closed-connection ValueError is logged as a store refusal. The directory role notice says the open sessions were ended; the stale notifications.py comment and two SECURITY.md wording points are fixed. Round-two findings 2, 4 and 7 stay open. #3260: limb 2 BUILT on engine branch b257-d2, closing the row with limb 1 (engine PR 2190). revoke_own_session writes one auth.session_revoke_missed row per miss on both planes, through write_audit_soft, with the id cut to 12 characters (a console id that is not hex is recorded as malformed) and nothing else, so an unknown id and a foreign id get the same answer and the same row (owner ruling 2026-10-16, wiki note 20261016T211331989Z-vfzmx7). The engine UI seam moved to 0971e2440e4237c7. Register 1af question 4 is answered by this build, and the 1af entry needs a re-draft. #3145: BUILT on engine branch b257-d2. Items 4, 5 and 6(b)(c)(d) built, item 3 built for the opener's argument assembly; items 1, 2, 6(a), the checks.py preamble half of 3, and 6's extra session read declined in one written line each (see the #3145 commit body). #2950: BUILT on engine branch b257-d2. login, reauth, verify_mfa and confirm_mfa wait 30 s for their reply (SIGN_IN_TIMEOUT_SECONDS, the IDE's value), read phase only; every other request keeps 5 s. The per-account queue has no bound, so a lost reply is rarer, not impossible. --- changelog.d/2950-sign-in-timeout.changed.md | 14 +-- docs/SECURITY.md | 8 +- ide/src/stepUp.ts | 2 +- messagefoundry/api/_ui_seam.py | 7 +- messagefoundry/api/security.py | 3 + messagefoundry/apiclient/client.py | 39 +++++--- messagefoundry/auth/service.py | 89 +++++++++---------- messagefoundry_webconsole/__init__.py | 2 +- .../messagefoundry-webconsole/CHANGELOG.md | 5 +- .../test_ui_session_revoke_miss_audit.py | 8 +- tests/golden/webconsole_seam.snapshot | 4 +- tests/test_apiclient_sign_in_timeout.py | 30 ++++--- 12 files changed, 118 insertions(+), 93 deletions(-) diff --git a/changelog.d/2950-sign-in-timeout.changed.md b/changelog.d/2950-sign-in-timeout.changed.md index 9bc2244f17..011693e040 100644 --- a/changelog.d/2950-sign-in-timeout.changed.md +++ b/changelog.d/2950-sign-in-timeout.changed.md @@ -1,6 +1,8 @@ -- **The API client waits longer for a sign-in.** `EngineClient.login`, `reauth` and `verify_mfa` - now wait up to 30 seconds per phase, matching the IDE's sign-in timeout, instead of the client's - 5 second default. Each can wait in the engine's per-account queue, and a sign-in the client gave - up on could still end the session it replaced, leaving the client signed out. The longer wait - makes that rarer, not impossible, because the queue has no bound. Every other request keeps the - client's timeout. (`vault BACKLOG #2950`) +- **The API client waits longer for a sign-in's reply.** `EngineClient.login`, `reauth`, + `verify_mfa` and `confirm_mfa` now wait up to 30 seconds for the engine's reply, matching the + IDE's sign-in timeout, instead of the client's 5 second default. Each can wait in the engine's + per-account queue, and a request the client gave up on could still end or rotate the session it + held, leaving the client signed out (and, for `confirm_mfa`, without its recovery codes). Only the + read phase is raised: an unreachable engine is still reported within the client's own timeout. + The longer wait makes a lost reply rarer, not impossible, because the queue has no bound. Every + other request keeps the client's timeout. (`vault BACKLOG #2950`) diff --git a/docs/SECURITY.md b/docs/SECURITY.md index 3d4c48e79c..db47ce1ea3 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -756,7 +756,10 @@ stored id matters on the console. A self-service revoke of one session that ends `404` of `DELETE /me/sessions/{session_id}` and the console's "Nothing was revoked" notice, writes one `auth.session_revoke_missed` row (vault BACKLOG #3260, limb 2). It names the actor, the caller's address and the first 12 characters of the id, and nothing else: an unknown id and -another account's id get the same answer and the same row. +another account's id get the same answer and the same row. The console takes the id as plain +text, so an id that is not lower-case hex is recorded as `malformed` rather than copied. The +write is soft: if the audit log refuses it, the answer still stands and the loss is logged at +ERROR, so the row is the record of a miss only while the audit log accepts writes. **Delegated identity & admin device posture (#193 sibling; ASVS 13.2.1 / 13.3.2 / 8.4.2 — the delegation boundary).** Three controls whose enforcement is largely the deploying organization's to provide. @@ -3859,7 +3862,8 @@ token claim**. There is no new `auth_provider` value: a federated login resolves also carry `start_client`, the address the start leg staged on the flow, and `client_moved`, whether the callback came from another host by the new-address comparison, or null when either address is unknown (BACKLOG #2160, vault BACKLOG #3145). Neither is ever read from the token. A - refusal before the `state` check names neither. A move is recorded, never refused. Behind a proxy + refusal before the `state` check names neither, nor does a sign-in flow refused as + `flow_purpose_mismatch`, whose start was another ceremony. A move is recorded, never refused. Behind a proxy the engine does not trust, both legs carry the proxy's address, so `client_moved` reads false there. A cancel at the IdP returns to the same page. A callback without the flow's own `state` consumes nothing, and the flow waits for the real diff --git a/ide/src/stepUp.ts b/ide/src/stepUp.ts index c73683797e..f8e63787bb 100644 --- a/ide/src/stepUp.ts +++ b/ide/src/stepUp.ts @@ -30,7 +30,7 @@ export class IdpStepUpRequiredError extends Error { constructor() { super( "this session signed in through the identity provider, so it confirms a sensitive action " + - "there, in a browser: re-authenticate in the web console at /ui/reauth, then try again", + "there, in a browser: repeat the action in the web console, which opens /ui/reauth for it", ); this.name = "IdpStepUpRequiredError"; } diff --git a/messagefoundry/api/_ui_seam.py b/messagefoundry/api/_ui_seam.py index 27901b2e09..77f334a894 100644 --- a/messagefoundry/api/_ui_seam.py +++ b/messagefoundry/api/_ui_seam.py @@ -406,15 +406,16 @@ #: #: Vault BACKLOG #3260, limb 2: ``AuthService.revoke_own_session`` takes a required keyword #: ``client``, the address on its new ``auth.session_revoke_missed`` row and on its -#: ``auth.session_revoked`` row, and ``/ui/account/sessions/{id}/revoke`` passes it. The digest did -#: not move, for the same reason as #3259, and the same TypeError skew applies to that call. +#: ``auth.session_revoked`` row, and ``/ui/account/sessions/{id}/revoke`` passes it. Unlike the +#: ``AdminHandlers`` fields of #3259, an ``AuthService`` method is on the discovered surface with +#: its signature, so the digest moved and a skew in either direction is refused at mount. #: #: The digest below covers the surface DISCOVERED from the console's own imports and uses, which is #: strictly larger than the five hand-maintained tuples it replaced -- those had drifted, and the #: proof is that commit 40a4d5d9 added a REQUIRED ``UploadedFileList.scope`` field the console renders #: unconditionally while touching no seam file at all. Regenerate with #: ``python scripts/webconsole_seam_snapshot.py --write``; never hand-edit it to silence a gate. -ENGINE_UI_SEAM: str = "1b9da0aae75dbd52" +ENGINE_UI_SEAM: str = "0971e2440e4237c7" @dataclass(frozen=True, slots=True) diff --git a/messagefoundry/api/security.py b/messagefoundry/api/security.py index ba46bac395..954937f16e 100644 --- a/messagefoundry/api/security.py +++ b/messagefoundry/api/security.py @@ -1150,10 +1150,13 @@ async def _step_up_refusal( # Vault BACKLOG #3145 item 6: the read only picks the wording, and the gate has already # refused, so a store fault keeps the 403 rather than becoming a 500. It answers with the # password wording; an oidc client's POST /me/reauth then asks again and is told the IdP. + # With its traceback: ValueError and RuntimeError are also what a bug raises, and the log + # line is the only place a fault and a defect can be told apart. log.warning( "a step-up refusal could not read the session's sign-in mechanism (%s); it answers" " with the password wording", type(exc).__name__, + exc_info=True, ) via_idp = False if via_idp: diff --git a/messagefoundry/apiclient/client.py b/messagefoundry/apiclient/client.py index 726d37ef48..c578518240 100644 --- a/messagefoundry/apiclient/client.py +++ b/messagefoundry/apiclient/client.py @@ -135,15 +135,15 @@ # handlers by test_apiclient_approval_hold.py, which reads the code back rather than trusting 202. _HTTP_PENDING_APPROVAL = 202 -#: How long a request that can wait in the engine's per-account queue may go silent, in seconds -#: (vault BACKLOG #2950): the sign-in, and the two elevations that also rotate the token and queue -#: on that account, ``reauth`` behind the re-proof lock and ``verify_mfa`` behind the credential -#: lock. Matches the IDE's ``SIGN_IN_TIMEOUT_MS`` (``ide/src/signInSession.ts``). The queue has no -#: bound (``keyed_lock.hold_keyed_lock`` takes no timeout), so no value covers every wait: this -#: makes a lost reply rarer than the 5 s default did, and does not rule one out. Every other -#: request keeps the client's own timeout. +#: How long the reply to a request that can wait in the engine's per-account queue may take, in +#: seconds (vault BACKLOG #2950): the sign-in, and the elevations that rotate the token, which +#: queue on that account too: ``reauth``, ``verify_mfa`` and ``confirm_mfa`` (each rotation runs +#: under the account's re-proof lock, and ``verify_mfa`` also under its credential lock). Matches +#: the IDE's ``SIGN_IN_TIMEOUT_MS`` (``ide/src/signInSession.ts``). The queue has no bound +#: (``keyed_lock.hold_keyed_lock`` takes no timeout), so no value covers every wait: this makes a +#: lost reply rarer than the 5 s default did, and does not rule one out. It raises the read phase +#: only (:meth:`EngineClient._sign_in_timeout`); every other request keeps the client's timeout. SIGN_IN_TIMEOUT_SECONDS = 30.0 -_SIGN_IN_TIMEOUT = httpx.Timeout(SIGN_IN_TIMEOUT_SECONDS) class ApiError(RuntimeError): @@ -1084,6 +1084,13 @@ def _request( raise ApiError(_error_detail(response), status=response.status_code) return response + def _sign_in_timeout(self) -> httpx.Timeout: + """This client's timeout with only the READ phase raised to :data:`SIGN_IN_TIMEOUT_SECONDS` + (vault BACKLOG #2950). The engine's per-account queue is waited out after the request is + sent, so connect, write and pool keep the client's own bound: a caller on a GUI thread + still learns of an unreachable engine as fast as before.""" + return httpx.Timeout(self._timeout, read=SIGN_IN_TIMEOUT_SECONDS) + def _adopt_rotated(self, response: httpx.Response) -> None: """Adopt the re-keyed session token an elevation route hands back (ASVS 7.2.4). @@ -1142,7 +1149,7 @@ def reauth(self, password: str, *, purpose: str | None = None) -> None: "/me/reauth", json=body, _allow_step_up=False, - timeout=_SIGN_IN_TIMEOUT, + timeout=self._sign_in_timeout(), ) ) @@ -1175,7 +1182,11 @@ def confirm_mfa(self, code: str) -> list[str]: ``MfaConfirmResponse.token`` is a required ``str`` today, so a token-less body fails in ``_decode`` rather than clearing the session. That is a property of a different file, not of the rule, and the rule is the one thing all three sites must share.""" - response = self._request("POST", "/me/mfa/confirm", json={"code": code}) + # Vault BACKLOG #2950: the rotation queues behind the account's re-proof lock, and a lost + # reply here also loses the one-time recovery codes, so it gets the sign-in's timeout. + response = self._request( + "POST", "/me/mfa/confirm", json={"code": code}, timeout=self._sign_in_timeout() + ) self._adopt_rotated(response) return _decode(response, MfaConfirmResponse).recovery_codes @@ -1194,7 +1205,7 @@ def verify_mfa(self, code: str) -> None: "/auth/mfa-verify", json={"code": code}, _allow_mfa=False, - timeout=_SIGN_IN_TIMEOUT, + timeout=self._sign_in_timeout(), ) ) @@ -1678,8 +1689,8 @@ def login( ``login`` raises :class:`ApiError` and every later call on the held token is refused with a 401 until the caller signs in again. A timeout is the likely way in: a sign-in waiting in the engine's per-account queue can outlast the client, while the engine goes - on to finish it. This request therefore gets :data:`SIGN_IN_TIMEOUT_SECONDS` (30 s per - phase) rather than the client's 5 s default (vault BACKLOG #2950). That makes a lost + on to finish it. This request therefore waits :data:`SIGN_IN_TIMEOUT_SECONDS` (30 s) for + its reply rather than the client's 5 s default (vault BACKLOG #2950). That makes a lost reply rarer, not impossible: the queue has no bound, so a long enough queue still outlasts it. Before, the old token stayed live in that case. The new session nobody holds stays valid until it idles out or expires, as it did before. This is the accepted price of ending the @@ -1694,7 +1705,7 @@ def login( if prior and self._token_cell.issued_here: body["supersedes"] = prior result = _decode( - self._request("POST", "/auth/login", json=body, timeout=_SIGN_IN_TIMEOUT), + self._request("POST", "/auth/login", json=body, timeout=self._sign_in_timeout()), LoginResponse, ) self._hold_issued(result.token) diff --git a/messagefoundry/auth/service.py b/messagefoundry/auth/service.py index 597f60571b..7208b9a3e9 100644 --- a/messagefoundry/auth/service.py +++ b/messagefoundry/auth/service.py @@ -19,6 +19,7 @@ import logging import math import os +import re import secrets import time import unicodedata @@ -2031,6 +2032,8 @@ def _idp_error_fields(exc: BaseException) -> dict[str, object]: #: session writes when it ends nothing, because the id names no session or another account's. #: One action for both, and the same detail, so the row cannot tell the two apart either. SESSION_REVOKE_MISSED_ACTION: Final = "auth.session_revoke_missed" +#: What a session id can look like: a token hash, lower-case hex, cut to at most its full length. +_SESSION_ID_SHAPE: Final = re.compile(r"[0-9a-f]{1,64}") #: Vault BACKLOG #3260: the row an admin route writes when it refuses the caller's own account. #: Its detail's ``op`` names the route, one of :data:`SelfTargetOp`. @@ -4848,9 +4851,7 @@ async def abandon_oidc_step_up( return flow actor = await self._step_up_actor(flow) if flow.step_up_session_hash is None: - return await self._step_up_refused( - FLOW_PURPOSE_MISMATCH, actor=actor, client=client, flow=flow - ) + return await self._step_up_refused(FLOW_PURPOSE_MISMATCH, actor=actor, client=client) return await self._step_up_refused( reason, actor=actor, client=client, return_to=flow.return_to, flow=flow ) @@ -4912,8 +4913,9 @@ async def complete_oidc_step_up( The session is re-anchored to the callback's address when the callback has one. The ``auth.reauth`` row a completed step-up writes, including one whose rotation lost the session, also records the start leg's address as ``start_client`` and whether the two are - different hosts as ``client_moved`` (BACKLOG #2160). So does a refusal made after the flow - was redeemed (vault BACKLOG #3145); one made before carries neither. A move is recorded, + different hosts as ``client_moved`` (BACKLOG #2160). So does a refusal made after a + step-up flow was redeemed (vault BACKLOG #3145); :meth:`_step_up_refused` names the rows + that carry neither. A move is recorded, never refused. Behind a proxy the engine does not trust, both legs carry the proxy's address, so ``client_moved`` reads false there. @@ -4929,9 +4931,7 @@ async def complete_oidc_step_up( actor = await self._step_up_actor(flow) token_hash = flow.step_up_session_hash if token_hash is None: - return await self._step_up_refused( - FLOW_PURPOSE_MISMATCH, actor=actor, client=client, flow=flow - ) + return await self._step_up_refused(FLOW_PURPOSE_MISMATCH, actor=actor, client=client) # Every refusal below is made after the flow was redeemed, so each carries the address # fields (vault BACKLOG #3145): bound once, so a new arm cannot leave them out. refuse = partial( @@ -4956,23 +4956,14 @@ async def complete_oidc_step_up( except oidc.TokenRefusedError as exc: # The endpoint answered. As on the sign-in leg, that is not an outage (#1948), and the # status is what tells a junk code from a misconfiguration or a failing IdP. - return await refuse( - "token_refused", - extra={"status": exc.status}, - ) + return await refuse("token_refused", extra={"status": exc.status}) except (OSError, ValueError, http.client.HTTPException) as exc: # The sign-in leg's outage arm, for the same reasons (#1948). Only the exception TYPE, # and a received status, are kept. if not _is_idp_transport_failure(exc): - return await refuse( - IDP_ANSWER_UNUSABLE, - extra=_idp_error_fields(exc), - ) + return await refuse(IDP_ANSWER_UNUSABLE, extra=_idp_error_fields(exc)) self.mark_oidc_unavailable(type(exc).__name__) - return await refuse( - "idp_unavailable", - extra=_idp_error_fields(exc), - ) + return await refuse("idp_unavailable", extra=_idp_error_fields(exc)) floor_exempt_amr = ( self._oidc_callback_floor_exemption(principal_claims.amr) if arrived_too_early else () ) @@ -4990,16 +4981,10 @@ async def complete_oidc_step_up( # stepped up, so the operator is not told "verified" and then signed out. or not session.is_live(now=now, idle_seconds=self.session_idle_seconds) ): - return await refuse( - "session_gone", - lost=True, - ) + return await refuse("session_gone", lost=True) user = await self._load_user(session.user_id) if user is None or user.disabled or user.auth_provider != AuthProvider.AD.value: - return await refuse( - "session_gone", - lost=True, - ) + return await refuse("session_gone", lost=True) if (user.oidc_issuer, user.oidc_subject) != ( principal_claims.issuer, principal_claims.subject, @@ -5033,13 +5018,9 @@ async def complete_oidc_step_up( # row and the operator's text say which one refused. held_auth_time = session.idp_auth_time if held_auth_time is None: - return await refuse( - STEP_UP_IDP_AUTH_TIME_MISSING, - ) + return await refuse(STEP_UP_IDP_AUTH_TIME_MISSING) if principal_claims.auth_time <= held_auth_time: - return await refuse( - STEP_UP_IDP_AUTH_TIME_NOT_LATER, - ) + return await refuse(STEP_UP_IDP_AUTH_TIME_NOT_LATER) # The DIRECTORY still has the account. The password re-bind this leg replaces failed for a # disabled or deleted AD object, and the sign-in leg refuses one as not_in_directory. An IdP # can still sign such a user in (sync lag, or its own credential store), so the engine asks @@ -5049,9 +5030,7 @@ async def complete_oidc_step_up( if not user.directory_object_id: # Never a name-only lookup: a directory may reissue a freed name to someone else. The # sign-in leg refuses such a row for the same reason (BACKLOG #2027). - return await refuse( - DIRECTORY_OBJECT_ID_MISSING, - ) + return await refuse(DIRECTORY_OBJECT_ID_MISSING) try: principal = await asyncio.to_thread( self._ldap.resolve_principal, user.username, object_id=user.directory_object_id @@ -5076,10 +5055,7 @@ async def complete_oidc_step_up( # as before, through the loader (BACKLOG #2592). fresh = await self._load_user(user.id) if fresh is None: - return await refuse( - "session_gone", - lost=True, - ) + return await refuse("session_gone", lost=True) if not set(fresh.role_ids) <= await self._ad_roles_for_groups(principal.groups): return await refuse(DIRECTORY_ROLES_DEMOTED) purpose = flow.step_up_purpose @@ -5171,9 +5147,10 @@ async def _step_up_refused( type (BACKLOG #1948). It never carries IdP text. ``flow`` adds ``start_client`` and ``client_moved``, as the success row carries them (vault - BACKLOG #3145). Only a refusal made after the flow was redeemed passes it: before ``state`` - matched, the callback is not shown to belong to the flow, and some refusals have no flow - at all, so those rows carry neither field.""" + BACKLOG #3145). Only a refusal made after a STEP-UP flow was redeemed passes it: before + ``state`` matched, the callback is not shown to belong to the flow; a sign-in flow refused + as ``flow_purpose_mismatch`` started another ceremony, so its address is not a step-up's; + and some refusals have no flow at all. Those rows carry neither field.""" # `extra` first, so it can never overwrite a closed-set field. detail: dict[str, object] = { **(extra or {}), @@ -8492,7 +8469,7 @@ async def revoke_own_session( or letting a user touch another's session. Audited. A miss writes :data:`SESSION_REVOKE_MISSED_ACTION` (vault BACKLOG #3260, limb 2; owner - ruling 2026-10-16). Its detail is the id's first 12 characters, as the success row's is, + ruling 2026-10-16). Its detail is the id's first 12 characters (or ``malformed``), as the success row's is, and nothing else: an unknown id and another account's id write the same row, so the trail records no ownership a caller could not already learn. The write is soft with no defect re-raised, for the reason ``api.auth_routes._refuse_if_self`` gives: a raise would replace @@ -8500,11 +8477,14 @@ async def revoke_own_session( :meth:`audit_self_target_refused`, because both planes call this from a request.""" session = await self._load_session(session_id) if session is None or session.user_id != identity.user_id: + # The console passes the id as plain text, so only an id-shaped one is copied into the + # trail; anything else is recorded as malformed, never as the caller's own characters. + shown = session_id[:12] if _SESSION_ID_SHAPE.fullmatch(session_id) else "malformed" await write_audit_soft( lambda: self._audit( SESSION_REVOKE_MISSED_ACTION, actor=actor, - detail=_json({"session": session_id[:12]}), + detail=_json({"session": shown}), client=client, ), log=_log, @@ -10192,8 +10172,12 @@ async def confirm_mfa_enrollment( # first, so no exception the revoke raises can skip it. ``session_ended`` records # what the revoke did. The token is not handed back either way, so the caller still # cannot use the session; a session the revoke left live expires on its own timers. - # A fault that also refuses the read and the audit write below still loses the row. + # The row's own write is soft too, so a store fault there is logged at ERROR rather + # than replacing the answer, or a defect already propagating. Known limit: a + # cancellation can land after the store committed the revoke, and the row then says + # ``session_ended: false`` for a session that did end. rotated = hash_token(elevation.token) + refused = (*store_call_errors(), CipherError) session_ended = False try: session_ended = await write_audit_soft( @@ -10201,11 +10185,18 @@ async def confirm_mfa_enrollment( log=_log, message="a refused TOTP enrolment for %s could not end the session it rotated", args=lambda: (scrub_log_argument(identity.username),), - errors=(*store_call_errors(), CipherError), + errors=refused, defects=AUDIT_WRITE_DEFECTS, ) finally: - await self._audit_enroll_refused(identity, session_ended, client=client) + await write_audit_soft( + lambda: self._audit_enroll_refused(identity, session_ended, client=client), + log=_log, + message="a refused TOTP enrolment for %s could not write its refusal row", + args=lambda: (scrub_log_argument(identity.username),), + errors=refused, + defects=AUDIT_WRITE_DEFECTS, + ) return Elevation(session_lost=True) await self._audit("auth.mfa_enrolled", actor=identity.username, client=client) # ADR 0197 Amendment A: enrolment now comes BEFORE the first rotation, so whoever intercepts diff --git a/messagefoundry_webconsole/__init__.py b/messagefoundry_webconsole/__init__.py index 3d9fc76524..bd54da3b42 100644 --- a/messagefoundry_webconsole/__init__.py +++ b/messagefoundry_webconsole/__init__.py @@ -44,7 +44,7 @@ # If cross-seam support is ever genuinely wanted, re-widen this set AND add the CI matrix that # installs the MIN and MAX supported engine builds — the claim and its test land together, or not # at all. -SUPPORTED_ENGINE_SEAMS: frozenset[str] = frozenset({"1b9da0aae75dbd52"}) +SUPPORTED_ENGINE_SEAMS: frozenset[str] = frozenset({"0971e2440e4237c7"}) #: The vendored static assets shipped in THIS wheel (mounted at /ui/static by :func:`mount_ui`). STATIC_DIR = Path(__file__).parent / "static" diff --git a/packaging/messagefoundry-webconsole/CHANGELOG.md b/packaging/messagefoundry-webconsole/CHANGELOG.md index 04cf29289c..2637da32db 100644 --- a/packaging/messagefoundry-webconsole/CHANGELOG.md +++ b/packaging/messagefoundry-webconsole/CHANGELOG.md @@ -95,8 +95,9 @@ this line.** - **A session revoke that ends nothing is audited, with your address.** The revoke button on `/ui/account/sessions` now passes the request's address to the engine, which writes one `auth.session_revoke_missed` row when the id names no session or another account's. The notice - is unchanged. `AuthService.revoke_own_session` takes a required keyword `client`, which the seam - digest does not record, so a skew raises TypeError on that call. (vault BACKLOG #3260) + is unchanged. `AuthService.revoke_own_session` takes a required keyword `client`; the seam + digest records that signature, so it moved, and a console and engine on either side of this + change refuse each other at mount. (vault BACKLOG #3260) - **The audit page shows where each entry came from.** `/ui/audit` has a Client column, which shows the address the engine recorded for the entry, or a dash when no client was in scope. An operator tracing a run of refused sign-ins can now see their source on the page, not only in diff --git a/packaging/messagefoundry-webconsole/tests/test_ui_session_revoke_miss_audit.py b/packaging/messagefoundry-webconsole/tests/test_ui_session_revoke_miss_audit.py index 2bbfad27e1..b7b584f44d 100644 --- a/packaging/messagefoundry-webconsole/tests/test_ui_session_revoke_miss_audit.py +++ b/packaging/messagefoundry-webconsole/tests/test_ui_session_revoke_miss_audit.py @@ -4,7 +4,7 @@ Owner ruling 2026-10-16: one ``auth.session_revoke_missed`` row per miss, with the id cut to its first 12 characters, and the answer unchanged and the same for an unknown id and another account's. -The console takes the id as plain text, so the cut also bounds what a caller can put in the row. +The console takes the id as plain text, so one that is not lower-case hex is recorded as malformed. The JSON plane's twin is ``tests/test_self_target_refusal_audit.py``. """ @@ -63,5 +63,7 @@ async def test_a_missed_console_revoke_writes_one_row_and_the_same_answer( assert len(rows) == 1, "one miss, one row" assert rows[0]["actor"] == "a" assert rows[0]["client"] == "127.0.0.1", "the miss row dropped the caller's address" - # The cut id and nothing else, so the row records no ownership either. - assert json.loads(str(rows[0]["detail"])) == {"session": target[:12]} + # The cut id and nothing else, so the row records no ownership either. Text that cannot be a + # session id is recorded as malformed, never copied. + shown = "malformed" if kind == "long" else target[:12] + assert json.loads(str(rows[0]["detail"])) == {"session": shown} diff --git a/tests/golden/webconsole_seam.snapshot b/tests/golden/webconsole_seam.snapshot index 41041dc563..904360b47c 100644 --- a/tests/golden/webconsole_seam.snapshot +++ b/tests/golden/webconsole_seam.snapshot @@ -7,7 +7,7 @@ # by hand (BACKLOG #1220) - so a newly rendered DTO is covered with nobody editing a list. ## ENGINE_UI_SEAM -1b9da0aae75dbd52 +0971e2440e4237c7 ## dataclass messagefoundry.api._ui_seam.UiDeps engine_seam @@ -195,7 +195,7 @@ oidc_flow_ttl_seconds: property -> 'int' password_change_owes_factor: (self, token: 'str | None') -> 'bool' reauth: (self, identity: 'Identity', password: 'str', *, token: 'str', client: 'str | None' = None, purpose: 'str | None' = None) -> 'Elevation' revoke_other_sessions: (self, identity: 'Identity', current_token_hash: 'str', *, actor: 'str') -> 'int' -revoke_own_session: (self, identity: 'Identity', session_id: 'str', *, actor: 'str') -> 'bool' +revoke_own_session: (self, identity: 'Identity', session_id: 'str', *, actor: 'str', client: 'str | None') -> 'bool' session_steps_up_at_idp: (self, token: 'str | None') -> 'bool' store: property -> 'AdminStore' suggested_notify_email: (profile_email: 'str | None') -> 'str | None' diff --git a/tests/test_apiclient_sign_in_timeout.py b/tests/test_apiclient_sign_in_timeout.py index 599bfeb376..7a8c74b158 100644 --- a/tests/test_apiclient_sign_in_timeout.py +++ b/tests/test_apiclient_sign_in_timeout.py @@ -6,9 +6,10 @@ ``EngineClient.login`` names the token it replaces, and the engine ends that session inside the sign-in (BACKLOG #2281). A sign-in waiting in the engine's per-account queue could outlast the client's 5 s default, and a lost reply then leaves the client holding a token the engine ended. -``login``, ``reauth`` and ``verify_mfa`` now carry ``SIGN_IN_TIMEOUT_SECONDS``, which matches the -IDE's ``SIGN_IN_TIMEOUT_MS``; every other request keeps the client's own timeout. The queue has no -bound, so this makes the lost reply rarer, not impossible. +``login``, ``reauth``, ``verify_mfa`` and ``confirm_mfa`` now wait ``SIGN_IN_TIMEOUT_SECONDS`` +for their reply, which matches the IDE's ``SIGN_IN_TIMEOUT_MS``; their other phases, and every +other request, keep the client's own timeout. The queue has no bound, so this makes the lost reply +rarer, not impossible. The transport is an ``httpx.MockTransport``: what is pinned is the timeout each request carries, read from ``request.extensions["timeout"]``, not what an engine does with it. @@ -42,6 +43,8 @@ def _reply(request: httpx.Request) -> httpx.Response: body: object = {"token": _TOKEN, "user": _USER} elif path in ("/me/reauth", "/auth/mfa-verify"): body = {"detail": "re-verified", "token": _ROTATED} + elif path == "/me/mfa/confirm": + body = {"recovery_codes": ["code-one"], "token": _ROTATED} else: body = _USER return httpx.Response(200, json=body) @@ -69,21 +72,28 @@ def _every_phase(seconds: float) -> dict[str, float | None]: return {"connect": seconds, "read": seconds, "write": seconds, "pool": seconds} -def test_the_sign_in_and_the_queued_elevations_carry_the_longer_timeout() -> None: - """RED when: ``/auth/login``, ``/me/reauth`` or ``/auth/mfa-verify`` runs under the client's - 5 s default (vault BACKLOG #2950), or ``/auth/me`` loses that default.""" +#: The requests that wait in the engine's per-account queue. +_QUEUED = ("/auth/login", "/me/reauth", "/auth/mfa-verify", "/me/mfa/confirm") + + +def test_the_sign_in_and_the_queued_elevations_wait_longer_for_their_reply() -> None: + """RED when: ``/auth/login``, ``/me/reauth``, ``/auth/mfa-verify`` or ``/me/mfa/confirm`` + waits only the client's 5 s default for its reply (vault BACKLOG #2950), when anything but + the read phase is raised, or when ``/auth/me`` loses that default.""" client, seen = _recording_client() try: client.login("jdoe", "a-strong-test-passphrase") client.reauth("a-strong-test-passphrase") client.verify_mfa("123456") + assert client.confirm_mfa("123456") == ["code-one"] client.me() finally: client.close() - longer = _every_phase(SIGN_IN_TIMEOUT_SECONDS) - assert seen["/auth/login"] == longer - assert seen["/me/reauth"] == longer - assert seen["/auth/mfa-verify"] == longer + # Only the read phase waits out the queue; connect, write and pool keep the client's bound, + # so an unreachable engine is still reported as fast as before. + longer = {**_every_phase(5.0), "read": SIGN_IN_TIMEOUT_SECONDS} + for path in _QUEUED: + assert seen[path] == longer, path # The control: any other request keeps the client's default. assert seen["/auth/me"] == _every_phase(5.0) From 2cae56b43f1f7f28f029bec91a69398017a1c7e4 Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Fri, 16 Oct 2026 20:05:03 -0500 Subject: [PATCH 17/57] Review round one: Who am I value decode, socket close on a search, unbind never masks (BACKLOG #3010) The boundary also recognises ldap3's extended-operation decode frames, so a Who am I value that is not text (UnicodeError in populate_result) is an outage and the group read still runs. A search that meets an undecodable response closes its socket, since the stream may be out of step and the service connection is never unbound. _released keeps an unbind error from replacing a bind's refusal, so the decoy and real binds still fail alike. A failed RFC 2696 abandon is logged at debug level by type. The changelog names what is not covered (ldap3's ranged memberOf follow-ups) and the reconciler log-level cost, in shorter sentences. --- changelog.d/3010.security.md | 53 ++++++------ messagefoundry/auth/ldap.py | 114 ++++++++++++++++--------- tests/test_ldap_decode_boundary.py | 60 +++++++++++-- tests/test_ldap_search_result_codes.py | 23 +++-- 4 files changed, 174 insertions(+), 76 deletions(-) diff --git a/changelog.d/3010.security.md b/changelog.d/3010.security.md index cbcae0e211..455d824601 100644 --- a/changelog.d/3010.security.md +++ b/changelog.d/3010.security.md @@ -1,26 +1,31 @@ -- **A directory search that does not end with success is an outage, never "no such account".** - Every LDAP search refuses any result code but success, with or without entries: busy, - unavailable, unwillingToPerform, adminLimitExceeded, a time or size limit, and noSuchObject on - a missing search base. Before, such a search read as no entries, so the session reconciler would - have struck a present account toward revocation, and a busy nested group search read as fewer - groups. The reconciler reads these as unavailable, which never revokes, and a sign-in writes - `auth.login_error` naming the code. `check-privileges` names the code for the bind-account read. - The text of these refusals names the operation, the code and ldap3's name for it, never the - directory's diagnostic message; other directory errors keep the text they had. +- **An LDAP search that does not end with success is an outage, never "no such account".** Each + search the engine sends refuses any result code but success, with or without entries. That + includes busy, unavailable, unwillingToPerform and adminLimitExceeded. It includes a time or size + limit, and noSuchObject on a missing search base. Before, such a search read as no entries. The + session reconciler would then have struck a present account toward revocation. A busy nested + group search read as fewer groups. The reconciler now reads these as unavailable, which never + revokes. A sign-in writes `auth.login_error` naming the code, and `check-privileges` names the + code for the bind-account read. These refusals name the operation and the code, never the + directory's diagnostic message. Other directory errors keep the text they had. + **Not covered:** ldap3 fetches a ranged `memberOf` with follow-up searches of its own, and those + are not refused. **The cost:** a code that recurs on every pass, such as a deleted search base, stops revocation - and the role re-diff for as long as it lasts, with one WARNING per code per process and no - latched alert. - **The nested group search is paged** (the simple paged results control, 500 per page), so an - account in more groups than a directory's unpaged cap (AD's MaxPageSize, 1000 by default) does - not end it with sizeLimitExceeded and fail sign-in. A page that does not end with success refuses - the whole search, and so does a later page that drops the paging control, or a search still - paging after 20 pages; the pages already read are never used as the whole answer. A refused paged - search is abandoned (RFC 2696) once the directory has sent a cookie. Paging was measured against - a loopback server through ldap3, not against a domain controller. + and the role re-diff for as long as it lasts. It warns once per process for each operation and + code, and latches no alert. + **The nested group search is paged**, at 500 entries per page. So an account in more groups than + a directory's unpaged cap no longer fails sign-in with sizeLimitExceeded. AD's cap, MaxPageSize, + is 1000 by default. A page that does not end with success refuses the whole search. So does a + later page that drops the paging control, and a search still paging after 20 pages. The pages + already read are never used as the whole answer. A refused paged search is abandoned (RFC 2696) + once the directory has sent a cookie, at the cost of one more round trip. Paging was measured + against a loopback server through ldap3, not against a domain controller. **A response ldap3 cannot decode is an outage, not an unhandled error.** ldap3's default decoder - raises `KeyError`, `TypeError`, `IndexError` or `AttributeError` on at least a bind response - carrying a referral, a result code outside its table, and a malformed paged results control. A - sign-in that met one would have failed with an unhandled error and no audit row. Each search, - bind and Who am I read now runs inside one boundary that maps such a fault to an outage when ldap3 - raised it while reading the response, and lets any other exception propagate. The service - account's socket is closed when its bind cannot be decoded. (vault `BACKLOG #3010`) + raises `KeyError`, `TypeError`, `IndexError`, `AttributeError` or `UnicodeError` on a malformed + response. Cases include at least a bind response carrying a referral, a result code outside its + table, a malformed paged results control, and a Who am I value that is not text. A sign-in that + met one would have failed with an unhandled error and no audit row. The searches, binds and + Who am I read the engine sends now run inside one boundary. It maps such a fault to an outage when + ldap3 raised it reading the response, and lets any other exception propagate. Such a fault now + reaches the session reconciler as an outage, which it logs at debug level, rather than as an + unexpected error, which it logged at WARNING for each account. After one, the engine closes the + socket of the service-account bind or search that met it. (vault `BACKLOG #3010`) diff --git a/messagefoundry/auth/ldap.py b/messagefoundry/auth/ldap.py index a64b7379a8..da0c02fabf 100644 --- a/messagefoundry/auth/ldap.py +++ b/messagefoundry/auth/ldap.py @@ -868,7 +868,7 @@ def _search(conn: Any, operation: str, **kwargs: Any) -> None: (:func:`_refuse_unanswered`). Every search in this module goes through here, so none can read either as "no entries"; a test pins that. The search runs inside :func:`_ldap3_call`. """ - with _ldap3_call(operation): + with _ldap3_call(operation, close=True): conn.search(**kwargs) _refuse_unanswered(conn, operation) @@ -880,13 +880,20 @@ def _search(conn: Any, operation: str, **kwargs: Any) -> None: #: ``KeyError(code)`` (``ldap_result_to_dict_fast``, ``operation/bind.py``, #: ``operation/extended.py``); a paged results control with no value is ``TypeError`` and one with no #: cookie is ``IndexError`` (``strategy/base.py``, ``decode_control_fast``); a bind response whose -#: matchedDN is not a string is ``AttributeError`` (``decode_response_fast``). At least these; the -#: decoder trusts every length and tag it reads, so others are reachable. +#: matchedDN is not a string is ``AttributeError`` (``decode_response_fast``); a Who am I value that +#: is not UTF-8 is ``UnicodeError``, a ``ValueError`` (``extend/standard/whoAmI.py``, +#: ``populate_result``). At least these; the decoder trusts every length and tag it reads, so others +#: are reachable. _DECODE_FAULTS: tuple[type[Exception], ...] = (LookupError, TypeError, ValueError, AttributeError) -#: The ldap3 strategy methods that read and decode a response. A fault raised under one of them was -#: raised reading what the directory sent, not building what the engine asked for. -_RESPONSE_READERS = frozenset({"get_response", "_get_response"}) +#: Where ldap3 reads and decodes a response: a module prefix, and the methods under it that do. A +#: fault raised under one was raised reading what the directory sent, not building what the engine +#: asked for. The strategy reads the message; an extended operation such as Who am I decodes its +#: value after that read has returned. ``self`` in each holds the ``connection`` it read on. +_RESPONSE_READERS: tuple[tuple[str, frozenset[str]], ...] = ( + ("ldap3.strategy.", frozenset({"get_response", "_get_response"})), + ("ldap3.extend.", frozenset({"decode_response", "populate_result", "set_response"})), +) def _type_name(exc: BaseException) -> str: @@ -898,31 +905,30 @@ def _type_name(exc: BaseException) -> str: return f"{kind.__module__}.{kind.__qualname__}" -def _response_reader(exc: BaseException) -> Any: - """The ldap3 strategy that was reading a response when ``exc`` was raised, or ``None`` when +def _reading_connection(exc: BaseException) -> Any: + """The ldap3 connection whose response was being read when ``exc`` was raised, or ``None`` when ``exc`` was not raised by ldap3 while reading one. Decided by ORIGIN, never by the exception's value: the innermost frame belongs to the ldap3 - package, and an outer frame is one of :data:`_RESPONSE_READERS` in ``ldap3.strategy``. A - ``KeyError(3)`` the decoder raises and a ``KeyError(3)`` engine code raises carry the same - value, and only the first is the directory's. The second condition keeps out what ldap3 raises - while building a request from the engine's own arguments, which is the engine's defect. + package, and an outer frame is one of :data:`_RESPONSE_READERS`. A ``KeyError(3)`` the decoder + raises and a ``KeyError(3)`` engine code raises carry the same value, and only the first is the + directory's. The second condition keeps out what ldap3 raises while building a request from the + engine's own arguments, which is the engine's defect. """ - reader: Any = None + connection: Any = None module = "" tb = exc.__traceback__ while tb is not None: frame = tb.tb_frame module = str(frame.f_globals.get("__name__", "")) - if ( - reader is None - and module.startswith("ldap3.strategy.") - and frame.f_code.co_name in _RESPONSE_READERS + if connection is None and any( + module.startswith(prefix) and frame.f_code.co_name in names + for prefix, names in _RESPONSE_READERS ): - reader = frame.f_locals.get("self") + connection = getattr(frame.f_locals.get("self"), "connection", None) tb = tb.tb_next in_ldap3 = module == "ldap3" or module.startswith("ldap3.") - return reader if in_ldap3 else None + return connection if in_ldap3 else None @contextlib.contextmanager @@ -936,12 +942,14 @@ def _ldap3_call(operation: str, *, close: bool = False) -> Iterator[None]: outage. Unmapped, it would escape every ``except LdapError``, and a sign-in would fail as an unhandled error with no audit row. An exception from anywhere else propagates unchanged, since mapped it would hide an engine defect as an outage, which the session reconciler logs only at - debug level (:func:`_response_reader` says how the two are told apart). An ``LDAPException`` + debug level (:func:`_reading_connection` says how the two are told apart). An ``LDAPException`` passes through untouched, for its own handler. - ``close`` closes the socket the failed read was on. The service account's ``auto_bind`` needs - it: that bind runs inside ldap3's ``Connection`` constructor, so when it raises no caller holds - the connection to unbind, and the socket would stay open for as long as the error is kept. + ``close`` closes the socket the failed read was on, where nothing else will. The service + account's ``auto_bind`` runs inside ldap3's ``Connection`` constructor, so when it raises no + caller holds the connection to unbind. A search's caller abandons the connection on any error, + and the stream may be out of step after a fault partway through a response; the service + connection is never unbound, so its socket would stay open for as long as the error is kept. """ import ldap3 @@ -950,14 +958,18 @@ def _ldap3_call(operation: str, *, close: bool = False) -> Iterator[None]: except ldap3.core.exceptions.LDAPException: raise except _DECODE_FAULTS as exc: - reader = _response_reader(exc) - if reader is None: + connection = _reading_connection(exc) + if connection is None: raise if close: # BaseStrategy.close shuts the socket without sending anything on a stream that may # be out of step. Best effort: the refusal below is the answer either way. - with contextlib.suppress(ldap3.core.exceptions.LDAPException, OSError): - reader.close() + try: + connection.strategy.close() + except (ldap3.core.exceptions.LDAPException, OSError) as failed: + logger.debug( + "AD socket close after an undecodable response: %s", _type_name(failed) + ) raise _undecodable(operation, exc) from exc @@ -1018,7 +1030,8 @@ def _search_paged(conn: Any, operation: str, **kwargs: Any) -> list[Any]: **A refused search is abandoned** (:func:`_abandon_paging`) once the directory has sent a cookie, so it need not hold the paging state until its own timeout. Not after a response ldap3 - could not decode: the cookie in it could not be read either. + could not decode: the stream may be out of step after one, so nothing more is sent on it, and + :func:`_search` closes the socket instead, which drops the paging state with the connection. """ entries: list[Any] = [] cookie: bytes | None = None # what the next page is asked with @@ -1053,15 +1066,18 @@ def _abandon_paging( A finding of BACKLOG #3010's review. ``cookie`` is the newest non-empty one the directory sent; with none, it never held state for this search and nothing is sent. Best effort: the search is - refused whatever this answers, so a refusal of the abandon is warned once by :func:`_search` and - not raised. It costs one more round trip, which can take up to ``[auth].ad_receive_timeout``. + refused whatever this answers, so a failed abandon is not raised: a refusal of it is warned once + by :func:`_search`, and an ldap3 error from it is logged at debug level. It costs one more round trip, which can take up to ``[auth].ad_receive_timeout``. """ import ldap3 if not cookie: return - with contextlib.suppress(LdapError, ldap3.core.exceptions.LDAPException): + try: _search(conn, f"{operation} abandon", paged_size=0, paged_cookie=cookie, **kwargs) + except (LdapError, ldap3.core.exceptions.LDAPException) as failed: + # The type only: an ldap3 error's text can carry the directory's diagnostic message. + logger.debug("AD abandon of a refused %s failed: %s", operation, _type_name(failed)) def _paging_refused(operation: str, case: str, what: str) -> LdapUnansweredError: @@ -1129,6 +1145,27 @@ def _socket_faults_as_ldap_error() -> Iterator[None]: raise LdapError(f"AD directory call failed: {_type_name(exc)}") from exc +@contextlib.contextmanager +def _released(conn: Any) -> Iterator[None]: + """Unbind ``conn`` when the block ends, however it ends. + + An ldap3 error from the unbind never replaces an exception the block raised (BACKLOG #3010). + The decoy bind swallows an ``LDAPException`` and the real bind maps it to :class:`LdapError`, so + an unbind failing after an undecodable bind would otherwise answer an absent account and a + present one differently. After a block that raised nothing, the unbind's error propagates as + before. + """ + import ldap3 + + try: + yield + except BaseException: + with contextlib.suppress(ldap3.core.exceptions.LDAPException): + conn.unbind() + raise + conn.unbind() + + class LdapAuthenticator: """Binds against Active Directory over LDAPS and resolves a user's (nested) group membership.""" @@ -1313,13 +1350,10 @@ def _equalizing_bind(self, password: str) -> None: receive_timeout=_ldap3_receive_timeout(self._s.ad_receive_timeout), auto_referrals=False, # BACKLOG #2530: see _refuse_referral ) - try: - # Labelled as the real bind is, so an undecodable answer fails both branches with - # the same text (_ldap3_call). - with _ldap3_call("user bind"): - conn.bind() # result deliberately ignored — this branch always fails the login - finally: - conn.unbind() + # Labelled as the real bind is, and released the same way, so an undecodable answer + # fails both branches with the same text (_ldap3_call, _released). + with _released(conn), _ldap3_call("user bind"): + conn.bind() # result deliberately ignored — this branch always fails the login except ldap3.core.exceptions.LDAPException: return @@ -1628,7 +1662,7 @@ def authenticate( # Released on BOTH paths. A rejected password is the common adversarial case, so # returning early without unbinding would leave the connection to GC under exactly # the load that matters (ASVS 13.1.3 — resource release). - try: + with _released(user_conn): # BACKLOG #2566: the bind opens the socket, so only the bind is wrapped. The # build above does no I/O, and the release runs on an open socket. with _socket_faults_as_ldap_error(), _ldap3_call("user bind"): @@ -1637,8 +1671,6 @@ def authenticate( _refuse_referral(user_conn, "user bind") # The entry was found enabled, so the refusal judged this account. return DirectoryBind(DirectoryAnswer.FOUND) - finally: - user_conn.unbind() groups = self._resolve_groups(svc, user_dn, info["memberOf"]) except ldap3.core.exceptions.LDAPException as exc: raise LdapError(str(exc)) from exc diff --git a/tests/test_ldap_decode_boundary.py b/tests/test_ldap_decode_boundary.py index 0925493fad..a3f103fb73 100644 --- a/tests/test_ldap_decode_boundary.py +++ b/tests/test_ldap_decode_boundary.py @@ -36,6 +36,7 @@ from tests._ast_sites import callee_name, parse_source from tests.test_ldap_referrals import _BIND_PASSWORD, _BIND_RESPONSE, _SEARCH_DONE, _tlv from tests.test_ldap_search_result_codes import ( + _EXTENDED_RESPONSE, _IN_CHAIN, _USER_DN, _done, @@ -111,9 +112,9 @@ def _bare_bind(port: int) -> None: def _reading_connection(exc: BaseException) -> Any: """The ldap3 connection whose response read raised ``exc``, found the way the boundary finds it. A premise check closes it, so the loopback server's thread does not wait out its timeout.""" - reader = ldap_module._response_reader(exc) - assert reader is not None, "the premise: ldap3 raised this while reading a response" - return reader.connection + connection = ldap_module._reading_connection(exc) + assert connection is not None, "the premise: ldap3 raised this while reading a response" + return connection # --- the service-account bind, and the socket it leaves open ------------------------------------- @@ -239,8 +240,10 @@ def answer(raw: bytes) -> list[_Reply]: with _service_conn(authenticator) as conn, pytest.raises(fault): conn.search(search_base="OU=Groups", search_filter="(cn=*)", paged_size=5) dc.searches.clear() - with _service_conn(authenticator) as conn, pytest.raises(LdapError) as refused: - authenticator._resolve_groups(conn, _USER_DN, []) + with _service_conn(authenticator) as conn: + with pytest.raises(LdapError) as refused: + authenticator._resolve_groups(conn, _USER_DN, []) + assert conn.closed, "the stream may be out of step, so the socket is closed" _assert_undecodable(refused.value, "group search", fault) assert len(dc.searches) == (2 if later else 1), "a search was sent after the refusal" assert all(_IN_CHAIN in raw for raw in dc.searches) @@ -275,6 +278,31 @@ def test_a_search_ending_with_a_code_ldap3_has_no_name_for_is_an_outage(code: in _assert_undecodable(refused.value, "user search", KeyError) +def test_a_who_am_i_value_that_is_not_utf8_is_reported_and_the_group_read_still_runs() -> None: + """Round-one review finding. ldap3 decodes the Who am I value in ``populate_result``, after + the response read has returned, so the boundary looks for that frame too. THE PREMISE first: + the bare call raises ``UnicodeError``, whose text quotes the directory's bytes. ldap3 decodes + that value with the client encoding (``stdin``'s) and then UTF-8, so the bytes are ones neither + cp1252 nor UTF-8 decodes.""" + value = b"u:EXAMPLE\\synthetic-\x81\x8d" + reply = ( + _result(_EXTENDED_RESPONSE, RESULT_SUCCESS, extra=_tlv(0x8B, value)), + b"", + ) + with _scripted_dc(lambda _raw: [_done(RESULT_SUCCESS)], extended_reply=reply) as ( + dc, + authenticator, + ): + with _service_conn(authenticator) as conn, pytest.raises(UnicodeError): + conn.extend.standard.who_am_i() + reading = authenticator.read_bind_account() + assert reading.bound and reading.authzid is None + assert reading.whoami_error is not None + assert "Who am I with a response ldap3 could not decode (UnicodeError)" in reading.whoami_error + assert "synthetic" not in reading.whoami_error + assert len(dc.searches) == 1, "the group read did not run after the failed Who am I" + + def test_a_who_am_i_ldap3_cannot_decode_is_reported_and_the_group_read_still_runs() -> None: """A failed Who am I is not a bind failure: ``whoami_error`` says why, in engine text.""" with _scripted_dc(lambda _raw: [_done(RESULT_SUCCESS)], extended_code=4242) as ( @@ -356,12 +384,32 @@ def test_an_ldap3_error_passes_through_for_its_own_handler() -> None: def test_the_ldap3_methods_the_origin_check_names_still_exist() -> None: """A rename in ldap3 would send every decode fault back to an unhandled error at sign-in; this names the cause in one place, where the loopback tests would each fail on it.""" + from ldap3.extend.operation import ExtendedOperation + from ldap3.extend.standard.whoAmI import WhoAmI from ldap3.strategy.base import BaseStrategy from ldap3.strategy.sync import SyncStrategy - assert {"get_response", "_get_response"} == set(ldap_module._RESPONSE_READERS) + readers = dict(ldap_module._RESPONSE_READERS) + assert readers["ldap3.strategy."] == {"get_response", "_get_response"} assert "get_response" in vars(BaseStrategy) assert "_get_response" in vars(SyncStrategy) + assert readers["ldap3.extend."] == {"decode_response", "populate_result", "set_response"} + assert {"decode_response", "set_response"} <= set(vars(ExtendedOperation)) + assert "populate_result" in vars(WhoAmI) + + +def test_an_unbind_error_never_replaces_the_refusal() -> None: + """Round-one review finding. The decoy bind swallows an ``LDAPException`` and the real bind + maps it, so an unbind failing after an undecodable bind would tell an absent account from a + present one. After a clean block, the unbind's error still propagates.""" + unbind_fault = ldap3.core.exceptions.LDAPSocketSendError("synthetic") + conn = SimpleNamespace(unbind=_raise(unbind_fault)) + refusal = LdapUndecodableError("synthetic refusal") + with pytest.raises(LdapUndecodableError) as raised, ldap_module._released(conn): + raise refusal + assert raised.value is refusal + with pytest.raises(ldap3.core.exceptions.LDAPSocketSendError), ldap_module._released(conn): + pass def test_the_refusal_warns_once_per_operation_and_fault(caplog: pytest.LogCaptureFixture) -> None: diff --git a/tests/test_ldap_search_result_codes.py b/tests/test_ldap_search_result_codes.py index 5fe3e31817..5908d41d2c 100644 --- a/tests/test_ldap_search_result_codes.py +++ b/tests/test_ldap_search_result_codes.py @@ -560,8 +560,11 @@ def test_a_later_page_answered_with_a_referral_is_refused_and_abandoned() -> Non ], ids=["ldap3-error", "refused"], ) -def test_an_abandon_that_fails_leaves_the_original_refusal(fault: Exception) -> None: +def test_an_abandon_that_fails_leaves_the_original_refusal( + fault: Exception, caplog: pytest.LogCaptureFixture +) -> None: """Best effort: what the abandon answers, or raises, never replaces why the search failed.""" + caplog.set_level(logging.DEBUG, logger=ldap_module.__name__) conn = _PagedConn( [ _page(RESULT_SUCCESS, [object()], b"synthetic-cookie-1"), @@ -573,6 +576,10 @@ def test_an_abandon_that_fails_leaves_the_original_refusal(fault: Exception) -> ldap_module._search_paged(conn, "group search") assert refused.value is not fault assert len(conn.abandons) == 1 + if not isinstance(fault, LdapError): # a refusal is warned by _search; an ldap3 error is not + logged = [r for r in caplog.records if "abandon of a refused group search" in r.message] + assert [r.levelname for r in logged] == ["DEBUG"] + assert "synthetic" not in logged[0].getMessage() def test_a_directory_that_never_stops_paging_is_refused(monkeypatch: pytest.MonkeyPatch) -> None: @@ -730,8 +737,8 @@ def _entry(dn: str) -> _Reply: class _ScriptedDc: """A loopback LDAP server, after ``tests/test_ldap_referrals.py``'s ``_FakeDc``. A bind is answered by ``bind_reply(raw request)``, or with success where that is absent or returns - ``None``; an extended operation such as Who am I ends with ``extended_code``; each search is - answered by ``answer(raw request)``. ``searches`` holds every search request's bytes, and + ``None``; an extended operation such as Who am I is answered with ``extended_reply``, or ends + with ``extended_code``; each search is answered by ``answer(raw request)``. ``searches`` holds every search request's bytes, and ``ended`` is set when a client's socket closes.""" def __init__( @@ -739,8 +746,10 @@ def __init__( answer: Callable[[bytes], list[_Reply]], extended_code: int, bind_reply: Callable[[bytes], _Reply | None] | None = None, + extended_reply: _Reply | None = None, ) -> None: self._answer, self._extended_code, self._bind_reply = answer, extended_code, bind_reply + self._extended_reply = extended_reply self.searches: list[bytes] = [] self.ended = threading.Event() self._listener = socket.create_server(("127.0.0.1", 0)) @@ -767,7 +776,10 @@ def _session(self, sock: socket.socket) -> None: scripted = self._bind_reply(raw) if self._bind_reply else None replies = [scripted or _done(RESULT_SUCCESS, tag=_BIND_RESPONSE)] elif op == _EXTENDED_REQUEST: - replies = [_done(self._extended_code, tag=_EXTENDED_RESPONSE)] + replies = [ + self._extended_reply + or _done(self._extended_code, tag=_EXTENDED_RESPONSE) + ] elif op == _SEARCH_REQUEST: self.searches.append(raw) replies = self._answer(raw) @@ -791,10 +803,11 @@ def _scripted_dc( *, extended_code: int = RESULT_SUCCESS, bind_reply: Callable[[bytes], _Reply | None] | None = None, + extended_reply: _Reply | None = None, ) -> Iterator[tuple[_ScriptedDc, LdapAuthenticator]]: """A loopback server, and an authenticator pointed at it over plain ``ldap://``: the property under test is the decode, not the hop's TLS.""" - dc = _ScriptedDc(answer, extended_code, bind_reply) + dc = _ScriptedDc(answer, extended_code, bind_reply, extended_reply) try: settings = _ad_settings( ad_server=f"ldap://127.0.0.1:{dc.port}", ad_allow_insecure_ldap=True From e6511df5cbe8f5980a69d8e00fba0889bfa0d153 Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Fri, 16 Oct 2026 20:07:43 -0500 Subject: [PATCH 18/57] The revocation-guard census lists the shared IdP opener wrapper as a builder (BACKLOG #3260 #3145 #2950) Review round two found tests/test_revocation_guard_census.py red on this branch: build_idp_opener_from_settings became a site the census did not list, and its two callers stopped counting as sites, so the census could no longer see AuthService.__init__'s tie to _refuse_idp_revocation. The wrapper is now in _BUILDERS, so both callers are sites again and the census passes (5 of 5). Round-two findings left open, for the PR body: a client timeout above 30 s shortens the read on the queued calls (use max); store_call_errors lacks pyodbc.Error for an unmapped SQLSTATE such as a deadlock; change_password also queues behind the re-proof lock and keeps 5 s; the store-fault fallback drops X-Step-Up-Via, so a client prompts a federated user for a password; the finally row write can replace a propagating CancelledError with a defect; the miss row's soft write catches Exception; three overlapping store-fault sets; the queue itself is unbounded (engine-side bound would close the lost reply); the 3260 changelog omits the malformed case; store_call_errors now imports the server drivers at every AuthService construction; _SESSION_ID_SHAPE admits short hex and duplicates DIGEST_ID_PATTERN; some added comment lines exceed 100 columns. Proposed PR title: b257 D1+D2 auth follow-ups: TOTP refusal row, directory notices, revoke-miss audit, OIDC step-up follow-ups, sign-in timeout Proposed vault banners: #3113 (D1 carry-over, round-two findings 1, 3, 5, 6): BUILT on engine branch b257-d2. The auth.mfa_enroll_refused row is written in a finally around the revoke, soft-written itself, so no exception the revoke raises skips it, and aiosqlite's closed-connection ValueError is logged as a store refusal. The directory role notice says the open sessions were ended; the stale notifications.py comment and two SECURITY.md wording points are fixed. Round-two findings 2, 4 and 7 stay open. #3260: limb 2 BUILT on engine branch b257-d2, closing the row with limb 1 (engine PR 2190). revoke_own_session writes one auth.session_revoke_missed row per miss on both planes, through write_audit_soft, with the id cut to 12 characters (a console id that is not hex is recorded as malformed) and nothing else, so an unknown id and a foreign id get the same answer and the same row (owner ruling 2026-10-16, wiki note 20261016T211331989Z-vfzmx7). The engine UI seam moved to 0971e2440e4237c7. Register 1af question 4 is answered by this build, and the 1af entry needs a re-draft. #3145: BUILT on engine branch b257-d2. Items 4, 5 and 6(b)(c)(d) built, item 3 built for the opener's argument assembly; items 1, 2, 6(a), the checks.py preamble half of 3, and 6's extra session read declined in one written line each (see the #3145 commit body). #2950: BUILT on engine branch b257-d2. login, reauth, verify_mfa and confirm_mfa wait 30 s for their reply (SIGN_IN_TIMEOUT_SECONDS, the IDE's value), read phase only; every other request keeps 5 s. The per-account queue has no bound, so a lost reply is rarer, not impossible. --- tests/test_revocation_guard_census.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_revocation_guard_census.py b/tests/test_revocation_guard_census.py index 75153d4f51..15947c4fcf 100644 --- a/tests/test_revocation_guard_census.py +++ b/tests/test_revocation_guard_census.py @@ -69,6 +69,9 @@ "_verifying_context", "_build_tls_context", "build_idp_opener", + # The settings-reading wrapper both IdP opener sites call (vault BACKLOG #3145), so those + # two stay the sites and the wrapper hands its opener on, as build_idp_opener does. + "build_idp_opener_from_settings", "NarrowedTls", } ) From ebd88b2903844ee89160789461dd75bdeb43d705 Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Fri, 16 Oct 2026 20:33:28 -0500 Subject: [PATCH 19/57] Handover for the third limb: proposed PR title and consolidated banner (BACKLOG #3010) No code change. Review round two ran on 2cae56b4 and is the last round; its findings ship open, listed below and in the Builder's report. Proposed PR title: An LDAP search that does not end with success is an outage, not "no such account" (BACKLOG #3010) Proposed vault banner (consolidated, replaces the texts in c6113d94 and 6512a27d): BUILT on branches b257-3010, b257-3010b and b257-3010c (stacked). - _search in auth/ldap.py refuses any result code but success, with or without entries, through _refuse_unanswered; the reconciler reads it as unavailable and sign-in as a login error. Closing items 1-3 built and tested; item 4 deferred to #3100 item 3, which must widen to the except LdapError arm before the deferral holds. - The nested group search is paged at 500 per page. A page that does not end with success, a later page that drops the paging control, or a search still paging after 20 pages refuses the whole search, never a shorter group list. A refused paged search is abandoned (RFC 2696) once the directory has sent a cookie. - One boundary, _ldap3_call, wraps every ldap3 call that reads a response: each search and page, the service auto_bind, the user and decoy binds, and Who am I. A KeyError, TypeError, IndexError, ValueError or AttributeError that ldap3 raised while reading the response, decided by traceback origin and not by value, becomes LdapUndecodableError, an outage, with no result code claimed. Engine-raised exceptions propagate. This covers a bind response carrying a referral (KeyError(3)), a code outside ldap3's table, a malformed paged control, and a Who am I value that is not text. The unknown-code value mapping is gone. A static test pins every search, bind and Who am I call inside the boundary. - Measured through real ldap3 2.9.1 against a loopback server, not a DC. Shipped open: the user bind still reads busy and other non-success codes as a wrong password (#3169); ldap3's ranged memberOf follow-up searches are not refused (a row); the close of an undecodable search's socket is undone by ldap3's Connection.__exit__, which reopens it (measured); a referral in a bind response reads as undecodable, not REFERRED; standing configuration codes and decode faults reach the reconciler at debug level only; no overall deadline across pages; the abandon is sent after a busy page and can reuse a spent cookie; no cap on the paged cookie's length. From cf6b2b78c024e49c654a6fdd4a9d3700332476f4 Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Fri, 16 Oct 2026 20:55:06 -0500 Subject: [PATCH 20/57] Revert the socket close on the search path: ldap3's with-exit reopens it (BACKLOG #3010) _search no longer passes close=True to _ldap3_call. Callers hold the service connection in a with block, and ldap3 2.9.1's Connection.__exit__ reopens a connection that was open on entry and is closed on exit, so the close cost one more TCP and TLS connect per malformed search and closed nothing. The service-account auto_bind close stays. The paged-control test now pins that a search refusal leaves the caller's connection open; the docstrings and the changelog claim the close for the service-account bind only. Proposed PR title: An LDAP search that does not end with success is an outage, not "no such account" (BACKLOG #3010) Proposed vault banner (consolidated, replaces the texts in c6113d94, 6512a27d and ebd88b29): BUILT on branches b257-3010, b257-3010b, b257-3010c and b257-3010d (stacked). - _search in auth/ldap.py refuses any result code but success, with or without entries, through _refuse_unanswered; the reconciler reads it as unavailable and sign-in as a login error. Closing items 1-3 built and tested; item 4 deferred to #3100 item 3, which must widen to the except LdapError arm before the deferral holds. - The nested group search is paged at 500 per page. A page that does not end with success, a later page that drops the paging control, or a search still paging after 20 pages refuses the whole search, never a shorter group list. A refused paged search is abandoned (RFC 2696) once the directory has sent a cookie. - One boundary, _ldap3_call, wraps every ldap3 call that reads a response: each search and page, the service auto_bind, the user and decoy binds, and Who am I. A KeyError, TypeError, IndexError, ValueError or AttributeError that ldap3 raised while reading the response, decided by traceback origin and not by value, becomes LdapUndecodableError, an outage, with no result code claimed. Engine-raised exceptions propagate. This covers a bind response carrying a referral (KeyError(3)), a code outside ldap3's table, a malformed paged control, and a Who am I value that is not text. The unknown-code value mapping is gone. A static test pins every search, bind and Who am I call inside the boundary. After an undecodable response the socket is closed for the service-account bind only; a search's socket is left to its caller, because ldap3's Connection.__exit__ would reopen it. - Measured through real ldap3 2.9.1 against a loopback server, not a DC. Shipped open: the user bind still reads busy and other non-success codes as a wrong password (#3169); ldap3's ranged memberOf follow-up searches are not refused (a row); after an undecodable search response the connection stays open on a stream that may be out of step, and authenticate and probe_principal never unbind the service connection (a row); a referral in a bind response reads as undecodable, not REFERRED; standing configuration codes and decode faults reach the reconciler at debug level only; no overall deadline across pages; the abandon is sent after a busy page and can reuse a spent cookie; no cap on the paged cookie's length. --- changelog.d/3010.security.md | 2 +- messagefoundry/auth/ldap.py | 16 ++++++++-------- tests/test_ldap_decode_boundary.py | 3 ++- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/changelog.d/3010.security.md b/changelog.d/3010.security.md index 455d824601..f207f5cdff 100644 --- a/changelog.d/3010.security.md +++ b/changelog.d/3010.security.md @@ -28,4 +28,4 @@ ldap3 raised it reading the response, and lets any other exception propagate. Such a fault now reaches the session reconciler as an outage, which it logs at debug level, rather than as an unexpected error, which it logged at WARNING for each account. After one, the engine closes the - socket of the service-account bind or search that met it. (vault `BACKLOG #3010`) + socket of a service-account bind that met it. (vault `BACKLOG #3010`) diff --git a/messagefoundry/auth/ldap.py b/messagefoundry/auth/ldap.py index da0c02fabf..c8628a95ad 100644 --- a/messagefoundry/auth/ldap.py +++ b/messagefoundry/auth/ldap.py @@ -868,7 +868,7 @@ def _search(conn: Any, operation: str, **kwargs: Any) -> None: (:func:`_refuse_unanswered`). Every search in this module goes through here, so none can read either as "no entries"; a test pins that. The search runs inside :func:`_ldap3_call`. """ - with _ldap3_call(operation, close=True): + with _ldap3_call(operation): conn.search(**kwargs) _refuse_unanswered(conn, operation) @@ -945,11 +945,12 @@ def _ldap3_call(operation: str, *, close: bool = False) -> Iterator[None]: debug level (:func:`_reading_connection` says how the two are told apart). An ``LDAPException`` passes through untouched, for its own handler. - ``close`` closes the socket the failed read was on, where nothing else will. The service - account's ``auto_bind`` runs inside ldap3's ``Connection`` constructor, so when it raises no - caller holds the connection to unbind. A search's caller abandons the connection on any error, - and the stream may be out of step after a fault partway through a response; the service - connection is never unbound, so its socket would stay open for as long as the error is kept. + ``close`` closes the socket the failed read was on, where nothing else will. Only the service + account's bind passes it: its ``auto_bind`` runs inside ldap3's ``Connection`` constructor, so + when it raises no caller holds the connection to unbind. A search does not. Its caller holds + the connection in a ``with`` block, and ldap3 2.9.1's ``Connection.__exit__`` reopens a + connection that was open on entry and is closed on exit, so a close there would cost one more + connect and close nothing (measured, BACKLOG #3010). """ import ldap3 @@ -1030,8 +1031,7 @@ def _search_paged(conn: Any, operation: str, **kwargs: Any) -> list[Any]: **A refused search is abandoned** (:func:`_abandon_paging`) once the directory has sent a cookie, so it need not hold the paging state until its own timeout. Not after a response ldap3 - could not decode: the stream may be out of step after one, so nothing more is sent on it, and - :func:`_search` closes the socket instead, which drops the paging state with the connection. + could not decode: the stream may be out of step after one, so nothing more is sent on it. """ entries: list[Any] = [] cookie: bytes | None = None # what the next page is asked with diff --git a/tests/test_ldap_decode_boundary.py b/tests/test_ldap_decode_boundary.py index a3f103fb73..7389ff566e 100644 --- a/tests/test_ldap_decode_boundary.py +++ b/tests/test_ldap_decode_boundary.py @@ -243,7 +243,8 @@ def answer(raw: bytes) -> list[_Reply]: with _service_conn(authenticator) as conn: with pytest.raises(LdapError) as refused: authenticator._resolve_groups(conn, _USER_DN, []) - assert conn.closed, "the stream may be out of step, so the socket is closed" + # Not closed: the caller's ``with`` would reopen it (_ldap3_call says why). + assert not conn.closed, "a search refusal must not close the caller's connection" _assert_undecodable(refused.value, "group search", fault) assert len(dc.searches) == (2 if later else 1), "a search was sent after the refusal" assert all(_IN_CHAIN in raw for raw in dc.searches) From 87c0405028b2b5b294befaea856919823701e36a Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Fri, 16 Oct 2026 21:17:46 -0500 Subject: [PATCH 21/57] Review round one: the search-close docstring names what the close did and what not closing costs (BACKLOG #3010) The previous commit said a close on the search path closed nothing. It did shut the failed socket; ldap3's Connection.__exit__ then opened a fresh, unbound one, which is the extra connect. _ldap3_call now says so, names authenticate and probe_principal as the callers whose with block reopens it, and states the cost of not closing: their connection stays open and bound on a stream that may be out of step until it is collected, while read_bind_account unbinds its own. _search_paged says the directory keeps the paging state after a decode fault. The test comment no longer cites a with block the test helper does not use. Proposed PR title: An LDAP search that does not end with success is an outage, not "no such account" (BACKLOG #3010) Proposed vault banner (consolidated, replaces the texts in c6113d94, 6512a27d, ebd88b29 and cf6b2b78): BUILT on branches b257-3010, b257-3010b, b257-3010c and b257-3010d (stacked). - _search in auth/ldap.py refuses any result code but success, with or without entries, through _refuse_unanswered; the reconciler reads it as unavailable and sign-in as a login error. Closing items 1-3 built and tested; item 4 deferred to #3100 item 3, which must widen to the except LdapError arm before the deferral holds. - The nested group search is paged at 500 per page. A page that does not end with success, a later page that drops the paging control, or a search still paging after 20 pages refuses the whole search, never a shorter group list. A refused paged search is abandoned (RFC 2696) once the directory has sent a cookie. - One boundary, _ldap3_call, wraps every ldap3 call that reads a response: each search and page, the service auto_bind, the user and decoy binds, and Who am I. A KeyError, TypeError, IndexError, ValueError or AttributeError that ldap3 raised while reading the response, decided by traceback origin and not by value, becomes LdapUndecodableError, an outage, with no result code claimed. Engine-raised exceptions propagate. This covers a bind response carrying a referral (KeyError(3)), a code outside ldap3's table, a malformed paged control, and a Who am I value that is not text. The unknown-code value mapping is gone. A static test pins every search, bind and Who am I call inside the boundary. After an undecodable response the socket is closed for the service-account bind only. A search's is not: authenticate and probe_principal hold the connection in ldap3's with block, whose __exit__ reopens a closed connection, so a close there cost one more connect. - Measured through real ldap3 2.9.1 against a loopback server, not a DC. Shipped open: the user bind still reads busy and other non-success codes as a wrong password (#3169); ldap3's ranged memberOf follow-up searches are not refused (a row); after an undecodable search response in authenticate or probe_principal the service connection stays open and bound, on a stream that may be out of step and holding any paging state, until it is collected, because neither method ever unbinds it (a row; the fix is the release read_bind_account uses, not a close flag); read_bind_account's unbind is then sent on that stream; a referral in a bind response reads as undecodable, not REFERRED; standing configuration codes and decode faults reach the reconciler at debug level only; no overall deadline across pages; the abandon is sent after a busy page and can reuse a spent cookie; no cap on the paged cookie's length. --- messagefoundry/auth/ldap.py | 14 +++++++++----- tests/test_ldap_decode_boundary.py | 2 +- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/messagefoundry/auth/ldap.py b/messagefoundry/auth/ldap.py index c8628a95ad..cc33ae8f32 100644 --- a/messagefoundry/auth/ldap.py +++ b/messagefoundry/auth/ldap.py @@ -947,10 +947,13 @@ def _ldap3_call(operation: str, *, close: bool = False) -> Iterator[None]: ``close`` closes the socket the failed read was on, where nothing else will. Only the service account's bind passes it: its ``auto_bind`` runs inside ldap3's ``Connection`` constructor, so - when it raises no caller holds the connection to unbind. A search does not. Its caller holds - the connection in a ``with`` block, and ldap3 2.9.1's ``Connection.__exit__`` reopens a - connection that was open on entry and is closed on exit, so a close there would cost one more - connect and close nothing (measured, BACKLOG #3010). + when it raises no caller holds the connection to unbind. A search does not (BACKLOG #3010). + ``authenticate`` and ``probe_principal`` hold the service connection in ldap3's own ``with``, + and ldap3 2.9.1's ``Connection.__exit__`` reopens a connection that was open on entry and is + closed on exit. So a close there shut the failed socket and then cost one more connect for a + fresh, unbound one. The cost of not closing: after an undecodable search response, those two + callers' connection stays open and bound, on a stream that may be out of step, until it is + collected. ``read_bind_account`` unbinds its own in a ``finally``. """ import ldap3 @@ -1031,7 +1034,8 @@ def _search_paged(conn: Any, operation: str, **kwargs: Any) -> list[Any]: **A refused search is abandoned** (:func:`_abandon_paging`) once the directory has sent a cookie, so it need not hold the paging state until its own timeout. Not after a response ldap3 - could not decode: the stream may be out of step after one, so nothing more is sent on it. + could not decode: the stream may be out of step after one, so no abandon is sent on it, and + the directory holds the paging state until its own timeout or the connection ends. """ entries: list[Any] = [] cookie: bytes | None = None # what the next page is asked with diff --git a/tests/test_ldap_decode_boundary.py b/tests/test_ldap_decode_boundary.py index 7389ff566e..365933571a 100644 --- a/tests/test_ldap_decode_boundary.py +++ b/tests/test_ldap_decode_boundary.py @@ -243,7 +243,7 @@ def answer(raw: bytes) -> list[_Reply]: with _service_conn(authenticator) as conn: with pytest.raises(LdapError) as refused: authenticator._resolve_groups(conn, _USER_DN, []) - # Not closed: the caller's ``with`` would reopen it (_ldap3_call says why). + # Not closed: _search leaves the connection to its caller (_ldap3_call says why). assert not conn.closed, "a search refusal must not close the caller's connection" _assert_undecodable(refused.value, "group search", fault) assert len(dc.searches) == (2 if later else 1), "a search was sent after the refusal" From 2d1ddf145babaf92a19c9bc88de65ca811c3271b Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Fri, 16 Oct 2026 21:38:16 -0500 Subject: [PATCH 22/57] A directory reconcile pass that newly grants Administrator raises administrator_granted (BACKLOG #3141) - AuthService takes an admin_grant_alert callable, the #3229 shape: the API lifespan builds it with pipeline.alerts.administrator_granted_alert and the marker, so auth/ imports no sink. - _apply_reconcile_revocation raises it right after the role write, on the sealed user.role_ids read at the top of the method, never the plan's roles. Only Administrator absent before and present after pages. Only a pass that did not abort applies a revocation. via = directory_reconcile. It pops the no_enabled_administrator cooldown, as intended. - Docs: alert_sinks comment, changelog.d/3229 and 2610 (2610 now states the server-error-after-role-write limit and the never-seen grant), new changelog.d/3141, SECURITY.md "A directory grant" bullets and the signal table, PHI.md alert_instance row, AlertSink docstring, the settings alert-type comment. - Item 3 of the row, from PR 2076 "Known, not fixed": the two low test findings are fixed (the console refused-after-sync tests assert the role was written; the caplog check filters on the logger name). Declined: two concurrent first sign-ins can each page once (a duplicate page is the safe direction); a future directory sign-in route must remember to call the alert (no route registry to enforce it against, and a census test is its own item). --- changelog.d/2610.security.md | 6 +- changelog.d/3141.security.md | 10 ++ changelog.d/3229.security.md | 4 +- docs/PHI.md | 2 +- docs/SECURITY.md | 23 ++- messagefoundry/api/app.py | 7 + messagefoundry/auth/service.py | 36 ++++ messagefoundry/config/settings.py | 3 +- messagefoundry/pipeline/alert_sinks.py | 8 +- messagefoundry/pipeline/alerts.py | 28 ++- .../tests/test_ui_directory_admin_alert.py | 9 + tests/test_directory_last_admin_alert.py | 165 +++++++++++++++++- tests/test_directory_sign_in_admin_alert.py | 5 +- 13 files changed, 282 insertions(+), 24 deletions(-) create mode 100644 changelog.d/3141.security.md diff --git a/changelog.d/2610.security.md b/changelog.d/2610.security.md index af9dcb5c82..b8df3fa63b 100644 --- a/changelog.d/2610.security.md +++ b/changelog.d/2610.security.md @@ -6,4 +6,8 @@ `user:`, with `via` naming the route and `granted_by` the fixed marker ``. A sign-in refused after the role was written still raises it, since a later sign-in gains nothing to page on. A sign-in that gains only other roles, or by an account that already held - Administrator, raises nothing. (`vault BACKLOG #2610`) + Administrator, raises nothing. At least two cases still raise no alert. A sign-in that fails with + a server error after the role write loses the page, because the retry gains nothing. And an + account that never signs in again and holds no live session raises nothing, because the engine + never sees the grant. A directory reconcile pass that applies the grant first raises the alert + itself (vault BACKLOG #3141). See docs/SECURITY.md, *A directory grant*. (`vault BACKLOG #2610`) diff --git a/changelog.d/3141.security.md b/changelog.d/3141.security.md new file mode 100644 index 0000000000..8ba45684c5 --- /dev/null +++ b/changelog.d/3141.security.md @@ -0,0 +1,10 @@ +- **A directory reconcile pass that newly grants Administrator now raises the + `administrator_granted` operator alert.** The session reconciler re-diffs a signed-in directory + account's roles and writes them. When that write gave the account Administrator, it raised only + `ad_session_revoked`, which does not name the role, and the account's next sign-in then gained + nothing, so no `administrator_granted` alert ever fired. The pass now raises it right after the + role write, keyed `user:`, with `via` set to `directory_reconcile` and `granted_by` the + fixed marker ``. An account that already held Administrator raises nothing. A pass the + mass-revoke breaker aborts writes no roles, so it raises nothing either. Like every + `administrator_granted` alert, it clears the account's `no_enabled_administrator` cooldown in the + engine process that raises it. (`vault BACKLOG #3141`) diff --git a/changelog.d/3229.security.md b/changelog.d/3229.security.md index 0aa2b60b62..aba01eb54b 100644 --- a/changelog.d/3229.security.md +++ b/changelog.d/3229.security.md @@ -7,8 +7,8 @@ `auth.no_enabled_administrator` audit row. With no enabled Administrator left, `provision-admin` runs again at the host. An `administrator_granted` alert for an account clears that account's `no_enabled_administrator` cooldown. It clears it only in the engine process that raises the - `administrator_granted` alert. A reconciler re-grant raises no `administrator_granted` alert, so - it leaves the cooldown in place. A directory disable or deletion of the last Administrator raises + `administrator_granted` alert. A reconciler re-grant raises `administrator_granted` too (vault + BACKLOG #3141), so it clears the cooldown the same way. A directory disable or deletion of the last Administrator raises no `no_enabled_administrator` alert, and `provision-admin` still refuses. For at least these cases and others that raise no `no_enabled_administrator` alert, see docs/SECURITY.md, *A directory role change can remove the last Administrator, and it alerts*. (`vault BACKLOG #3229`) diff --git a/docs/PHI.md b/docs/PHI.md index d3db7efce9..7ee01635a1 100644 --- a/docs/PHI.md +++ b/docs/PHI.md @@ -1808,7 +1808,7 @@ with materially different PHI profiles, so they get their own rows; stream 4 is | **5. `audit_log` table** (SQLite, Postgres, SQL Server) | who / what / **where-from** / when of auth + PHI *access* and admin actions — plus, while `[security].audit_all_authorization_decisions` is on (**default `true`** since BACKLOG #1277, 2026-09-02, which reversed the ADR 0118 §5 `false`; the internal field it desugars to is `audit_all_authz`, whose old `[diagnostics]` TOML spelling is **refused at load** — ADR 0118), an `authz` row for **every** authorization decision including successes. That is the shipped volume of this stream, not an opt-in addition to it: one row per authenticated request on each `require()`-gated route, and this row's retention cell records that **nothing prunes the table** — `actor`, `action`, `channel_id`, `client`, `detail`, `row_hash` | JSON `detail`; **tamper-evident hash chain** over `prev_hash` + the row (the `client` address is **inside** the chained payload — ADR 0150) | the store database | HIPAA §164.312(b) audit controls; incident response; `verify_audit_chain` integrity checks | `GET /audit` requires **`audit:read`**; `GET /audit/export` requires the separate **`audit:export`** and streams CSV with formula-injection neutralisation, recording its own `audit.export` row *before* streaming; `GET /me/security-events` is a per-user view of the same table | **`[retention].audit_days` is reserved and NOT enforced — keep-forever by design** (the audit-retention requirement, ~6 years — **not** chain-breakage; [§8](#8-retention--purge) states the position and cites its source of record) | `detail` is stored **in the clear** (it is not a cipher-covered column): its protection is that writers store at least filter shapes, counts, ids, keyed digests, bounded engine-authored text and client addresses, the `audit_log.detail` row above says which — never bodies or credentials — plus the store ACL and the volume layer. *Corrected 2026-10-16 (vault BACKLOG #3145):* this said writers "only ever store filter shapes, counts and ids" | | **6. `message_events` table** | the per-message disposition timeline — the **complete** vocabulary is `received`, `routed`, `unrouted`, `filtered`, `transformed`, `delivered`, `failed`, `dead`, `error`, `replayed`, `resent`, `reingressed`, `passthrough`, `passthrough_dropped`, `cancelled`, `edit_resend`, `edit_resubmit`, `viewed`, `not_deployed`, the ADR 0154 synchronous-reply pair `reply_returned` / `reply_timeout` (names, counts and `waited_ms` only — **never** a fragment of the partner's reply body), and `reply_dropped`, a delivery whose 2xx reply body was over the byte bound and dropped (the status and the bound only, vault BACKLOG #3040) (CI asserts this list against the engine's own `MESSAGE_EVENT_KINDS`). `[diagnostics].message_events` can thin the set, but never below the compliance floor `viewed` / `dead` / `error` / `failed` / `not_deployed` / `reply_timeout` / `reply_dropped` | rows: `message_id`, `ts`, `event`, `destination`, `detail` | the store database | operator timeline on the message-detail view; the `viewed` row is the HIPAA PHI-access record | `GET /messages/{id}` under **`messages:view_raw`** + `require_phi_read`; the read itself writes a `viewed` event **and** a `message_view` audit row | no dedicated window — `purge_message_bodies` sets `message_events.detail` to `NULL` in the same transaction that blanks the body, so it inherits `[retention].messages_days` | `detail` is `safe_text()`-scrubbed **then** cipher-encrypted (AAD `("message_events","detail",message_id,ts,event)`). Verbosity gate `[diagnostics].message_events` = `all` (default) / `errors` / `off`, with a **compliance floor that can never be thinned**: `viewed`, `dead`, `error`, `failed`, `not_deployed`, `reply_timeout`, `reply_dropped` are retained at every level (`reply_timeout` is the one row that explains a "we called you and got a 504" complaint, so an instance that thinned its logs would lose exactly the record it is later asked for; `reply_dropped` is the one row that answers "you ignored our reply", for the same reason) | | **7. `connection_event` table — DEFAULT ON** (`[diagnostics].connection_events = true`) | transport/lifecycle events per connection: `established`, `closed` (reason `eof`, `idle_timeout` or `frame_deadline` — a fixed vocabulary chosen by the listener, never free text), `idle_timeout`, `at_capacity`, `peer_not_allowlisted`, `frame_oversize`, `framing_error`, `peer_reset`, the MLLP listener's `handler_error` (BACKLOG #1619 -- the inbound handler faulted on a frame the listener read cleanly, such as a store outage at the ingress commit; the reason is scrubbed at the source like every other), the MLLP listener's `tls_handshake_failed` (vault BACKLOG #2613 -- a TLS handshake on a listener that runs its own handshakes (the standard asyncio loop, not uvloop) failed; the first at once, then one per listener per minute carrying the count since the last, and the reason is a count and an exception class name, never the peer's bytes), the DATABASE poll source's `row_undecodable` (BACKLOG #1662 — a polled row the source cannot turn into a body; the reason is scrubbed at the source and names the exception type or the operator's own configured column, never a row value), the FILE poll source's quarantine kinds `file_oversize` / `file_decompress_failed` / `file_content_mismatch` / `file_scan_rejected` (BACKLOG #1621 — a drop moved to `.error`; the reason is a size, a content type or a scrubbed codec or scanner message, never the file name), the inbound-HTTP intake-auth refusals `intake_auth_failed` / `auth_subject_denied` / `auth_rate_limited` (ADR 0154 D6 — peer address and mode only; **never** the credential, a prefix of it, or its length. Each of these also writes a tamper-evident audit-log row — the copy that survives an operator turning this diagnostics stream off), plus the runner's `connection_lost` / `connection_restored`. That is the whole vocabulary, asserted in CI against the literal emit call sites in `transports/` and the pipeline runner **and** cross-checked against the console's own filter tuple. The MLLP, raw-TCP, X12 and HTTP listeners emit these; the **DATABASE poll source** emits `row_undecodable` and nothing else (it dials out, so there is no lifecycle to report and its rows carry a **NULL peer column**); the **FILE poll source** emits only its four quarantine kinds, also with a NULL peer column; the **DICOM inbound C-STORE SCP** emits none — the runner injects the sink onto **every** source (`wiring_runner.py`, over the base-class `on_connection_event` field), so that connector *has* the wiring and simply never calls it — so this stream covers those four listeners plus those poll-source events and the runner's outbound-lane transitions — not literally every connection. A DICOM association's connects and refusals are therefore **absent** from this stream. The **`ISA`/`IEA`-framed X12 inbound** was equally absent until BACKLOG #1665 and is not any more: it emits the same seven kinds as the raw-TCP listener | rows: `ts`, `connection`, `transport`, `direction`, `kind`, `peer_host`, `message_id` (correlation hint), `reason` | the store database, **all three backends** | Corepoint-style transport diagnostics — "did the sender connect, and why did it drop" | `GET /events` and `GET /connections/{name}/events` under **`monitoring:read`** (**not** a PHI permission) with per-channel RBAC — an out-of-scope `connection=` is 403'd *and* audited — server-clamped to ≤1000 rows. `reason` alone is gated on `messages:view_summary` and masked as `****` until the audited per-event `reveal=` act (BACKLOG #2443; the `connection_event.reason` row of §3's PL-2 block says how) | `[retention].connection_event_retention_hours` (its own **hours** window); 0 inherits `[retention].messages_days`; both 0 = keep forever. Plain age `DELETE` (metadata-only) | **`reason` is free text that can carry sensitive fragments.** Defended twice — `safe_exc()` at the source, `safe_text(reason, limit=200)` at the store (the bound §3's PL-2 block states) — then cipher-encrypted (AAD `("connection_event","reason",connection,ts,kind)`). Every other column is config metadata; the table is documented **metadata-only** — never a frame, body or HL7 field value. Writes are a pure side observer: a bounded in-memory queue drained by a background task outside any handoff transaction, so a flood can never block a listener or pin a message disposition | -| **8. `alert_instance` table — default on wherever an `[alerts]` notifier exists** | resolvable operator alerts: `connection_stopped`, `queue_buildup`, `lane_stuck`, `message_stall`, `saturation`, `connection_error`, `storage_threshold`, `cert_expiry`, `secret_rotation`, `initial_credential_expiring` (an UNCLAIMED admin-issued temporary password nearing the instant the login gate stops accepting it -- ASVS 6.4.5, BACKLOG #1141; keyed on `user:`, and the payload is the ISO deadline plus whole hours remaining, repeated in the reason column, never the password), `integrity_drift`, `update_available`, `backup_failed`, `store_privilege_warning` (the store privilege preflight found the store principal over-granted or could not read it -- BACKLOG #305; the payload is the finding, a count and the preflight's summary line, which names the login, the database and role names, never a password or message content), `leadership_acquired`, `dr_activated`, `log_write_failed` (an application-log sink was rolled after a write failure, or is UNWRITABLE and this process's connections were stopped -- BACKLOG #122, ADR 0162; the payload is the sink LABEL, the stage, a `safe_exc` reason and a count of connections stopped, never the record whose write failed), `log_forward_failed` (the off-box log forwarder is absent, losing records or not sending -- BACKLOG #2612; keyed on `forwarder:`, with `@node:` or `@shard:` appended when several engine processes share the store, and the payload is that key, fixed reason words and a count, never a record, the collector's address or an error text; an unpinned cluster node's `node_id` holds this engine's own host name and process id, so the key carries them), `gcm_invocations` (the per-key AES-GCM invocation bound crossing its 2^31 soft warn — ASVS 11.3.4; its payload carries a one-way `key_id` fingerprint plus counters, never key bytes), `approval_stale_requester` (a dual-control release refused because the requester no longer holds the authority the operation needs -- ASVS 8.3.2, BACKLOG #289; keyed on the approval id, and the payload is that id, the operation key and a closed-set reason slug, never the requester's name or the captured params), `approval_too_early` (a dual-control release refused because it arrived before `[approvals].min_dwell_seconds` -- ASVS 2.4.2, BACKLOG #287; keyed on `approval:`, and the payload is that key, the operation key and a fixed reason string, never the approver's or requester's name or the captured params), `audit_write_failed` (the approval gate could not write one of its audit rows -- vault BACKLOG #2255; keyed on `approval:`, or on `config_reload:inline` when an ungated config reload could not write its `config_reload_attempted` row -- vault BACKLOG #2254; the payload is that key and the lost row's audit action name, never the row's detail, a username or the captured params; the ERROR log line beside it holds the detail on some paths and only the exception on others), `ad_reconcile_aborted` (a directory reconciler pass left signed-in accounts unrevoked and needs an operator: the mass-revoke breaker tripped and nothing was revoked, or one or more probes were answered with an LDAP referral and those accounts were left unjudged while the rest of the pass was applied, BACKLOG #2538; the payload is a fixed source label, `directory-reconciler` for the breaker and `directory-reconciler-referral` for a referral so each is its own instance, the closed-set reason slug `mass_revoke_breaker` or `directory_referral`, the probed count and the latched operator explanation, never a username), `ad_reconcile_held` (the reconciler is holding accounts whose `userAccountControl` it cannot read, ADR 0195; the payload is the same fixed source label, a closed-set reason slug, the count of undetermined accounts and the latched operator explanation, never a username), `ad_session_revoked` (the reconciler revoked a directory principal's sessions; keyed on that operator account's username, with a `directory_absent`, `directory_disabled`, `directory_undetermined`, `roles_changed` or `scope_changed` reason), `approval_approver_provenance` (a dual-control release, or the resolution of an interrupted one, went ahead with an approver or resolver account created, re-passworded or TOTP-enrolled after the request -- BACKLOG #315, #2125; keyed on `approval:`, or `approval::resolve` for a resolution, and the payload is the operation key and the closed-set change slugs, never a username or the captured params), `administrator_granted` (the built-in Administrator role was granted by account creation, a role change, a directory group newly mapped to it -- BACKLOG #315 -- or a directory sign-in whose role sync newly gave the account the role -- vault BACKLOG #2610; keyed on `user:` or `ad-group:`, with the grant route and the granting administrator's username, or `` for a directory sign-in), `no_enabled_administrator` (a directory sign-in's role sync or the session reconciler took Administrator from its last enabled holder -- vault BACKLOG #3229; keyed on `user:`, and the payload is `via`, `ad-sync` or `ad-reconcile`, and a fixed reason sentence, never message content), `intake_paused` (the engine paused intake because the staged backlog went over `[inbound].max_staged_depth` or the SQLite volume fell below `[retention].min_free_disk_mb` -- BACKLOG #290; keyed on `intake:staged_depth` or `intake:disk_floor`, with `@node:` or `@shard:` appended when several engine processes share the store (BACKLOG #2272), and the payload is the reason, a count or a MiB figure, its limit and the store backend's name, never message content), `config_changed` (a start loaded a config whose fingerprint differs from the store's baseline -- vault BACKLOG #2597; keyed on `config:`, and the payload is the two digests, the node and engine shard labels, and the action, actor username and time of the row the older digest came from, never the config directory, a git commit or message content), `crl_reload_refused` (the CRL reload refused a replaced CRL file, so running TLS contexts keep the copy they hold -- vault BACKLOG #2904; keyed on `crl_reload:`, with `@node:` or `@shard:` appended when several engine processes share the store, and the payload is that key, the file's path, a short class token for the remedy, a count of contexts, the engine's own fixed remedy sentence and, when a start would refuse the file too, a `start_refuses` flag, never the refusal's reason, which quotes CRL issuer names and dates; that reason is in the application log, and in the `detail` of the CRL expiry alert once the held copy nears its `nextUpdate`), `remote_debug_refused` (the engine's remote-debugging guard refused scripts that another process injected into the engine process -- vault BACKLOG #2742; keyed on `remote_debug:refused`, with `@node:` or `@shard:` appended when several engine processes share the store, and the payload is that key, the count of scripts refused since the alert was last raised, the count since the process started and a fixed sentence built from the two, never a script's file name, which the injecting process chooses; that name is in the application log, in the guard's WARNING line), and the six security signals of the audit-stream rule layer -- vault BACKLOG #2613: `signin_failure_burst` (keyed on `signin:`, never the typed username), `access_denied_burst`, `body_view_burst` and `bulk_export` (each keyed on `account:`), `log_level_debug` (keyed on `logging:debug`) and `posture_loosened` (keyed on `posture:start`); the payload is a count and a fixed sentence holding a level name, switch names or a username, never a message body, a message id or an audit row's detail. The reachable **inverse** signals — `connection_restored`, `leadership_lost`, `dr_released`, `store_privilege_clean`, `intake_resumed`, `crl_reload_applied` (it resolves `crl_reload_refused` for the same file on the same engine process, once the reload's record no longer holds the refusal, and when the reload stops -- vault BACKLOG #2904), `ad_reconcile_breaker_cleared`, `ad_reconcile_hold_released` (the last two resolve `ad_reconcile_aborted`, under either source, and `ad_reconcile_held`; when they do, and where they never do, is stated once in [CONFIGURATION.md](CONFIGURATION.md#when-the-reconcilers-two-alerts-resolve-themselves) -- BACKLOG #2136) — are never rows here: `_record_state` routes an inverse through `_AUTO_RESOLVE` to `resolve_alert_instances_for`, never to `upsert_alert_instance`. (One more mapped key, `connection_started`, is emitted by no code path today.) | rows: `event_type`, `connection`, `severity`, `status`, `first_seen`, `last_seen`, `count`, `reason`, `acked_by`, `acked_at`, `resolved_at`, `suspended_until`, `escalation_tier` | the store database, **all three backends** | the operator alert list — acknowledge / resolve / suspend. Durable state is recorded **before** any suppression or throttle return, so a muted alert still leaves a record | `GET /alerts/active` under **`monitoring:diagnose`** (**not** a PHI permission) with the same per-channel scope, and `reason` alone gated on `messages:view_summary` and masked as `****` until the audited per-alert `reveal=` act (BACKLOG #2443; the `alert_instance.reason` row of §3's PL-2 block says how); ack/resolve/suspend/**resume** are POSTs on the same tier, and the separate read-only `GET /alerts/rules` view sits on its own gate | shares the connection-event window; **only RESOLVED instances are DELETEd**, by `resolved_at` — an open or acknowledged condition is never aged out from under an operator | **`reason` is free text** taken from the event's `detail`/`reason`: `safe_text(reason, limit=200)` (the bound §3's PL-2 block states) then cipher-encrypted (AAD `("alert_instance","reason",event_type,connection)` — the de-dup grain, so one AAD covers both the INSERT and the re-fire UPDATE). | +| **8. `alert_instance` table — default on wherever an `[alerts]` notifier exists** | resolvable operator alerts: `connection_stopped`, `queue_buildup`, `lane_stuck`, `message_stall`, `saturation`, `connection_error`, `storage_threshold`, `cert_expiry`, `secret_rotation`, `initial_credential_expiring` (an UNCLAIMED admin-issued temporary password nearing the instant the login gate stops accepting it -- ASVS 6.4.5, BACKLOG #1141; keyed on `user:`, and the payload is the ISO deadline plus whole hours remaining, repeated in the reason column, never the password), `integrity_drift`, `update_available`, `backup_failed`, `store_privilege_warning` (the store privilege preflight found the store principal over-granted or could not read it -- BACKLOG #305; the payload is the finding, a count and the preflight's summary line, which names the login, the database and role names, never a password or message content), `leadership_acquired`, `dr_activated`, `log_write_failed` (an application-log sink was rolled after a write failure, or is UNWRITABLE and this process's connections were stopped -- BACKLOG #122, ADR 0162; the payload is the sink LABEL, the stage, a `safe_exc` reason and a count of connections stopped, never the record whose write failed), `log_forward_failed` (the off-box log forwarder is absent, losing records or not sending -- BACKLOG #2612; keyed on `forwarder:`, with `@node:` or `@shard:` appended when several engine processes share the store, and the payload is that key, fixed reason words and a count, never a record, the collector's address or an error text; an unpinned cluster node's `node_id` holds this engine's own host name and process id, so the key carries them), `gcm_invocations` (the per-key AES-GCM invocation bound crossing its 2^31 soft warn — ASVS 11.3.4; its payload carries a one-way `key_id` fingerprint plus counters, never key bytes), `approval_stale_requester` (a dual-control release refused because the requester no longer holds the authority the operation needs -- ASVS 8.3.2, BACKLOG #289; keyed on the approval id, and the payload is that id, the operation key and a closed-set reason slug, never the requester's name or the captured params), `approval_too_early` (a dual-control release refused because it arrived before `[approvals].min_dwell_seconds` -- ASVS 2.4.2, BACKLOG #287; keyed on `approval:`, and the payload is that key, the operation key and a fixed reason string, never the approver's or requester's name or the captured params), `audit_write_failed` (the approval gate could not write one of its audit rows -- vault BACKLOG #2255; keyed on `approval:`, or on `config_reload:inline` when an ungated config reload could not write its `config_reload_attempted` row -- vault BACKLOG #2254; the payload is that key and the lost row's audit action name, never the row's detail, a username or the captured params; the ERROR log line beside it holds the detail on some paths and only the exception on others), `ad_reconcile_aborted` (a directory reconciler pass left signed-in accounts unrevoked and needs an operator: the mass-revoke breaker tripped and nothing was revoked, or one or more probes were answered with an LDAP referral and those accounts were left unjudged while the rest of the pass was applied, BACKLOG #2538; the payload is a fixed source label, `directory-reconciler` for the breaker and `directory-reconciler-referral` for a referral so each is its own instance, the closed-set reason slug `mass_revoke_breaker` or `directory_referral`, the probed count and the latched operator explanation, never a username), `ad_reconcile_held` (the reconciler is holding accounts whose `userAccountControl` it cannot read, ADR 0195; the payload is the same fixed source label, a closed-set reason slug, the count of undetermined accounts and the latched operator explanation, never a username), `ad_session_revoked` (the reconciler revoked a directory principal's sessions; keyed on that operator account's username, with a `directory_absent`, `directory_disabled`, `directory_undetermined`, `roles_changed` or `scope_changed` reason), `approval_approver_provenance` (a dual-control release, or the resolution of an interrupted one, went ahead with an approver or resolver account created, re-passworded or TOTP-enrolled after the request -- BACKLOG #315, #2125; keyed on `approval:`, or `approval::resolve` for a resolution, and the payload is the operation key and the closed-set change slugs, never a username or the captured params), `administrator_granted` (the built-in Administrator role was granted by account creation, a role change, a directory group newly mapped to it -- BACKLOG #315 --, a directory sign-in whose role sync newly gave the account the role -- vault BACKLOG #2610 -- or a directory reconcile pass whose role re-diff did -- vault BACKLOG #3141; keyed on `user:` or `ad-group:`, with the grant route and the granting administrator's username, or `` for a directory sign-in or reconcile pass), `no_enabled_administrator` (a directory sign-in's role sync or the session reconciler took Administrator from its last enabled holder -- vault BACKLOG #3229; keyed on `user:`, and the payload is `via`, `ad-sync` or `ad-reconcile`, and a fixed reason sentence, never message content), `intake_paused` (the engine paused intake because the staged backlog went over `[inbound].max_staged_depth` or the SQLite volume fell below `[retention].min_free_disk_mb` -- BACKLOG #290; keyed on `intake:staged_depth` or `intake:disk_floor`, with `@node:` or `@shard:` appended when several engine processes share the store (BACKLOG #2272), and the payload is the reason, a count or a MiB figure, its limit and the store backend's name, never message content), `config_changed` (a start loaded a config whose fingerprint differs from the store's baseline -- vault BACKLOG #2597; keyed on `config:`, and the payload is the two digests, the node and engine shard labels, and the action, actor username and time of the row the older digest came from, never the config directory, a git commit or message content), `crl_reload_refused` (the CRL reload refused a replaced CRL file, so running TLS contexts keep the copy they hold -- vault BACKLOG #2904; keyed on `crl_reload:`, with `@node:` or `@shard:` appended when several engine processes share the store, and the payload is that key, the file's path, a short class token for the remedy, a count of contexts, the engine's own fixed remedy sentence and, when a start would refuse the file too, a `start_refuses` flag, never the refusal's reason, which quotes CRL issuer names and dates; that reason is in the application log, and in the `detail` of the CRL expiry alert once the held copy nears its `nextUpdate`), `remote_debug_refused` (the engine's remote-debugging guard refused scripts that another process injected into the engine process -- vault BACKLOG #2742; keyed on `remote_debug:refused`, with `@node:` or `@shard:` appended when several engine processes share the store, and the payload is that key, the count of scripts refused since the alert was last raised, the count since the process started and a fixed sentence built from the two, never a script's file name, which the injecting process chooses; that name is in the application log, in the guard's WARNING line), and the six security signals of the audit-stream rule layer -- vault BACKLOG #2613: `signin_failure_burst` (keyed on `signin:`, never the typed username), `access_denied_burst`, `body_view_burst` and `bulk_export` (each keyed on `account:`), `log_level_debug` (keyed on `logging:debug`) and `posture_loosened` (keyed on `posture:start`); the payload is a count and a fixed sentence holding a level name, switch names or a username, never a message body, a message id or an audit row's detail. The reachable **inverse** signals — `connection_restored`, `leadership_lost`, `dr_released`, `store_privilege_clean`, `intake_resumed`, `crl_reload_applied` (it resolves `crl_reload_refused` for the same file on the same engine process, once the reload's record no longer holds the refusal, and when the reload stops -- vault BACKLOG #2904), `ad_reconcile_breaker_cleared`, `ad_reconcile_hold_released` (the last two resolve `ad_reconcile_aborted`, under either source, and `ad_reconcile_held`; when they do, and where they never do, is stated once in [CONFIGURATION.md](CONFIGURATION.md#when-the-reconcilers-two-alerts-resolve-themselves) -- BACKLOG #2136) — are never rows here: `_record_state` routes an inverse through `_AUTO_RESOLVE` to `resolve_alert_instances_for`, never to `upsert_alert_instance`. (One more mapped key, `connection_started`, is emitted by no code path today.) | rows: `event_type`, `connection`, `severity`, `status`, `first_seen`, `last_seen`, `count`, `reason`, `acked_by`, `acked_at`, `resolved_at`, `suspended_until`, `escalation_tier` | the store database, **all three backends** | the operator alert list — acknowledge / resolve / suspend. Durable state is recorded **before** any suppression or throttle return, so a muted alert still leaves a record | `GET /alerts/active` under **`monitoring:diagnose`** (**not** a PHI permission) with the same per-channel scope, and `reason` alone gated on `messages:view_summary` and masked as `****` until the audited per-alert `reveal=` act (BACKLOG #2443; the `alert_instance.reason` row of §3's PL-2 block says how); ack/resolve/suspend/**resume** are POSTs on the same tier, and the separate read-only `GET /alerts/rules` view sits on its own gate | shares the connection-event window; **only RESOLVED instances are DELETEd**, by `resolved_at` — an open or acknowledged condition is never aged out from under an operator | **`reason` is free text** taken from the event's `detail`/`reason`: `safe_text(reason, limit=200)` (the bound §3's PL-2 block states) then cipher-encrypted (AAD `("alert_instance","reason",event_type,connection)` — the de-dup grain, so one AAD covers both the INSERT and the re-fire UPDATE). | | **9. `response` rows with `kind='ack_sent'` — DEFAULT ON** (`[diagnostics].response_sent = true`) | the ACK/NAK the engine returned to an inbound sender, under a sentinel destination `\x1fack:` | rows: `ack_code` (`AA`/`AE`/`AR`/`CA`/`CE`/`CR`), `ack_phase` (`decode`/`parse`/`strict`/`ingest`), `outcome`, `body`, `detail` | the store database | "what did we actually reply, and why" — the operator's answer to a sender disputing an ACK | `GET /messages/{id}/responses` under `messages:read` + `require_phi_read`; the `body` only for a caller who also holds `messages:view_raw` and `messages:view_summary`; every read writes a `response.read` audit row | `body`, `detail` and `resp_headers` are set to `NULL` in place by `purge_message_bodies` on the message-body window, on all three backends | **PHI fail-safe:** the ACK **body** is stored **only when the store cipher is active** — on a keyless store it is `NULL` rather than plaintext — and every NAK passes no body at all, so the offending field value is never persisted. The disposition metadata (`ack_code`/`ack_phase`/`outcome`) is non-PHI and always captured; `detail` is `safe_text`-scrubbed, held to the 200-character bound §3's PL-2 block states, and encrypted | | **10. `[alerts]` webhook transport** (off by default — `webhook_url` unset) | one HTTPS POST per alert, carrying every non-underscore event key as JSON | JSON | the operator's webhook endpoint (Slack/Teams/PagerDuty/custom) | operator notification | **`https` only** — a plaintext `http://` webhook URL is refused at construction unless the `MEFOR_ALLOW_INSECURE_TLS` escape is set (and then a warning is logged); since #329 this path routes that escape through the clamped `weakened_tls_escape_permitted(posture)` (the instance posture threaded from the API lifespan), so on an enforcing-PHI instance the escape is inert and a cleartext webhook POST stays refused — the same clamp as the connectors, no longer the raw escape. Redirects are refused; an optional `webhook_allowed_hosts` egress allowlist gates the host | the endpoint's | **carries the alert's `detail`/`reason` free text** (`safe_exc()`-scrubbed at the emit sites, but **not** re-run through `safe_text` on this path). Internal `_`-prefixed keys (per-rule recipients, rule id, cooldown) are stripped before send, so recipient addresses never cross the wire | | **11. `[alerts]` SMTP transport — operator alert list** (off unless `email_smtp_host` + `email_from` + ≥1 `email_to`) | one email per alert; default subject `[MessageFoundry] — `, default body every non-underscore event key as `k: v` | plain text (always kept — never HTML-only); optional HTML alternative | the operators' mailboxes | operator notification | `smtp_allowed_hosts` egress allowlist; the SMTP password comes from `MEFOR_ALERTS_EMAIL_PASSWORD` or a `[secrets]` provider, never the config file; per-send timeout `email_timeout` | the mail system's | carries the same `detail`/`reason` free text as the webhook. #138 operator templates are constrained to a **closed non-PHI variable allowlist** validated fail-closed at config load. **Transport posture:** `send_plain_email` builds an explicit **verifying** context (chain + hostname + strict RFC 5280, TLS 1.2 floor) via `tls_policy.build_smtp_tls_context()` and passes it to `starttls()`, anchored to the OS roots, `[alerts].email_tls_ca_file`, or `[tls].internal_ca_file` — the same factory the EMAIL and DIRECT *message destinations* use, so all three SMTP cells now share one policy ([#323](BACKLOG.md), closed 2026-08-02). Before that this call passed **no** context and Python's stdlib default applied (`ssl._create_stdlib_context` **is** `ssl._create_unverified_context` — `CERT_NONE`, `check_hostname = False`), leaving the hop encrypted but unauthenticated. There is still **no hop gradient or attestation on this path** — unlike the connectors, this cell is constructed outside the `active_hop_posture` scope, so its deviations (`email_use_tls = false`, or `email_tls_verify = false`) are gated by a `[security].allow_unverified_alert_smtp_tls` **acknowledgment switch at the serve gate** rather than by the clamped escape: on an enforcing PHI instance `serve` refuses to start without it, and permits + `AUDIT`-logs the start with it. Both deviations are named by `security_loosenings()` and reported by `messagefoundry check`'s `alert-smtp-tls` advisory | diff --git a/docs/SECURITY.md b/docs/SECURITY.md index db47ce1ea3..c37922f1e8 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -2402,7 +2402,7 @@ What the engine does instead is make the cheap routes loud. It refuses none of t | Signal | When | Where it goes | |---|---|---| | `approval.approver_provenance` audit row and `approval_approver_provenance` alert | A release goes ahead and the approver's account was created, had its password changed, or enrolled TOTP **after** the request was made. A resolution of an `interrupted` release does the same for the resolver's account (BACKLOG #2125) | Audit row against the approver or resolver, with their `client` address (ADR 0150). Its `act` field is `release` or `resolve`. Alert keyed `approval:` for a release and `approval::resolve` for a resolution, carrying the changed facts only | -| `administrator_granted` alert | `POST /users` creates an account with the Administrator role, `PUT /users/{id}/roles` adds it, `PUT /ad-group-map` newly maps a group to it, or a directory sign-in's role sync newly gives an account the role (vault BACKLOG #2610) | Alert keyed `user:` or `ad-group:`, naming the granting administrator, or `` for a directory sign-in | +| `administrator_granted` alert | `POST /users` creates an account with the Administrator role, `PUT /users/{id}/roles` adds it, `PUT /ad-group-map` newly maps a group to it, or a directory sign-in's role sync or a directory reconcile pass newly gives an account the role (vault BACKLOG #2610, #3141) | Alert keyed `user:` or `ad-group:`, naming the granting administrator, or `` for a directory sign-in or reconcile pass | | `client` on the `user.created` audit row | An account created through `POST /users` or `POST /users/directory` (BACKLOG #2021) | The creating administrator's address, like the approval rows | | `account_created` notice | An account created through `POST /users` or `POST /users/directory`, neither of which creates one without a notification address (BACKLOG #2018, #2021) | The new account's own notification address | @@ -2425,15 +2425,16 @@ with no page. `account_created` notice. - **A directory grant.** An account can get Administrator because the *directory* added it to a group already mapped to Administrator. The engine sees that grant in one of the two places below, - whichever comes first. A sign-in that sees it first raises `administrator_granted`, unless that - sign-in fails with a server error after the role write; the retry then gains nothing. A reconciler - pass that sees it first raises `ad_session_revoked`, which does not name the role. The later - sign-in then gains nothing, so it raises no `administrator_granted`. An account that never signs - in again and holds no live session raises nothing. So watch membership of the mapped group in the - directory itself. + whichever comes first. Either one raises `administrator_granted`, unless a sign-in that sees it + first fails with a server error after the role write; the retry then gains nothing. An account + that never signs in again and holds no live session raises nothing. So watch membership of the + mapped group in the directory itself. **CORRECTED 2026-10-01:** this read "raises no alert. The engine never sees that grant." **CORRECTED 2026-10-06:** this read "raises no `administrator_granted` alert", and the sign-in case below read "It raises no alert." Vault BACKLOG #2610 added the sign-in alert. + **CORRECTED 2026-10-16:** this read "A reconciler pass that sees it first raises + `ad_session_revoked`, which does not name the role. The later sign-in then gains nothing, so it + raises no `administrator_granted`." Vault BACKLOG #3141 added the reconciler's alert. - **At the account's next sign-in.** The engine writes an `auth.ad_roles_resynced` audit row with the old and new roles. It also sends a best-effort roles-changed notice to that account's own notification address, when the account has one and security notices are set @@ -2447,8 +2448,12 @@ with no page. [directory reconciler](#directory-session-reconciliation--propagating-an-ad-disable-adr-0079-mechanism-2) that completes stores the new roles and revokes the session. It writes `auth.ad_session_revoked` with the new roles, sends the same notice, and raises - `ad_session_revoked` with reason `roles_changed`. The next sign-in then writes no - `auth.ad_roles_resynced` row. That section lists when a pass revokes or alerts nothing. + `ad_session_revoked` with reason `roles_changed`. Right after the role write it raises + `administrator_granted`, keyed `user:`, with `granted_by` set to `` and + `via` set to `directory_reconcile`. That alert clears the account's `no_enabled_administrator` + cooldown, as a sign-in's does. An account that already held Administrator raises nothing. The + next sign-in then writes no `auth.ad_roles_resynced` row and raises no second + `administrator_granted`. That section lists when a pass revokes or alerts nothing. No pass runs at `[auth].ad_session_recheck_seconds = 0`. A pass probes at most `ad_session_recheck_max_users` accounts (200 by default), least recently probed first. When more directory accounts than that hold live sessions, an account can wait several passes. diff --git a/messagefoundry/api/app.py b/messagefoundry/api/app.py index d406f33509..93e51545aa 100644 --- a/messagefoundry/api/app.py +++ b/messagefoundry/api/app.py @@ -220,6 +220,7 @@ from messagefoundry.api.paging import page_total from messagefoundry.api.request_timeout import RequestTimeoutMiddleware from messagefoundry.api.security import ( + DIRECTORY_GRANTED_BY, AuthenticatedBeforeBodyRoute, ExportStopped, alert_sink_for, @@ -399,6 +400,7 @@ from messagefoundry.pipeline.alerts import ( AlertSink, LoggingAlertSink, + administrator_granted_alert, authority_seal_alert, no_enabled_administrator_alert, store_cipher_refusal_forwarder, @@ -10360,6 +10362,11 @@ async def _audit_convergence_reload(outcome: ReloadOutcome) -> None: # Vault BACKLOG #3229: a directory role change that leaves no enabled # Administrator is applied, and raises no_enabled_administrator here. admin_loss_alert=no_enabled_administrator_alert(notifier or LoggingAlertSink()), + # Vault BACKLOG #3141: a reconcile pass that newly grants Administrator raises + # administrator_granted here, as a directory sign-in does in its route. + admin_grant_alert=administrator_granted_alert( + notifier or LoggingAlertSink(), granted_by=DIRECTORY_GRANTED_BY + ), ) # Started only once the service is built, so a constructor refusal starts no task. if security_notifier is not None: diff --git a/messagefoundry/auth/service.py b/messagefoundry/auth/service.py index 7208b9a3e9..3e74856870 100644 --- a/messagefoundry/auth/service.py +++ b/messagefoundry/auth/service.py @@ -2356,6 +2356,7 @@ def __init__( hop_posture: HopPosture | None = None, integrity_alert: Callable[[str], None] | None = None, admin_loss_alert: Callable[[str, str], None] | None = None, + admin_grant_alert: Callable[[str, str], None] | None = None, ) -> None: self._store = store self._settings = settings @@ -2365,6 +2366,9 @@ def __init__( # Vault BACKLOG #3229: called with (username, via) when a directory-driven role change leaves # no enabled Administrator. Injected by the API lifespan, as above. None logs only. self._admin_loss_alert = admin_loss_alert + # Vault BACKLOG #3141: called with (username, via) when a reconcile pass's role write newly + # gives an account Administrator. Injected by the API lifespan, as above. None logs only. + self._admin_grant_alert = admin_grant_alert # Session token-hash prefix -> monotonic time its seal failure was last reported, so one # bad row presented on every request writes one audit row and one alert per interval. self._seal_failures_reported = SealFailureThrottle() @@ -7332,6 +7336,12 @@ async def _apply_reconcile_revocation(self, revocation: reconcile.SessionRevocat await self._report_directory_admin_loss( user, before=user.role_ids, after=revocation.role_ids, via="ad-reconcile" ) + # Vault BACKLOG #3141. The gain side, from the same sealed read and at the same point, + # so a later step's failure cannot lose the page. The next sign-in finds the role + # already written and gains nothing, so this pass is the only chance to raise it. + self._report_directory_admin_grant( + user, before=user.role_ids, after=revocation.role_ids + ) revoked = await self._store.revoke_user_sessions(revocation.user_id) await self._audit( "auth.ad_session_revoked", @@ -7398,6 +7408,32 @@ async def _apply_reconcile_revocation(self, revocation: reconcile.SessionRevocat ) return True + def _report_directory_admin_grant( + self, user: UserRecord, *, before: Iterable[str], after: Iterable[str] + ) -> None: + """Raise ``administrator_granted`` when a reconcile pass's role write newly gave ``user`` + the Administrator role (vault BACKLOG #3141). + + The sign-in rule (vault BACKLOG #2610): only a role this write ADDED pages, so an account + that already held Administrator raises nothing. ``before`` is the account's roles as read + through its seal just before the write, never the plan's, which can be a pass old. Only an + applied revocation reaches here, and only a pass that did not abort applies one. The alert + pops the account's ``no_enabled_administrator`` cooldown in the sink, which is intended: + a later loss of the role pages again. + + Never raises, a defect in the callable included: it runs between the role write and the + session revocation, and a raise would skip the revocation (see + :meth:`_report_directory_admin_loss`).""" + admin = Role.ADMINISTRATOR.value + if admin in set(before) or admin not in set(after): + return + _log.warning("a directory reconcile pass gave an account the Administrator role") + if self._admin_grant_alert is not None: + try: + self._admin_grant_alert(user.username, "directory_reconcile") + except Exception: # noqa: BLE001 -- logged with its traceback; see the docstring + _log.exception("the administrator_granted alert callable raised") + async def _report_directory_admin_loss( self, user: UserRecord, diff --git a/messagefoundry/config/settings.py b/messagefoundry/config/settings.py index 20982ebc92..37ab6f318a 100644 --- a/messagefoundry/config/settings.py +++ b/messagefoundry/config/settings.py @@ -4813,7 +4813,8 @@ class AlertSeverity(str, Enum): # noqa: UP042 "approval_too_early", # BACKLOG #315: a release by an approver account changed after the request, and an # Administrator grant through the console API or, since vault BACKLOG #2610, by a directory - # sign-in's role sync (granted_by ``). + # sign-in's role sync, and since vault BACKLOG #3141 by a directory reconcile pass (both + # granted_by ``). "approval_approver_provenance", "administrator_granted", # vault BACKLOG #3229: a directory-driven role change left no enabled Administrator. Keyed diff --git a/messagefoundry/pipeline/alert_sinks.py b/messagefoundry/pipeline/alert_sinks.py index cba4f01a42..e7d19f3aad 100644 --- a/messagefoundry/pipeline/alert_sinks.py +++ b/messagefoundry/pipeline/alert_sinks.py @@ -1153,12 +1153,14 @@ def administrator_granted(self, name: str, *, via: str, granted_by: str) -> None # BACKLOG #315: the Administrator role was granted. `user:` or `ad-group:` # stands in for "connection" (the key's grammar is on AlertSink). `granted_by` is the granting # administrator's operator account name, or the fixed marker `` when a directory - # sign-in's role sync made the grant (vault BACKLOG #2610). Never message content. + # sign-in's role sync or a reconcile pass made the grant (vault BACKLOG #2610, #3141). Never + # message content. # Vault BACKLOG #3229: every administrator_granted call pops the _last_sent key built from # the no_enabled_administrator alert type and the granted target, so a loss after an # administrator_granted re-grant is not held back by the earlier loss's cooldown. A - # reconciler re-grant calls no administrator_granted, so it pops nothing. The pop builds that key with its own - # f-string in `_emit`'s key format; tests/test_directory_last_admin_alert.py pins the match. + # reconciler re-grant raises administrator_granted too (vault BACKLOG #3141), so it pops the + # key the same way. The pop builds that key with its own f-string in `_emit`'s key format; + # tests/test_directory_last_admin_alert.py pins the match. # Only a `user:` target can match a loss key, so an `ad-group:` grant pops nothing. _last_sent is # per process, so this clears the cooldown in this process only. _occurrences and _suspended # are kept. Unlike crl_reload_applied, this resolves no open instance. diff --git a/messagefoundry/pipeline/alerts.py b/messagefoundry/pipeline/alerts.py index e1223e8337..29a07e6baf 100644 --- a/messagefoundry/pipeline/alerts.py +++ b/messagefoundry/pipeline/alerts.py @@ -505,12 +505,15 @@ def administrator_granted(self, name: str, *, via: str, granted_by: str) -> None ``ad_group_map`` with ``name`` = ``ad-group:`` when a group newly maps to the role. A directory sign-in whose role sync newly grants the role (vault BACKLOG #2610) raises it with ``name`` = ``user:`` and ``via`` = ``directory_sign_in_negotiate``, - ``directory_sign_in_sso`` or ``directory_sign_in_oidc``, naming the route. + ``directory_sign_in_sso`` or ``directory_sign_in_oidc``, naming the route. A directory + reconcile pass whose role re-diff newly grants it (vault BACKLOG #3141) raises it with + ``name`` = ``user:`` and ``via`` = ``directory_reconcile``. Both keys are outside the connection-name grammar for the same reason as :meth:`approval_approver_provenance`. ``granted_by`` is the acting administrator's username, - or ```` on a directory sign-in, where no administrator acted. - No PHI. Emitted by the API's user-administration and directory sign-in routes, never from - ``auth/``.""" + or ```` on a directory sign-in or reconcile pass, where no administrator acted. + No PHI. Emitted by the API's user-administration and directory sign-in routes, and for the + reconciler through the callable :func:`administrator_granted_alert` builds, which the API + lifespan hands to the auth service, so ``auth/`` never imports a sink.""" ... def no_enabled_administrator(self, name: str, *, via: str) -> None: @@ -1293,6 +1296,23 @@ def _raise(username: str, via: str) -> None: return _raise +def administrator_granted_alert(sink: AlertSink, *, granted_by: str) -> Callable[[str, str], None]: + """An alert callable for :class:`~messagefoundry.auth.service.AuthService`: each call, with a + username and a ``via``, raises :meth:`AlertSink.administrator_granted` keyed ``user:`` + and naming ``granted_by`` (vault BACKLOG #3141). The API passes its directory marker, because + the service calls this only for a grant the directory made. + + Never raises: an alert failure must not change what a reconcile pass does.""" + + def _raise(username: str, via: str) -> None: + try: + sink.administrator_granted(f"user:{username}", via=via, granted_by=granted_by) + except Exception: # noqa: BLE001 — an alert-sink failure must never break a request + log.warning("the administrator_granted alert could not be delivered", exc_info=True) + + return _raise + + def alert_store_cipher_refusal(sink: AlertSink, table: str, column: str) -> None: """Raise the refusal alert for an unmarked value in ``table.column``: ``upload-cipher`` for the uploaded-file store, ``store-cipher`` for everything else. diff --git a/packaging/messagefoundry-webconsole/tests/test_ui_directory_admin_alert.py b/packaging/messagefoundry-webconsole/tests/test_ui_directory_admin_alert.py index f21e51f616..368b8e3c98 100644 --- a/packaging/messagefoundry-webconsole/tests/test_ui_directory_admin_alert.py +++ b/packaging/messagefoundry-webconsole/tests/test_ui_directory_admin_alert.py @@ -105,6 +105,13 @@ async def _service(engine: Engine, *, held: Role | None, bind: bool) -> AuthServ return service +async def _jdoe_holds_administrator(engine: Engine) -> bool: + """Whether the role write landed: a refused-after-sync test proves nothing without it.""" + jdoe = await engine.store.get_user_by_username("jdoe") + assert jdoe is not None + return Role.ADMINISTRATOR.value in await engine.store.get_user_role_ids(jdoe.id) + + def _client(engine: Engine, service: AuthService, sink: _Sink) -> httpx.AsyncClient: app = create_app(engine, auth=service, serve_ui=True, public_origin="https://ops.example") app.state.notifier = sink @@ -208,6 +215,7 @@ async def test_ui_sso_refused_after_the_role_write_still_raises_the_alert( page.""" sink = await _sso(engine, monkeypatch, held=Role.OPERATOR, bind_lands_mid_login=True) assert sink.events == [_grant("directory_sign_in_sso")] + assert await _jdoe_holds_administrator(engine) async def test_ui_oidc_that_newly_grants_administrator_raises_one_alert( @@ -231,3 +239,4 @@ async def test_ui_oidc_refused_after_the_role_write_still_raises_the_alert( failed, and a later sign-in gains nothing, so this refusal is the one chance to page.""" sink = await _oidc(engine, monkeypatch, held=None, bind_lands_mid_login=True) assert sink.events == [_grant("directory_sign_in_oidc")] + assert await _jdoe_holds_administrator(engine) diff --git a/tests/test_directory_last_admin_alert.py b/tests/test_directory_last_admin_alert.py index e07178b991..424d7c3cd2 100644 --- a/tests/test_directory_last_admin_alert.py +++ b/tests/test_directory_last_admin_alert.py @@ -26,6 +26,7 @@ import pytest +from messagefoundry.api.security import DIRECTORY_GRANTED_BY from messagefoundry.auth.ldap import AdPrincipal, DirectoryAnswer, DirectoryProbe from messagefoundry.auth.permissions import Role from messagefoundry.auth.service import NO_ENABLED_ADMINISTRATOR_ACTION, AuthService @@ -34,7 +35,11 @@ _ALERT_EVENT_TYPES, AuthSettings, ) -from messagefoundry.pipeline.alerts import LoggingAlertSink, no_enabled_administrator_alert +from messagefoundry.pipeline.alerts import ( + LoggingAlertSink, + administrator_granted_alert, + no_enabled_administrator_alert, +) from messagefoundry.store.store import MessageStore ADMINS = "cn=mf-admins,dc=test,dc=invalid" @@ -75,10 +80,12 @@ def probe_principal(self, username: str, *, object_id: str | None = None) -> Dir class _Rig: - def __init__(self, store: MessageStore) -> None: + def __init__(self, store: MessageStore, **overrides: Any) -> None: self.store = store self.ldap = _FakeLdap() self.alerts: list[tuple[str, str]] = [] + #: What the vault BACKLOG #3141 grant callable was handed, ``(username, via)``. + self.grants: list[tuple[str, str]] = [] settings = AuthSettings( ad_enabled=True, ad_server="ldaps://dc.test.invalid", @@ -86,12 +93,14 @@ def __init__(self, store: MessageStore) -> None: ad_bind_dn="CN=svc-mefor,OU=Service,DC=test,DC=invalid", ad_bind_password="synthetic", ad_session_recheck_seconds=60, + **overrides, ) self.service = AuthService( store, settings, ldap=self.ldap, # type: ignore[arg-type] admin_loss_alert=lambda username, via: self.alerts.append((username, via)), + admin_grant_alert=lambda username, via: self.grants.append((username, via)), ) async def start(self) -> None: @@ -322,3 +331,155 @@ async def test_a_loss_after_a_regrant_is_not_held_back_by_the_cooldown() -> None losses = [e["connection"] for e in transport.events if e["type"] == "no_enabled_administrator"] assert losses.count("user:jdoe") == 2 assert losses.count("user:asmith") == 1 + + +# --- vault BACKLOG #3141: a reconcile pass that newly grants Administrator pages ---------------- +# +# The reconciler's role re-diff can give an account Administrator. It raised only +# ad_session_revoked, and the account's next sign-in then found the role already written and +# gained nothing, so no administrator_granted ever fired. The pass now raises it right after the +# role write, judged on the roles read through the account's seal at apply time. + +#: The ``via`` a reconcile pass's grant reports. +RECONCILE_VIA = "directory_reconcile" + + +async def test_a_reconcile_pass_that_newly_grants_administrator_pages_once(rig: _Rig) -> None: + """RED before #3141: the pass raised no grant, and the next sign-in gains nothing.""" + rig.ldap.present["jdoe"] = _principal("jdoe", OPERATORS) + await rig.sign_in("jdoe") + + await rig.directory_change("jdoe", ADMINS, via="ad-reconcile") + + assert await rig.roles("jdoe") == [Role.ADMINISTRATOR.value] + assert rig.grants == [("jdoe", RECONCILE_VIA)] + assert rig.alerts == [] + # The pass after it finds nothing to change, so it pages nothing more. + assert (await rig.service.reconcile_directory_sessions()).revocations == () + assert rig.grants == [("jdoe", RECONCILE_VIA)] + + +async def test_a_reconcile_change_to_an_account_that_held_administrator_is_quiet( + rig: _Rig, +) -> None: + """Negative control: Administrator held before the write, so nothing was granted.""" + rig.ldap.present["jdoe"] = _principal("jdoe", ADMINS) + await rig.sign_in("jdoe") + + await rig.directory_change("jdoe", ADMINS, OPERATORS, via="ad-reconcile") + + assert sorted(await rig.roles("jdoe")) == sorted( + [Role.ADMINISTRATOR.value, Role.OPERATOR.value] + ) + assert rig.grants == [] + + +async def test_a_reconcile_gain_of_another_role_only_is_quiet(rig: _Rig) -> None: + """Negative control: a role was gained, but not Administrator.""" + rig.ldap.present["jdoe"] = _principal("jdoe") + await rig.sign_in("jdoe") + + await rig.directory_change("jdoe", OPERATORS, via="ad-reconcile") + + assert await rig.roles("jdoe") == [Role.OPERATOR.value] + assert rig.grants == [] + + +async def test_the_grant_is_judged_on_the_roles_read_at_apply_time_not_the_plans( + rig: _Rig, monkeypatch: pytest.MonkeyPatch +) -> None: + """A sign-in that lands between the pass's read and its write has already written + Administrator, and paged for it. The write then adds nothing, so the pass must not page again. + RED when the before-set is taken from the plan, which read the roles before that sign-in.""" + rig.ldap.present["jdoe"] = _principal("jdoe", OPERATORS) + await rig.sign_in("jdoe") + rig.ldap.present["jdoe"] = _principal("jdoe", ADMINS) + real = rig.service._apply_reconcile_revocation + + async def a_sign_in_lands_first(revocation: Any) -> bool: + await rig.store.set_user_roles( + revocation.user_id, [Role.ADMINISTRATOR.value], assigned_by="ad-sync" + ) + return await real(revocation) + + monkeypatch.setattr(rig.service, "_apply_reconcile_revocation", a_sign_in_lands_first) + + plan = await rig.service.reconcile_directory_sessions() + + assert [(r.username, r.reason) for r in plan.revocations] == [("jdoe", "roles_changed")] + assert rig.grants == [] + + +async def test_a_pass_the_breaker_aborts_grants_nothing_and_pages_nothing() -> None: + """An aborted pass writes no roles, so there is no grant to page (ADR 0079 mechanism 2).""" + store = await MessageStore.open(":memory:") + try: + # One revocation of one probed account trips a breaker this tight. + rig = _Rig(store, ad_session_revoke_max=0, ad_session_revoke_max_fraction=0.01) + await rig.start() + rig.ldap.present["jdoe"] = _principal("jdoe", OPERATORS) + await rig.sign_in("jdoe") + rig.ldap.present["jdoe"] = _principal("jdoe", ADMINS) + + plan = await rig.service.reconcile_directory_sessions() + + assert plan.aborted == "mass_revoke_breaker" + assert await rig.roles("jdoe") == [Role.OPERATOR.value] + assert rig.grants == [] + finally: + await store.close() + + +async def test_a_raising_grant_callable_still_ends_the_sessions(rig: _Rig) -> None: + """Only the production wrapper promises not to raise, so the service contains a raise: the + session revocation after the role write still runs.""" + + def raising(username: str, via: str) -> None: + raise RuntimeError("synthetic: the alert path is down") + + rig.service._admin_grant_alert = raising + rig.ldap.present["jdoe"] = _principal("jdoe", OPERATORS) + await rig.sign_in("jdoe") + + await rig.directory_change("jdoe", ADMINS, via="ad-reconcile") + + assert await rig.roles("jdoe") == [Role.ADMINISTRATOR.value] + actions = [r["action"] for r in await rig.store.list_audit()] + assert "auth.ad_session_revoked" in actions + + +def test_the_grant_callable_keys_on_the_account_and_never_raises() -> None: + raised: list[tuple[str, str, str]] = [] + + class _Sink(LoggingAlertSink): + def administrator_granted(self, name: str, *, via: str, granted_by: str) -> None: + raised.append((name, via, granted_by)) + + administrator_granted_alert(_Sink(), granted_by=DIRECTORY_GRANTED_BY)("jdoe", RECONCILE_VIA) + assert raised == [("user:jdoe", RECONCILE_VIA, DIRECTORY_GRANTED_BY)] + + class _Broken(LoggingAlertSink): + def administrator_granted(self, name: str, *, via: str, granted_by: str) -> None: + raise RuntimeError("synthetic: the sink is down") + + administrator_granted_alert(_Broken(), granted_by=DIRECTORY_GRANTED_BY)("jdoe", RECONCILE_VIA) + + +async def test_a_loss_after_a_reconciler_regrant_is_not_held_back_by_the_cooldown() -> None: + """The #3229 cooldown pop, through the reconciler's own grant callable: the second loss is + sent.""" + from messagefoundry.pipeline.alert_sinks import NotifierAlertSink + from tests.test_alert_sinks import _drain, _RecordingTransport + + transport = _RecordingTransport("webhook") + sink = NotifierAlertSink([transport]) # the default cooldown, far longer than this test + lose = no_enabled_administrator_alert(sink) + regrant = administrator_granted_alert(sink, granted_by=DIRECTORY_GRANTED_BY) + + lose("jdoe", "ad-reconcile") + regrant("jdoe", RECONCILE_VIA) + lose("jdoe", "ad-reconcile") + await _drain(sink) + + losses = [e["connection"] for e in transport.events if e["type"] == "no_enabled_administrator"] + assert losses.count("user:jdoe") == 2 diff --git a/tests/test_directory_sign_in_admin_alert.py b/tests/test_directory_sign_in_admin_alert.py index 9c306a5500..38a935db1e 100644 --- a/tests/test_directory_sign_in_admin_alert.py +++ b/tests/test_directory_sign_in_admin_alert.py @@ -245,4 +245,7 @@ async def test_a_sink_that_raises_does_not_break_the_sign_in( r = await _negotiate(c) assert r.status_code == 200, r.text assert sink.events == [_GRANT] - assert any("failed to emit" in rec.getMessage() for rec in caplog.records) + assert any( + rec.name == "messagefoundry.api.security" and "failed to emit" in rec.getMessage() + for rec in caplog.records + ) From f9df1af50d6e8b922b7e80bbdc3599915aca27ad Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Fri, 16 Oct 2026 22:20:45 -0500 Subject: [PATCH 23/57] UNKEYED reconcile follow-ups: referrals count the probes asked, an outage still strikes (BACKLOG #3170) - Item 1: the referral's latched message, its auth.ad_reconcile_referred row (key renamed probed -> asked) and its ad_reconcile_aborted alert count plan.asked. The breaker-trip alert keeps plan.probed, its judged denominator. AlertSink docstring says which count is which. - Item 2: DECLINED, one line at the breaker call in plan_pass: exempting UNKEYED revocations changes what the mass-revoke breaker counts. - Item 3: plan_pass's two early returns (outage, referrals only) carry the UNKEYED rows' strikes, and only those: no outcomes, so the hold's record and judged_nothing are unchanged and the latch carries over. - Item 4: changelog.d/2434 names the auth.ad_reconcile_skipped probed -> asked rename. New changelog.d/3170. - Item 5: docs/PHI.md ad_session_revoked reasons gain directory_object_id_missing (as "at least"); the ad_reconcile_aborted count is described per source. - Item 6: the comment now in _directory_presence no longer says the reconciler probes an id-less row by name. - Item 7: the sign-in, step-up, re-proof, presence and bind refusals read _directory_key/_holds_directory_key; reason strings unchanged. _directory_key returns the id so mypy still narrows where it is sent. - Tests: an UNKEYED row is struck on a pass that judged nothing (three shapes, with an ordinary-pass control); a referral is counted over the probes asked, end to end and through _alert_reconcile_plan. --- changelog.d/2434.security.md | 4 +- changelog.d/3170.security.md | 9 ++++ docs/PHI.md | 2 +- docs/SECURITY.md | 5 +- messagefoundry/api/app.py | 4 +- messagefoundry/auth/reconcile.py | 20 ++++++- messagefoundry/auth/service.py | 57 ++++++++++++-------- messagefoundry/pipeline/alerts.py | 9 ++-- tests/test_ad_session_reconcile.py | 85 +++++++++++++++++++++++++++++- 9 files changed, 161 insertions(+), 34 deletions(-) create mode 100644 changelog.d/3170.security.md diff --git a/changelog.d/2434.security.md b/changelog.d/2434.security.md index 0a3e067971..fc09433c7d 100644 --- a/changelog.d/2434.security.md +++ b/changelog.d/2434.security.md @@ -3,7 +3,9 @@ directory row with no `objectGUID` and no federated binding is now judged without a lookup: it gets no roles written, and its sessions end at the strike threshold with the reason `directory_object_id_missing`, as an absent account's do. It never moves the ADR 0195 hold. No - sign-in or step-up admits such a row, so a session it holds is anomalous. + sign-in or step-up admits such a row, so a session it holds is anomalous. The count on the + outage's `auth.ad_reconcile_skipped` audit row is now the `asked` field, the lookups the pass + sent, and it leaves such a row out. It was the `probed` field. - **An empty password on the AD step-up re-bind is no longer counted as a wrong guess.** It is refused before the directory is asked, charged to neither the account lockout nor the session's re-proof budget, and audited on `auth.reauth` as `empty_password`. The caller sees a refused diff --git a/changelog.d/3170.security.md b/changelog.d/3170.security.md new file mode 100644 index 0000000000..11e238675a --- /dev/null +++ b/changelog.d/3170.security.md @@ -0,0 +1,9 @@ +- **The session reconciler counts a referral over the lookups it sent, and strikes an id-less row + on every pass.** A directory row with no `objectGUID` is judged without a lookup, so no referral + can answer it. The referral's `auth.ad_reconcile_referred` audit row now carries the `asked` + field in place of `probed`, and that row, the referral's latched message and its + `ad_reconcile_aborted` alert count only the lookups the pass sent. The breaker's own alert still + counts every account it judged. Such a row is also struck now on a pass that is a directory + outage or got only referrals, as on any other pass, so its sessions end at the strike threshold + however the directory answered. Its revocation still counts toward the mass-revoke breaker, as + before. (`vault BACKLOG #3170`) diff --git a/docs/PHI.md b/docs/PHI.md index 7ee01635a1..1ac085e254 100644 --- a/docs/PHI.md +++ b/docs/PHI.md @@ -1808,7 +1808,7 @@ with materially different PHI profiles, so they get their own rows; stream 4 is | **5. `audit_log` table** (SQLite, Postgres, SQL Server) | who / what / **where-from** / when of auth + PHI *access* and admin actions — plus, while `[security].audit_all_authorization_decisions` is on (**default `true`** since BACKLOG #1277, 2026-09-02, which reversed the ADR 0118 §5 `false`; the internal field it desugars to is `audit_all_authz`, whose old `[diagnostics]` TOML spelling is **refused at load** — ADR 0118), an `authz` row for **every** authorization decision including successes. That is the shipped volume of this stream, not an opt-in addition to it: one row per authenticated request on each `require()`-gated route, and this row's retention cell records that **nothing prunes the table** — `actor`, `action`, `channel_id`, `client`, `detail`, `row_hash` | JSON `detail`; **tamper-evident hash chain** over `prev_hash` + the row (the `client` address is **inside** the chained payload — ADR 0150) | the store database | HIPAA §164.312(b) audit controls; incident response; `verify_audit_chain` integrity checks | `GET /audit` requires **`audit:read`**; `GET /audit/export` requires the separate **`audit:export`** and streams CSV with formula-injection neutralisation, recording its own `audit.export` row *before* streaming; `GET /me/security-events` is a per-user view of the same table | **`[retention].audit_days` is reserved and NOT enforced — keep-forever by design** (the audit-retention requirement, ~6 years — **not** chain-breakage; [§8](#8-retention--purge) states the position and cites its source of record) | `detail` is stored **in the clear** (it is not a cipher-covered column): its protection is that writers store at least filter shapes, counts, ids, keyed digests, bounded engine-authored text and client addresses, the `audit_log.detail` row above says which — never bodies or credentials — plus the store ACL and the volume layer. *Corrected 2026-10-16 (vault BACKLOG #3145):* this said writers "only ever store filter shapes, counts and ids" | | **6. `message_events` table** | the per-message disposition timeline — the **complete** vocabulary is `received`, `routed`, `unrouted`, `filtered`, `transformed`, `delivered`, `failed`, `dead`, `error`, `replayed`, `resent`, `reingressed`, `passthrough`, `passthrough_dropped`, `cancelled`, `edit_resend`, `edit_resubmit`, `viewed`, `not_deployed`, the ADR 0154 synchronous-reply pair `reply_returned` / `reply_timeout` (names, counts and `waited_ms` only — **never** a fragment of the partner's reply body), and `reply_dropped`, a delivery whose 2xx reply body was over the byte bound and dropped (the status and the bound only, vault BACKLOG #3040) (CI asserts this list against the engine's own `MESSAGE_EVENT_KINDS`). `[diagnostics].message_events` can thin the set, but never below the compliance floor `viewed` / `dead` / `error` / `failed` / `not_deployed` / `reply_timeout` / `reply_dropped` | rows: `message_id`, `ts`, `event`, `destination`, `detail` | the store database | operator timeline on the message-detail view; the `viewed` row is the HIPAA PHI-access record | `GET /messages/{id}` under **`messages:view_raw`** + `require_phi_read`; the read itself writes a `viewed` event **and** a `message_view` audit row | no dedicated window — `purge_message_bodies` sets `message_events.detail` to `NULL` in the same transaction that blanks the body, so it inherits `[retention].messages_days` | `detail` is `safe_text()`-scrubbed **then** cipher-encrypted (AAD `("message_events","detail",message_id,ts,event)`). Verbosity gate `[diagnostics].message_events` = `all` (default) / `errors` / `off`, with a **compliance floor that can never be thinned**: `viewed`, `dead`, `error`, `failed`, `not_deployed`, `reply_timeout`, `reply_dropped` are retained at every level (`reply_timeout` is the one row that explains a "we called you and got a 504" complaint, so an instance that thinned its logs would lose exactly the record it is later asked for; `reply_dropped` is the one row that answers "you ignored our reply", for the same reason) | | **7. `connection_event` table — DEFAULT ON** (`[diagnostics].connection_events = true`) | transport/lifecycle events per connection: `established`, `closed` (reason `eof`, `idle_timeout` or `frame_deadline` — a fixed vocabulary chosen by the listener, never free text), `idle_timeout`, `at_capacity`, `peer_not_allowlisted`, `frame_oversize`, `framing_error`, `peer_reset`, the MLLP listener's `handler_error` (BACKLOG #1619 -- the inbound handler faulted on a frame the listener read cleanly, such as a store outage at the ingress commit; the reason is scrubbed at the source like every other), the MLLP listener's `tls_handshake_failed` (vault BACKLOG #2613 -- a TLS handshake on a listener that runs its own handshakes (the standard asyncio loop, not uvloop) failed; the first at once, then one per listener per minute carrying the count since the last, and the reason is a count and an exception class name, never the peer's bytes), the DATABASE poll source's `row_undecodable` (BACKLOG #1662 — a polled row the source cannot turn into a body; the reason is scrubbed at the source and names the exception type or the operator's own configured column, never a row value), the FILE poll source's quarantine kinds `file_oversize` / `file_decompress_failed` / `file_content_mismatch` / `file_scan_rejected` (BACKLOG #1621 — a drop moved to `.error`; the reason is a size, a content type or a scrubbed codec or scanner message, never the file name), the inbound-HTTP intake-auth refusals `intake_auth_failed` / `auth_subject_denied` / `auth_rate_limited` (ADR 0154 D6 — peer address and mode only; **never** the credential, a prefix of it, or its length. Each of these also writes a tamper-evident audit-log row — the copy that survives an operator turning this diagnostics stream off), plus the runner's `connection_lost` / `connection_restored`. That is the whole vocabulary, asserted in CI against the literal emit call sites in `transports/` and the pipeline runner **and** cross-checked against the console's own filter tuple. The MLLP, raw-TCP, X12 and HTTP listeners emit these; the **DATABASE poll source** emits `row_undecodable` and nothing else (it dials out, so there is no lifecycle to report and its rows carry a **NULL peer column**); the **FILE poll source** emits only its four quarantine kinds, also with a NULL peer column; the **DICOM inbound C-STORE SCP** emits none — the runner injects the sink onto **every** source (`wiring_runner.py`, over the base-class `on_connection_event` field), so that connector *has* the wiring and simply never calls it — so this stream covers those four listeners plus those poll-source events and the runner's outbound-lane transitions — not literally every connection. A DICOM association's connects and refusals are therefore **absent** from this stream. The **`ISA`/`IEA`-framed X12 inbound** was equally absent until BACKLOG #1665 and is not any more: it emits the same seven kinds as the raw-TCP listener | rows: `ts`, `connection`, `transport`, `direction`, `kind`, `peer_host`, `message_id` (correlation hint), `reason` | the store database, **all three backends** | Corepoint-style transport diagnostics — "did the sender connect, and why did it drop" | `GET /events` and `GET /connections/{name}/events` under **`monitoring:read`** (**not** a PHI permission) with per-channel RBAC — an out-of-scope `connection=` is 403'd *and* audited — server-clamped to ≤1000 rows. `reason` alone is gated on `messages:view_summary` and masked as `****` until the audited per-event `reveal=` act (BACKLOG #2443; the `connection_event.reason` row of §3's PL-2 block says how) | `[retention].connection_event_retention_hours` (its own **hours** window); 0 inherits `[retention].messages_days`; both 0 = keep forever. Plain age `DELETE` (metadata-only) | **`reason` is free text that can carry sensitive fragments.** Defended twice — `safe_exc()` at the source, `safe_text(reason, limit=200)` at the store (the bound §3's PL-2 block states) — then cipher-encrypted (AAD `("connection_event","reason",connection,ts,kind)`). Every other column is config metadata; the table is documented **metadata-only** — never a frame, body or HL7 field value. Writes are a pure side observer: a bounded in-memory queue drained by a background task outside any handoff transaction, so a flood can never block a listener or pin a message disposition | -| **8. `alert_instance` table — default on wherever an `[alerts]` notifier exists** | resolvable operator alerts: `connection_stopped`, `queue_buildup`, `lane_stuck`, `message_stall`, `saturation`, `connection_error`, `storage_threshold`, `cert_expiry`, `secret_rotation`, `initial_credential_expiring` (an UNCLAIMED admin-issued temporary password nearing the instant the login gate stops accepting it -- ASVS 6.4.5, BACKLOG #1141; keyed on `user:`, and the payload is the ISO deadline plus whole hours remaining, repeated in the reason column, never the password), `integrity_drift`, `update_available`, `backup_failed`, `store_privilege_warning` (the store privilege preflight found the store principal over-granted or could not read it -- BACKLOG #305; the payload is the finding, a count and the preflight's summary line, which names the login, the database and role names, never a password or message content), `leadership_acquired`, `dr_activated`, `log_write_failed` (an application-log sink was rolled after a write failure, or is UNWRITABLE and this process's connections were stopped -- BACKLOG #122, ADR 0162; the payload is the sink LABEL, the stage, a `safe_exc` reason and a count of connections stopped, never the record whose write failed), `log_forward_failed` (the off-box log forwarder is absent, losing records or not sending -- BACKLOG #2612; keyed on `forwarder:`, with `@node:` or `@shard:` appended when several engine processes share the store, and the payload is that key, fixed reason words and a count, never a record, the collector's address or an error text; an unpinned cluster node's `node_id` holds this engine's own host name and process id, so the key carries them), `gcm_invocations` (the per-key AES-GCM invocation bound crossing its 2^31 soft warn — ASVS 11.3.4; its payload carries a one-way `key_id` fingerprint plus counters, never key bytes), `approval_stale_requester` (a dual-control release refused because the requester no longer holds the authority the operation needs -- ASVS 8.3.2, BACKLOG #289; keyed on the approval id, and the payload is that id, the operation key and a closed-set reason slug, never the requester's name or the captured params), `approval_too_early` (a dual-control release refused because it arrived before `[approvals].min_dwell_seconds` -- ASVS 2.4.2, BACKLOG #287; keyed on `approval:`, and the payload is that key, the operation key and a fixed reason string, never the approver's or requester's name or the captured params), `audit_write_failed` (the approval gate could not write one of its audit rows -- vault BACKLOG #2255; keyed on `approval:`, or on `config_reload:inline` when an ungated config reload could not write its `config_reload_attempted` row -- vault BACKLOG #2254; the payload is that key and the lost row's audit action name, never the row's detail, a username or the captured params; the ERROR log line beside it holds the detail on some paths and only the exception on others), `ad_reconcile_aborted` (a directory reconciler pass left signed-in accounts unrevoked and needs an operator: the mass-revoke breaker tripped and nothing was revoked, or one or more probes were answered with an LDAP referral and those accounts were left unjudged while the rest of the pass was applied, BACKLOG #2538; the payload is a fixed source label, `directory-reconciler` for the breaker and `directory-reconciler-referral` for a referral so each is its own instance, the closed-set reason slug `mass_revoke_breaker` or `directory_referral`, the probed count and the latched operator explanation, never a username), `ad_reconcile_held` (the reconciler is holding accounts whose `userAccountControl` it cannot read, ADR 0195; the payload is the same fixed source label, a closed-set reason slug, the count of undetermined accounts and the latched operator explanation, never a username), `ad_session_revoked` (the reconciler revoked a directory principal's sessions; keyed on that operator account's username, with a `directory_absent`, `directory_disabled`, `directory_undetermined`, `roles_changed` or `scope_changed` reason), `approval_approver_provenance` (a dual-control release, or the resolution of an interrupted one, went ahead with an approver or resolver account created, re-passworded or TOTP-enrolled after the request -- BACKLOG #315, #2125; keyed on `approval:`, or `approval::resolve` for a resolution, and the payload is the operation key and the closed-set change slugs, never a username or the captured params), `administrator_granted` (the built-in Administrator role was granted by account creation, a role change, a directory group newly mapped to it -- BACKLOG #315 --, a directory sign-in whose role sync newly gave the account the role -- vault BACKLOG #2610 -- or a directory reconcile pass whose role re-diff did -- vault BACKLOG #3141; keyed on `user:` or `ad-group:`, with the grant route and the granting administrator's username, or `` for a directory sign-in or reconcile pass), `no_enabled_administrator` (a directory sign-in's role sync or the session reconciler took Administrator from its last enabled holder -- vault BACKLOG #3229; keyed on `user:`, and the payload is `via`, `ad-sync` or `ad-reconcile`, and a fixed reason sentence, never message content), `intake_paused` (the engine paused intake because the staged backlog went over `[inbound].max_staged_depth` or the SQLite volume fell below `[retention].min_free_disk_mb` -- BACKLOG #290; keyed on `intake:staged_depth` or `intake:disk_floor`, with `@node:` or `@shard:` appended when several engine processes share the store (BACKLOG #2272), and the payload is the reason, a count or a MiB figure, its limit and the store backend's name, never message content), `config_changed` (a start loaded a config whose fingerprint differs from the store's baseline -- vault BACKLOG #2597; keyed on `config:`, and the payload is the two digests, the node and engine shard labels, and the action, actor username and time of the row the older digest came from, never the config directory, a git commit or message content), `crl_reload_refused` (the CRL reload refused a replaced CRL file, so running TLS contexts keep the copy they hold -- vault BACKLOG #2904; keyed on `crl_reload:`, with `@node:` or `@shard:` appended when several engine processes share the store, and the payload is that key, the file's path, a short class token for the remedy, a count of contexts, the engine's own fixed remedy sentence and, when a start would refuse the file too, a `start_refuses` flag, never the refusal's reason, which quotes CRL issuer names and dates; that reason is in the application log, and in the `detail` of the CRL expiry alert once the held copy nears its `nextUpdate`), `remote_debug_refused` (the engine's remote-debugging guard refused scripts that another process injected into the engine process -- vault BACKLOG #2742; keyed on `remote_debug:refused`, with `@node:` or `@shard:` appended when several engine processes share the store, and the payload is that key, the count of scripts refused since the alert was last raised, the count since the process started and a fixed sentence built from the two, never a script's file name, which the injecting process chooses; that name is in the application log, in the guard's WARNING line), and the six security signals of the audit-stream rule layer -- vault BACKLOG #2613: `signin_failure_burst` (keyed on `signin:`, never the typed username), `access_denied_burst`, `body_view_burst` and `bulk_export` (each keyed on `account:`), `log_level_debug` (keyed on `logging:debug`) and `posture_loosened` (keyed on `posture:start`); the payload is a count and a fixed sentence holding a level name, switch names or a username, never a message body, a message id or an audit row's detail. The reachable **inverse** signals — `connection_restored`, `leadership_lost`, `dr_released`, `store_privilege_clean`, `intake_resumed`, `crl_reload_applied` (it resolves `crl_reload_refused` for the same file on the same engine process, once the reload's record no longer holds the refusal, and when the reload stops -- vault BACKLOG #2904), `ad_reconcile_breaker_cleared`, `ad_reconcile_hold_released` (the last two resolve `ad_reconcile_aborted`, under either source, and `ad_reconcile_held`; when they do, and where they never do, is stated once in [CONFIGURATION.md](CONFIGURATION.md#when-the-reconcilers-two-alerts-resolve-themselves) -- BACKLOG #2136) — are never rows here: `_record_state` routes an inverse through `_AUTO_RESOLVE` to `resolve_alert_instances_for`, never to `upsert_alert_instance`. (One more mapped key, `connection_started`, is emitted by no code path today.) | rows: `event_type`, `connection`, `severity`, `status`, `first_seen`, `last_seen`, `count`, `reason`, `acked_by`, `acked_at`, `resolved_at`, `suspended_until`, `escalation_tier` | the store database, **all three backends** | the operator alert list — acknowledge / resolve / suspend. Durable state is recorded **before** any suppression or throttle return, so a muted alert still leaves a record | `GET /alerts/active` under **`monitoring:diagnose`** (**not** a PHI permission) with the same per-channel scope, and `reason` alone gated on `messages:view_summary` and masked as `****` until the audited per-alert `reveal=` act (BACKLOG #2443; the `alert_instance.reason` row of §3's PL-2 block says how); ack/resolve/suspend/**resume** are POSTs on the same tier, and the separate read-only `GET /alerts/rules` view sits on its own gate | shares the connection-event window; **only RESOLVED instances are DELETEd**, by `resolved_at` — an open or acknowledged condition is never aged out from under an operator | **`reason` is free text** taken from the event's `detail`/`reason`: `safe_text(reason, limit=200)` (the bound §3's PL-2 block states) then cipher-encrypted (AAD `("alert_instance","reason",event_type,connection)` — the de-dup grain, so one AAD covers both the INSERT and the re-fire UPDATE). | +| **8. `alert_instance` table — default on wherever an `[alerts]` notifier exists** | resolvable operator alerts: `connection_stopped`, `queue_buildup`, `lane_stuck`, `message_stall`, `saturation`, `connection_error`, `storage_threshold`, `cert_expiry`, `secret_rotation`, `initial_credential_expiring` (an UNCLAIMED admin-issued temporary password nearing the instant the login gate stops accepting it -- ASVS 6.4.5, BACKLOG #1141; keyed on `user:`, and the payload is the ISO deadline plus whole hours remaining, repeated in the reason column, never the password), `integrity_drift`, `update_available`, `backup_failed`, `store_privilege_warning` (the store privilege preflight found the store principal over-granted or could not read it -- BACKLOG #305; the payload is the finding, a count and the preflight's summary line, which names the login, the database and role names, never a password or message content), `leadership_acquired`, `dr_activated`, `log_write_failed` (an application-log sink was rolled after a write failure, or is UNWRITABLE and this process's connections were stopped -- BACKLOG #122, ADR 0162; the payload is the sink LABEL, the stage, a `safe_exc` reason and a count of connections stopped, never the record whose write failed), `log_forward_failed` (the off-box log forwarder is absent, losing records or not sending -- BACKLOG #2612; keyed on `forwarder:`, with `@node:` or `@shard:` appended when several engine processes share the store, and the payload is that key, fixed reason words and a count, never a record, the collector's address or an error text; an unpinned cluster node's `node_id` holds this engine's own host name and process id, so the key carries them), `gcm_invocations` (the per-key AES-GCM invocation bound crossing its 2^31 soft warn — ASVS 11.3.4; its payload carries a one-way `key_id` fingerprint plus counters, never key bytes), `approval_stale_requester` (a dual-control release refused because the requester no longer holds the authority the operation needs -- ASVS 8.3.2, BACKLOG #289; keyed on the approval id, and the payload is that id, the operation key and a closed-set reason slug, never the requester's name or the captured params), `approval_too_early` (a dual-control release refused because it arrived before `[approvals].min_dwell_seconds` -- ASVS 2.4.2, BACKLOG #287; keyed on `approval:`, and the payload is that key, the operation key and a fixed reason string, never the approver's or requester's name or the captured params), `audit_write_failed` (the approval gate could not write one of its audit rows -- vault BACKLOG #2255; keyed on `approval:`, or on `config_reload:inline` when an ungated config reload could not write its `config_reload_attempted` row -- vault BACKLOG #2254; the payload is that key and the lost row's audit action name, never the row's detail, a username or the captured params; the ERROR log line beside it holds the detail on some paths and only the exception on others), `ad_reconcile_aborted` (a directory reconciler pass left signed-in accounts unrevoked and needs an operator: the mass-revoke breaker tripped and nothing was revoked, or one or more probes were answered with an LDAP referral and those accounts were left unjudged while the rest of the pass was applied, BACKLOG #2538; the payload is a fixed source label, `directory-reconciler` for the breaker and `directory-reconciler-referral` for a referral so each is its own instance, the closed-set reason slug `mass_revoke_breaker` or `directory_referral`, a count (the accounts the breaker judged, or for a referral the lookups the pass sent, which leaves out an id-less row -- vault BACKLOG #3170) and the latched operator explanation, never a username), `ad_reconcile_held` (the reconciler is holding accounts whose `userAccountControl` it cannot read, ADR 0195; the payload is the same fixed source label, a closed-set reason slug, the count of undetermined accounts and the latched operator explanation, never a username), `ad_session_revoked` (the reconciler revoked a directory principal's sessions; keyed on that operator account's username, with at least a `directory_absent`, `directory_disabled`, `directory_undetermined`, `directory_object_id_missing`, `roles_changed` or `scope_changed` reason), `approval_approver_provenance` (a dual-control release, or the resolution of an interrupted one, went ahead with an approver or resolver account created, re-passworded or TOTP-enrolled after the request -- BACKLOG #315, #2125; keyed on `approval:`, or `approval::resolve` for a resolution, and the payload is the operation key and the closed-set change slugs, never a username or the captured params), `administrator_granted` (the built-in Administrator role was granted by account creation, a role change, a directory group newly mapped to it -- BACKLOG #315 --, a directory sign-in whose role sync newly gave the account the role -- vault BACKLOG #2610 -- or a directory reconcile pass whose role re-diff did -- vault BACKLOG #3141; keyed on `user:` or `ad-group:`, with the grant route and the granting administrator's username, or `` for a directory sign-in or reconcile pass), `no_enabled_administrator` (a directory sign-in's role sync or the session reconciler took Administrator from its last enabled holder -- vault BACKLOG #3229; keyed on `user:`, and the payload is `via`, `ad-sync` or `ad-reconcile`, and a fixed reason sentence, never message content), `intake_paused` (the engine paused intake because the staged backlog went over `[inbound].max_staged_depth` or the SQLite volume fell below `[retention].min_free_disk_mb` -- BACKLOG #290; keyed on `intake:staged_depth` or `intake:disk_floor`, with `@node:` or `@shard:` appended when several engine processes share the store (BACKLOG #2272), and the payload is the reason, a count or a MiB figure, its limit and the store backend's name, never message content), `config_changed` (a start loaded a config whose fingerprint differs from the store's baseline -- vault BACKLOG #2597; keyed on `config:`, and the payload is the two digests, the node and engine shard labels, and the action, actor username and time of the row the older digest came from, never the config directory, a git commit or message content), `crl_reload_refused` (the CRL reload refused a replaced CRL file, so running TLS contexts keep the copy they hold -- vault BACKLOG #2904; keyed on `crl_reload:`, with `@node:` or `@shard:` appended when several engine processes share the store, and the payload is that key, the file's path, a short class token for the remedy, a count of contexts, the engine's own fixed remedy sentence and, when a start would refuse the file too, a `start_refuses` flag, never the refusal's reason, which quotes CRL issuer names and dates; that reason is in the application log, and in the `detail` of the CRL expiry alert once the held copy nears its `nextUpdate`), `remote_debug_refused` (the engine's remote-debugging guard refused scripts that another process injected into the engine process -- vault BACKLOG #2742; keyed on `remote_debug:refused`, with `@node:` or `@shard:` appended when several engine processes share the store, and the payload is that key, the count of scripts refused since the alert was last raised, the count since the process started and a fixed sentence built from the two, never a script's file name, which the injecting process chooses; that name is in the application log, in the guard's WARNING line), and the six security signals of the audit-stream rule layer -- vault BACKLOG #2613: `signin_failure_burst` (keyed on `signin:`, never the typed username), `access_denied_burst`, `body_view_burst` and `bulk_export` (each keyed on `account:`), `log_level_debug` (keyed on `logging:debug`) and `posture_loosened` (keyed on `posture:start`); the payload is a count and a fixed sentence holding a level name, switch names or a username, never a message body, a message id or an audit row's detail. The reachable **inverse** signals — `connection_restored`, `leadership_lost`, `dr_released`, `store_privilege_clean`, `intake_resumed`, `crl_reload_applied` (it resolves `crl_reload_refused` for the same file on the same engine process, once the reload's record no longer holds the refusal, and when the reload stops -- vault BACKLOG #2904), `ad_reconcile_breaker_cleared`, `ad_reconcile_hold_released` (the last two resolve `ad_reconcile_aborted`, under either source, and `ad_reconcile_held`; when they do, and where they never do, is stated once in [CONFIGURATION.md](CONFIGURATION.md#when-the-reconcilers-two-alerts-resolve-themselves) -- BACKLOG #2136) — are never rows here: `_record_state` routes an inverse through `_AUTO_RESOLVE` to `resolve_alert_instances_for`, never to `upsert_alert_instance`. (One more mapped key, `connection_started`, is emitted by no code path today.) | rows: `event_type`, `connection`, `severity`, `status`, `first_seen`, `last_seen`, `count`, `reason`, `acked_by`, `acked_at`, `resolved_at`, `suspended_until`, `escalation_tier` | the store database, **all three backends** | the operator alert list — acknowledge / resolve / suspend. Durable state is recorded **before** any suppression or throttle return, so a muted alert still leaves a record | `GET /alerts/active` under **`monitoring:diagnose`** (**not** a PHI permission) with the same per-channel scope, and `reason` alone gated on `messages:view_summary` and masked as `****` until the audited per-alert `reveal=` act (BACKLOG #2443; the `alert_instance.reason` row of §3's PL-2 block says how); ack/resolve/suspend/**resume** are POSTs on the same tier, and the separate read-only `GET /alerts/rules` view sits on its own gate | shares the connection-event window; **only RESOLVED instances are DELETEd**, by `resolved_at` — an open or acknowledged condition is never aged out from under an operator | **`reason` is free text** taken from the event's `detail`/`reason`: `safe_text(reason, limit=200)` (the bound §3's PL-2 block states) then cipher-encrypted (AAD `("alert_instance","reason",event_type,connection)` — the de-dup grain, so one AAD covers both the INSERT and the re-fire UPDATE). | | **9. `response` rows with `kind='ack_sent'` — DEFAULT ON** (`[diagnostics].response_sent = true`) | the ACK/NAK the engine returned to an inbound sender, under a sentinel destination `\x1fack:` | rows: `ack_code` (`AA`/`AE`/`AR`/`CA`/`CE`/`CR`), `ack_phase` (`decode`/`parse`/`strict`/`ingest`), `outcome`, `body`, `detail` | the store database | "what did we actually reply, and why" — the operator's answer to a sender disputing an ACK | `GET /messages/{id}/responses` under `messages:read` + `require_phi_read`; the `body` only for a caller who also holds `messages:view_raw` and `messages:view_summary`; every read writes a `response.read` audit row | `body`, `detail` and `resp_headers` are set to `NULL` in place by `purge_message_bodies` on the message-body window, on all three backends | **PHI fail-safe:** the ACK **body** is stored **only when the store cipher is active** — on a keyless store it is `NULL` rather than plaintext — and every NAK passes no body at all, so the offending field value is never persisted. The disposition metadata (`ack_code`/`ack_phase`/`outcome`) is non-PHI and always captured; `detail` is `safe_text`-scrubbed, held to the 200-character bound §3's PL-2 block states, and encrypted | | **10. `[alerts]` webhook transport** (off by default — `webhook_url` unset) | one HTTPS POST per alert, carrying every non-underscore event key as JSON | JSON | the operator's webhook endpoint (Slack/Teams/PagerDuty/custom) | operator notification | **`https` only** — a plaintext `http://` webhook URL is refused at construction unless the `MEFOR_ALLOW_INSECURE_TLS` escape is set (and then a warning is logged); since #329 this path routes that escape through the clamped `weakened_tls_escape_permitted(posture)` (the instance posture threaded from the API lifespan), so on an enforcing-PHI instance the escape is inert and a cleartext webhook POST stays refused — the same clamp as the connectors, no longer the raw escape. Redirects are refused; an optional `webhook_allowed_hosts` egress allowlist gates the host | the endpoint's | **carries the alert's `detail`/`reason` free text** (`safe_exc()`-scrubbed at the emit sites, but **not** re-run through `safe_text` on this path). Internal `_`-prefixed keys (per-rule recipients, rule id, cooldown) are stripped before send, so recipient addresses never cross the wire | | **11. `[alerts]` SMTP transport — operator alert list** (off unless `email_smtp_host` + `email_from` + ≥1 `email_to`) | one email per alert; default subject `[MessageFoundry] — `, default body every non-underscore event key as `k: v` | plain text (always kept — never HTML-only); optional HTML alternative | the operators' mailboxes | operator notification | `smtp_allowed_hosts` egress allowlist; the SMTP password comes from `MEFOR_ALERTS_EMAIL_PASSWORD` or a `[secrets]` provider, never the config file; per-send timeout `email_timeout` | the mail system's | carries the same `detail`/`reason` free text as the webhook. #138 operator templates are constrained to a **closed non-PHI variable allowlist** validated fail-closed at config load. **Transport posture:** `send_plain_email` builds an explicit **verifying** context (chain + hostname + strict RFC 5280, TLS 1.2 floor) via `tls_policy.build_smtp_tls_context()` and passes it to `starttls()`, anchored to the OS roots, `[alerts].email_tls_ca_file`, or `[tls].internal_ca_file` — the same factory the EMAIL and DIRECT *message destinations* use, so all three SMTP cells now share one policy ([#323](BACKLOG.md), closed 2026-08-02). Before that this call passed **no** context and Python's stdlib default applied (`ssl._create_stdlib_context` **is** `ssl._create_unverified_context` — `CERT_NONE`, `check_hostname = False`), leaving the hop encrypted but unauthenticated. There is still **no hop gradient or attestation on this path** — unlike the connectors, this cell is constructed outside the `active_hop_posture` scope, so its deviations (`email_use_tls = false`, or `email_tls_verify = false`) are gated by a `[security].allow_unverified_alert_smtp_tls` **acknowledgment switch at the serve gate** rather than by the clamped escape: on an enforcing PHI instance `serve` refuses to start without it, and permits + `AUDIT`-logs the start with it. Both deviations are named by `security_loosenings()` and reported by `messagefoundry check`'s `alert-smtp-tls` advisory | diff --git a/docs/SECURITY.md b/docs/SECURITY.md index c37922f1e8..bae51cb518 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -4114,7 +4114,10 @@ account's entry and write its roles onto the row. No sign-in or step-up admits s writes no roles, and it strikes and revokes as an absent account does, with the reason `directory_object_id_missing`. It is not a `userAccountControl` answer, so it never moves the ADR 0195 hold. It is never asked, so it is no part of an outage either: a pass whose every lookup fails -still audits `auth.ad_reconcile_skipped`, and that row's `asked` count leaves it out. Such a +still audits `auth.ad_reconcile_skipped`, and that row's `asked` count leaves it out. Such a pass, +and one whose every lookup was referred, still strikes the row, because its answer never came from +the directory (BACKLOG #3170). A referral's `auth.ad_reconcile_referred` row and alert count only +the lookups sent too. Its revocation does count toward the mass-revoke breaker below. Such a row cannot take a federated binding: the bind refuses it, so every binding the bind has made since BACKLOG #1143 slice C sits on a row probed by its id (ADR 0184 AC-5). A binding already on an id-less row, made before that refusal, is **never probed by name** (BACKLOG #2027). The pass skips the row and diff --git a/messagefoundry/api/app.py b/messagefoundry/api/app.py index 93e51545aa..8362ec7aa7 100644 --- a/messagefoundry/api/app.py +++ b/messagefoundry/api/app.py @@ -9512,7 +9512,9 @@ def _alert_reconcile_plan(plan: ReconcilePlan, auth: AuthService, sink: AlertSin sink.ad_reconcile_aborted( _REFERRAL_ALERT_SOURCE, reason=REFERRAL_ABORT, - probed=plan.probed, + # The probes ASKED (BACKLOG #3170): an UNKEYED row was never sent, so no referral + # could answer it. The breaker's count above stays its judged denominator. + probed=plan.asked, detail=auth.directory_reconcile_referral or REFERRAL_ABORT, ) if plan.hold: diff --git a/messagefoundry/auth/reconcile.py b/messagefoundry/auth/reconcile.py index 027892a3fb..017b88d42d 100644 --- a/messagefoundry/auth/reconcile.py +++ b/messagefoundry/auth/reconcile.py @@ -192,14 +192,17 @@ class ReconcilePlan: #: count``. A PRESENT probe maps to 0 (reset), and so does a HELD one (ADR 0195 rule item 7; #: ``held`` tells the two apart). An UNAVAILABLE or REFERRED probe is absent from this mapping, #: leaving whatever strike the user already carried untouched. Populated even on a breaker - #: abort — see :func:`plan_pass`. + #: abort — see :func:`plan_pass`. On an outage or a pass of referrals only it holds the UNKEYED + #: rows' strikes alone (BACKLOG #3170), since those were judged without the directory. strikes: Mapping[str, int] = field(default_factory=dict) #: Candidates this pass judged, asked or not. Excludes the per-pass budget remainder. An UNKEYED #: row counts here, because it can revoke. This is the breaker's denominator (``judged``). probed: int = 0 #: Probes this pass actually sent to the directory: ``probed`` less its UNKEYED rows (BACKLOG #: #2434). An outage and a pass of referrals only are judged over these, and the outage's log - #: line and the ``asked`` field of its ``auth.ad_reconcile_skipped`` row count these. + #: line and the ``asked`` field of its ``auth.ad_reconcile_skipped`` row count these. So do the + #: referral's latched message, the ``asked`` field of its ``auth.ad_reconcile_referred`` row + #: and its alert's count (BACKLOG #3170): a referral can only answer a probe that was sent. asked: int = 0 unavailable: int = 0 #: ``user_id`` of every probe the directory answered with a referral (BACKLOG #2538). Any one @@ -377,6 +380,14 @@ def plan_pass( # BACKLOG #2434. The caller reads a row it may not ask about as UNKEYED, so that outcome is # the one mark of "never asked": it says nothing about whether the directory answered. asked = sum(1 for p in probes if p.outcome is not ProbeOutcome.UNKEYED) + # BACKLOG #3170 item 3. An UNKEYED row was judged without the directory, so a pass that learned + # nothing from the directory still strikes it, as any other pass does. Only the strike: its + # outcome is not recorded here, so the hold's record, and so its carry-over, stays as it was. + unkeyed_strikes = { + p.user_id: prior_strikes.get(p.user_id, 0) + 1 + for p in probes + if p.outcome is ProbeOutcome.UNKEYED + } if referred and len(unavailable) + len(referred) == asked: # BACKLOG #2538. Nothing the pass asked was answered, as on an outage, so it judges nothing @@ -384,6 +395,7 @@ def plan_pass( # failures is not read as a plain outage, which pages nobody. A user search base in another # domain of the forest refers every probe, and this is the shape it takes. return ReconcilePlan( + strikes=unkeyed_strikes, probed=len(probes), asked=asked, unavailable=len(unavailable), @@ -399,6 +411,7 @@ def plan_pass( # zero-revocation pass. The hold is not judged on a pass that learned nothing, so its state # carries over. return ReconcilePlan( + strikes=unkeyed_strikes, probed=len(probes), asked=asked, unavailable=len(unavailable), @@ -501,6 +514,9 @@ def plan_pass( readable=readable, ) # Held and referred probes both stay in the breaker's denominator; ReconcilePlan.judged says why. + # UNKEYED revocations stay in its numerator, so while such rows hold sessions they push toward + # a trip on every pass. Exempting them is declined (BACKLOG #3170 item 2): it would change what + # the mass-revoke breaker counts, a security control, and that is not a follow-up's call. if breaker_tripped( revoke_count=len(revocations), probed=plan.judged, diff --git a/messagefoundry/auth/service.py b/messagefoundry/auth/service.py index 3e74856870..f41b25bbe2 100644 --- a/messagefoundry/auth/service.py +++ b/messagefoundry/auth/service.py @@ -1586,17 +1586,25 @@ def _holds_federated_binding(user: UserRecord) -> bool: return user.oidc_issuer is not None or user.oidc_subject is not None +def _directory_key(user: UserRecord) -> str | None: + """The ``directory_object_id`` the engine may ask the directory about ``user`` by, or ``None`` + when the row carries none (an empty value counts as none). :func:`_holds_directory_key` states + the rule; a caller that goes on to send the id takes it from here, so the type narrows.""" + return user.directory_object_id or None + + def _holds_directory_key(user: UserRecord) -> bool: - """Whether the reconciler may ask the directory about ``user``: it carries a + """Whether the engine may ask the directory about ``user``: it carries a ``directory_object_id`` (BACKLOG #2434). - **The reconcile pass's one statement of the rule**; the sign-in, step-up and bind refusals - still test the column inline. A row without one is never asked about by name, so it is no - directory evidence: the pass reads it as UNKEYED without a lookup, and a trip or a referral - leaves it unmarked, because no pass can ever read it PRESENT. ``reconcile.plan_pass`` sees only - the UNKEYED outcome this produces, and counts what was asked from that. + **The one statement of the rule**: the reconcile pass and the sign-in, step-up and bind + refusals all read it (BACKLOG #3170 item 7). A row without one is never asked about by name, + so on the reconcile pass it is no directory evidence: the pass reads it as UNKEYED without a + lookup, and a trip or a referral leaves it unmarked, because no pass can ever read it PRESENT. + ``reconcile.plan_pass`` sees only the UNKEYED outcome this produces, and counts what was asked + from that. """ - return bool(user.directory_object_id) + return _directory_key(user) is not None def _holds_unkeyed_federated_binding(user: UserRecord) -> bool: @@ -4610,7 +4618,8 @@ async def _authenticate_oidc( username, "oidc", "local_account_conflict", client=client ) return LoginOutcome(ok=False, error="account conflict", reason="local_account_conflict") - if not bound.directory_object_id: + object_id = _directory_key(bound) + if object_id is None: # BACKLOG #2027 (ADR 0184 AC-5). A BOUND ROW WITH NO IMMUTABLE ID IS REFUSED, NOT # RE-RESOLVED BY NAME. The bind refuses such a row since BACKLOG #1143 slice C, so this # reaches only a binding written before that, or planted in the store. Its only @@ -4641,7 +4650,7 @@ async def _authenticate_oidc( # re-resolve is never keyed on the name alone (BACKLOG #2027). try: principal = await asyncio.to_thread( - self._ldap.resolve_principal, bound.username, object_id=bound.directory_object_id + self._ldap.resolve_principal, bound.username, object_id=object_id ) except LdapError as exc: await self._audit( @@ -4661,7 +4670,7 @@ async def _authenticate_oidc( ) # BACKLOG #3174: the answer must name the bound row's own directory object, by the test the # IdP step-up leg applies, before its groups or name are used. - mismatch = _directory_answer_mismatch(principal, bound.directory_object_id) + mismatch = _directory_answer_mismatch(principal, object_id) if mismatch is not None: await self._directory_reject_audit(username, "oidc", mismatch, client=client) return LoginOutcome(ok=False, error="federated sign-in failed", reason=mismatch) @@ -5031,13 +5040,14 @@ async def complete_oidc_step_up( # the directory itself, by the row's immutable id as the sign-in leg does (BACKLOG #1532). if self._ldap is None: return await refuse("not_configured") - if not user.directory_object_id: + object_id = _directory_key(user) + if object_id is None: # Never a name-only lookup: a directory may reissue a freed name to someone else. The # sign-in leg refuses such a row for the same reason (BACKLOG #2027). return await refuse(DIRECTORY_OBJECT_ID_MISSING) try: principal = await asyncio.to_thread( - self._ldap.resolve_principal, user.username, object_id=user.directory_object_id + self._ldap.resolve_principal, user.username, object_id=object_id ) except LdapError as exc: _log.warning( @@ -5048,7 +5058,7 @@ async def complete_oidc_step_up( return await refuse("not_in_directory") # BACKLOG #3174: the answer must name the row's own directory object before its groups are # trusted, by the test the sign-in leg and the AD step-up re-bind apply. - mismatch = _directory_answer_mismatch(principal, user.directory_object_id) + mismatch = _directory_answer_mismatch(principal, object_id) if mismatch is not None: return await refuse(mismatch) # BACKLOG #2154: the account's stored roles must all be among the roles its current groups @@ -7251,7 +7261,7 @@ async def _record_reconcile_referral( self._reconcile_referral_alert = ( "LDAP referral: check [auth].ad_user_search_base and, for nested groups, " "[auth].ad_group_search_base; one names a base in another domain of the forest. " - f"{len(plan.referred)} of {plan.probed} signed-in directory account(s) probed were " + f"{len(plan.referred)} of {plan.asked} signed-in directory account(s) probed were " f"referred and left unjudged. {others} A referred account is never revoked while it " "is referred, because the engine does not follow referrals. Use a base in the bound " "domain controller's own domain, or a global catalog. The ERROR log line for each such " @@ -7273,7 +7283,9 @@ async def _record_reconcile_referral( "reason": reconcile.REFERRAL_ABORT, "referred": len(plan.referred), "unavailable": plan.unavailable, - "probed": plan.probed, + # Over the probes ASKED (BACKLOG #3170), as the outage's row is (#2434): an + # UNKEYED row was never sent, so no referral could answer it. + "asked": plan.asked, "aborted": plan.aborted, } ), @@ -8942,7 +8954,8 @@ async def _reproof_serialized( verdict: bool | None reason: str | None = None if directory: - if not user.directory_object_id: + object_id = _directory_key(user) + if object_id is None: # BACKLOG #2027, ADR 0184 AC-5. The re-bind finds its entry by the row's id, and a # row with none has only its name, which a directory may have reissued to someone # else whose password would then step this session up. Refused before the directory @@ -8950,9 +8963,7 @@ async def _reproof_serialized( # did not guess wrong. verdict, reason = None, DIRECTORY_OBJECT_ID_MISSING else: - rebind = await self._reauth_ad( - user.username, password, object_id=user.directory_object_id - ) + rebind = await self._reauth_ad(user.username, password, object_id=object_id) verdict, reason = rebind.verdict, rebind.reason else: verdict = user.password_hash is not None and await self._argon2( @@ -10655,11 +10666,11 @@ async def _directory_presence(self, user: UserRecord) -> reconcile.Probe | str: """ if self._ldap is None: return "not_configured" - if not user.directory_object_id: + if not _holds_directory_key(user): return DIRECTORY_OBJECT_ID_MISSING # The reconciler's own probe, so both ask the same question by the same key, off the loop. - # They differ on an id-less row only: this leg refuses it above, and the reconciler still - # probes an unbound one by name (BACKLOG #2027). + # Neither asks about an id-less row: this leg refuses it above, and the reconciler reads it + # as UNKEYED without a lookup (BACKLOG #2434, #3170 item 6). probe = await self._probe_principal(user) if probe.outcome is not reconcile.ProbeOutcome.PRESENT: return str(probe.outcome.value) @@ -12367,7 +12378,7 @@ async def audit_refused(reason: str, username: str, **extra: object) -> None: if user.auth_provider != AuthProvider.AD.value: await audit_refused("not_a_directory_account", user.username) raise ValueError("only a directory (AD) account can take a federated binding") - if not user.directory_object_id: + if not _holds_directory_key(user): # BACKLOG #1143 slice C: see DirectoryObjectIdMissing. After the no-op check, so a # legacy binding resubmitted as it stands gets the harmless answer above, while moving # it to another pair is still refused. Unbinding it stays open. diff --git a/messagefoundry/pipeline/alerts.py b/messagefoundry/pipeline/alerts.py index 29a07e6baf..b7462db6e9 100644 --- a/messagefoundry/pipeline/alerts.py +++ b/messagefoundry/pipeline/alerts.py @@ -542,10 +542,11 @@ def ad_reconcile_aborted(self, name: str, *, reason: str, probed: int, detail: s ``name`` labels the source: ``"directory-reconciler"`` for the breaker, and ``"directory-reconciler-referral"`` for a referral, so each is its own instance and - throttle. ``reason`` is one of those two closed-set slugs. ``probed`` is how many - principals the pass probed. ``detail`` is the operator-facing explanation the auth service - latches. No PHI. Emitted by the API-lifespan - reconciler task, never from ``auth/``. A whole-directory outage is NOT this event: it is + throttle. ``reason`` is one of those two closed-set slugs. ``probed`` is, for the breaker, + how many principals the pass judged, its denominator, and for a referral how many probes + the pass sent to the directory, which leaves out an id-less row (BACKLOG #3170). + ``detail`` is the operator-facing explanation the auth service latches. No PHI. Emitted by + the API-lifespan reconciler task, never from ``auth/``. A whole-directory outage is NOT this event: it is audited as ``auth.ad_reconcile_skipped`` and pages nothing, because the accounts are fine.""" ... diff --git a/tests/test_ad_session_reconcile.py b/tests/test_ad_session_reconcile.py index 5dbe0199e8..4ea7e352a8 100644 --- a/tests/test_ad_session_reconcile.py +++ b/tests/test_ad_session_reconcile.py @@ -616,6 +616,38 @@ def test_an_unkeyed_probe_does_not_hide_an_outage_or_a_referral_only_pass() -> N assert alone.aborted is None and (alone.probed, alone.asked) == (1, 0) +@pytest.mark.parametrize( + "asked", + [ + [reconcile.Probe("a", "a", reconcile.ProbeOutcome.UNAVAILABLE)], + [reconcile.Probe("r", "r", reconcile.ProbeOutcome.REFERRED)], + [ + reconcile.Probe("a", "a", reconcile.ProbeOutcome.UNAVAILABLE), + reconcile.Probe("r", "r", reconcile.ProbeOutcome.REFERRED), + ], + ], + ids=["outage", "referrals-only", "referrals-and-failures"], +) +def test_an_unkeyed_row_is_struck_on_a_pass_that_judged_nothing( + asked: list[reconcile.Probe], +) -> None: + """BACKLOG #3170 item 3. An UNKEYED row is judged without the directory, so a pass that learned + nothing from it still strikes the row. Only the strike: the asked probes' strikes, the outcome + record the hold reads, and the hold's latch all stay as they were. + + RED before #3170: both early returns carried no strikes, so such a row went unstruck.""" + unkeyed = reconcile.Probe("k", "k", reconcile.ProbeOutcome.UNKEYED) + plan = _plan([*asked, unkeyed], prior_strikes={"k": 1, "a": 1, "r": 1}, latched=True) + assert plan.judged_nothing + assert plan.strikes == {"k": 2} + assert plan.outcomes == {} and plan.latched and not plan.hold + assert plan.revocations == () + # The control: the same row on an ordinary pass strikes the same way, and revokes there. + ordinary = _plan([unkeyed], prior_strikes={"k": 1}) + assert ordinary.strikes == {"k": 2} + assert [r.reason for r in ordinary.revocations] == ["directory_object_id_missing"] + + def test_breaker_ceiling_reports_the_larger_of_the_two_thresholds() -> None: assert reconcile.breaker_ceiling(probed=10, max_absolute=5, max_fraction=0.34) == 5 assert reconcile.breaker_ceiling(probed=300, max_absolute=5, max_fraction=0.34) == 102 @@ -872,7 +904,7 @@ async def test_a_referring_user_search_base_judges_nothing_and_pages_every_pass( "reason": "directory_referral", "referred": 2, "unavailable": 0, - "probed": 2, + "asked": 2, "aborted": "directory_referral", } assert len(referred) == 3 and all(d == expected for d in referred) @@ -1501,6 +1533,57 @@ async def test_a_referral_does_not_mark_an_id_less_row_it_can_never_confirm() -> await store.close() +async def test_a_referral_is_counted_over_the_probes_asked() -> None: + """BACKLOG #3170 item 1. An id-less row is never sent to the directory, so no referral could + answer it. The referral's row, its latched message and its alert count the probes asked, and + the row names the field ``asked``, as the outage's row does (#2434). + + RED before #3170: all three counted ``probed``, which takes in the id-less row.""" + from messagefoundry.api.app import _alert_reconcile_plan + from messagefoundry.pipeline.alerts import LoggingAlertSink + + store = await MessageStore.open(":memory:") + try: + ldap = _FakeLdap( + { + "alice": _principal("alice"), + "bob": _principal("bob"), + "jdoe": replace(_principal("jdoe"), directory_object_id=None), + } + ) + service = AuthService(store, _ad_settings(), ldap=ldap) # type: ignore[arg-type] + await service.initialize() + for name in ("alice", "bob", "jdoe"): + assert await _signed_in_ad_user(service, store, name) is not None + ldap.referring.add("alice") + + plan = await service.reconcile_directory_sessions() + + assert (plan.probed, plan.asked, len(plan.referred)) == (3, 2, 1) + [row] = _referred_rows(await store.list_audit()) + assert row["asked"] == 2 and "probed" not in row + message = service.directory_reconcile_referral + assert message is not None and "1 of 2 signed-in" in message + + counts: list[tuple[str, int]] = [] + + class _Sink(LoggingAlertSink): + def ad_reconcile_aborted( + self, name: str, *, reason: str, probed: int, detail: str + ) -> None: + counts.append((reason, probed)) + + _alert_reconcile_plan(plan, service, _Sink()) + assert counts == [("directory_referral", 2)] + # The breaker's page keeps its judged denominator, which counts the id-less row. + counts.clear() + tripped = replace(plan, aborted="mass_revoke_breaker") + _alert_reconcile_plan(tripped, service, _Sink()) + assert counts == [("mass_revoke_breaker", 3), ("directory_referral", 2)] + finally: + await store.close() + + async def test_a_breaker_trip_does_not_mark_an_id_less_row_or_forfeit_its_clear() -> None: """BACKLOG #2434, the breaker's half of the referral test above. A trip marks every candidate unconfirmed until a pass reads it PRESENT, and revoking a marked account forfeits the clear. An From 1027a1aae669ebc9324f1436691c89bc399f71ae Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Fri, 16 Oct 2026 22:24:24 -0500 Subject: [PATCH 24/57] The reconcile-path store-fault set takes aiosqlite's closed-connection ValueError (b257 D2 carry-over) D2 carry-over. _audit_write_errors is now CipherError plus store_call_errors(), the one store-fault set D2 added, so a reconciler audit write against a closed aiosqlite connection (a bare ValueError, raised before sqlite3 is reached) is logged and passed over like any other refused write, instead of ending the pass. Callers that must not read a bug as a refusal still re-raise AUDIT_WRITE_DEFECTS first. The skip-report refusal test gains a closed-connection case, red on the previous set. --- messagefoundry/auth/service.py | 16 +++++++++------- tests/test_ad_session_reconcile.py | 12 +++++++++--- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/messagefoundry/auth/service.py b/messagefoundry/auth/service.py index f41b25bbe2..c313a91341 100644 --- a/messagefoundry/auth/service.py +++ b/messagefoundry/auth/service.py @@ -128,7 +128,7 @@ from messagefoundry.credential import constant_time_equal from messagefoundry.keyed_lock import KeyedLock as _KeyedLock from messagefoundry.keyed_lock import hold_keyed_lock as _hold_keyed_lock -from messagefoundry.store.base import AdminStore, store_call_errors, store_driver_errors +from messagefoundry.store.base import AdminStore, store_call_errors from messagefoundry.store.crypto import MARKER_PREFIX, CipherError from messagefoundry.store.row_seal import ( ADSET_KIND, @@ -187,14 +187,16 @@ def _audit_write_errors() -> tuple[type[Exception], ...]: """What a refused audit write raises, at least on the store backends and key modes named here (BACKLOG #2137). - The drivers' own errors, plus the two that :func:`~messagefoundry.store.base.store_driver_errors` - asks a caller to add: ``RuntimeError`` for the engine's own refusals (the acquire timeout, a - keyless audit append) and ``OSError`` for a connection lost at the socket. And ``CipherError``, - which a ``vault_transit`` store raises when Transit cannot compute the audit-chain MAC (ADR - 0138). Built on first use, because naming a server driver imports it; an ``AuthService`` that + :func:`~messagefoundry.store.base.store_call_errors`: the drivers' own errors, ``RuntimeError`` + for the engine's own refusals (the acquire timeout, a keyless audit append), ``OSError`` for a + connection lost at the socket, and ``ValueError``, which aiosqlite raises for a closed + connection before sqlite3 is reached. And ``CipherError``, which a ``vault_transit`` store + raises when Transit cannot compute the audit-chain MAC (ADR 0138). A caller re-raises + :data:`~messagefoundry.audit_write.AUDIT_WRITE_DEFECTS` first where a bug must not read as a + refusal. Built on first use, because naming a server driver imports it; an ``AuthService`` that runs the reconciler builds it at construction, so that import never runs on the event loop in the middle of a store incident.""" - return (RuntimeError, OSError, CipherError, *store_driver_errors()) + return (CipherError, *store_call_errors()) def _warn_if_corpus_unreadable(path: str | None) -> None: diff --git a/tests/test_ad_session_reconcile.py b/tests/test_ad_session_reconcile.py index 4ea7e352a8..93c752baec 100644 --- a/tests/test_ad_session_reconcile.py +++ b/tests/test_ad_session_reconcile.py @@ -1988,8 +1988,13 @@ async def _reported(store: MessageStore) -> list[str]: @pytest.mark.parametrize( "refusal", - [sqlite3.OperationalError("synthetic: disk I/O error"), CipherError("synthetic: Transit down")], - ids=["driver", "transit-mac"], + [ + sqlite3.OperationalError("synthetic: disk I/O error"), + CipherError("synthetic: Transit down"), + # aiosqlite's closed connection, raised before sqlite3 is reached (BACKLOG #3145 carry-over). + ValueError("no active connection"), + ], + ids=["driver", "transit-mac", "closed-connection"], ) async def test_a_refused_skip_report_logs_once_per_pass_and_tries_the_next_account_next( monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, refusal: Exception @@ -1998,7 +2003,8 @@ async def test_a_refused_skip_report_logs_once_per_pass_and_tries_the_next_accou the first failure ended the pass; now the report stops at the first refusal, which also keeps a pool that times out each write to one timeout per pass. The account it refused goes last next pass, so the next account is tried. Both are reported once the - store accepts the write. A Transit outage refuses like a driver error (ADR 0138).""" + store accepts the write. A Transit outage refuses like a driver error (ADR 0138), and so does a + closed aiosqlite connection.""" store = await MessageStore.open(":memory:") try: service = await _two_unkeyed_bindings(store) From 7aae0c7c46b9d7d14e43498393bda4c7c5bd5e49 Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Fri, 16 Oct 2026 22:31:10 -0500 Subject: [PATCH 25/57] Simplify pass: the judged-nothing UNKEYED strikes are built only where they are used (BACKLOG #3170) plan_pass built the UNKEYED strike map on every pass and read it only in the two early returns. It is now a small helper, _unkeyed_strikes, called from those returns; the rule and its reason are stated there. Skipped from the pass: a shared factory for the two user-keyed alert callables (two copies only); converting the bind and presence refusals to _directory_key (they need no id); a RolesGained-based grant test in the reconciler (a two-line rule); reusing another file's test sink. --- messagefoundry/auth/reconcile.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/messagefoundry/auth/reconcile.py b/messagefoundry/auth/reconcile.py index 017b88d42d..230586d7b3 100644 --- a/messagefoundry/auth/reconcile.py +++ b/messagefoundry/auth/reconcile.py @@ -346,6 +346,19 @@ def select_candidates( return ordered[: max(budget, 0)] +def _unkeyed_strikes(probes: list[Probe], prior_strikes: Mapping[str, int]) -> dict[str, int]: + """The strikes of a pass that judged nothing (BACKLOG #3170 item 3). + + An UNKEYED row was judged without the directory, so an outage or a pass of referrals only still + strikes it, as any other pass does. Only the strike: its outcome is not recorded, so the hold's + record, and so its carry-over, stays as it was, and an aborted pass revokes nothing.""" + return { + p.user_id: prior_strikes.get(p.user_id, 0) + 1 + for p in probes + if p.outcome is ProbeOutcome.UNKEYED + } + + def plan_pass( probes: Iterable[Probe], *, @@ -380,14 +393,6 @@ def plan_pass( # BACKLOG #2434. The caller reads a row it may not ask about as UNKEYED, so that outcome is # the one mark of "never asked": it says nothing about whether the directory answered. asked = sum(1 for p in probes if p.outcome is not ProbeOutcome.UNKEYED) - # BACKLOG #3170 item 3. An UNKEYED row was judged without the directory, so a pass that learned - # nothing from the directory still strikes it, as any other pass does. Only the strike: its - # outcome is not recorded here, so the hold's record, and so its carry-over, stays as it was. - unkeyed_strikes = { - p.user_id: prior_strikes.get(p.user_id, 0) + 1 - for p in probes - if p.outcome is ProbeOutcome.UNKEYED - } if referred and len(unavailable) + len(referred) == asked: # BACKLOG #2538. Nothing the pass asked was answered, as on an outage, so it judges nothing @@ -395,7 +400,7 @@ def plan_pass( # failures is not read as a plain outage, which pages nobody. A user search base in another # domain of the forest refers every probe, and this is the shape it takes. return ReconcilePlan( - strikes=unkeyed_strikes, + strikes=_unkeyed_strikes(probes, prior_strikes), probed=len(probes), asked=asked, unavailable=len(unavailable), @@ -411,7 +416,7 @@ def plan_pass( # zero-revocation pass. The hold is not judged on a pass that learned nothing, so its state # carries over. return ReconcilePlan( - strikes=unkeyed_strikes, + strikes=_unkeyed_strikes(probes, prior_strikes), probed=len(probes), asked=asked, unavailable=len(unavailable), From d7e3e9e4db1fa7118c184566897cff9d9d0b26a5 Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Fri, 16 Oct 2026 22:52:57 -0500 Subject: [PATCH 26/57] Review round one on the D3 reconcile follow-ups (BACKLOG #3141 #3170) - changelog.d/3170 and SECURITY.md: an outage or referral-only pass strikes an id-less row but revokes nothing, so its sessions end at the next pass that is not aborted, not at the threshold regardless. - SECURITY.md: a sign-in running at the same time as the pass can raise a second administrator_granted; a duplicate is the safe direction. - LoggingAlertSink's ad_reconcile_aborted line no longer calls the count the pass's size, since a referral's count is the probes asked. - Tests: the reconciler-grant test now proves the next sign-in gains nothing (the old second-pass check ran on no candidate); the reconciler cooldown-pop test gains the cross-account control. --- changelog.d/3170.security.md | 7 ++++--- docs/SECURITY.md | 7 +++++-- messagefoundry/pipeline/alerts.py | 3 ++- tests/test_directory_last_admin_alert.py | 12 ++++++++++-- 4 files changed, 21 insertions(+), 8 deletions(-) diff --git a/changelog.d/3170.security.md b/changelog.d/3170.security.md index 11e238675a..2795c080d7 100644 --- a/changelog.d/3170.security.md +++ b/changelog.d/3170.security.md @@ -4,6 +4,7 @@ field in place of `probed`, and that row, the referral's latched message and its `ad_reconcile_aborted` alert count only the lookups the pass sent. The breaker's own alert still counts every account it judged. Such a row is also struck now on a pass that is a directory - outage or got only referrals, as on any other pass, so its sessions end at the strike threshold - however the directory answered. Its revocation still counts toward the mass-revoke breaker, as - before. (`vault BACKLOG #3170`) + outage or got only referrals, as on any other pass. Such a pass still revokes nothing, so the + row's sessions end at the first pass after it that is not aborted, and only at their own expiry + while every pass aborts. Its revocation still counts toward the mass-revoke breaker, as before. + (`vault BACKLOG #3170`) diff --git a/docs/SECURITY.md b/docs/SECURITY.md index bae51cb518..b042428a3d 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -2453,7 +2453,9 @@ with no page. `via` set to `directory_reconcile`. That alert clears the account's `no_enabled_administrator` cooldown, as a sign-in's does. An account that already held Administrator raises nothing. The next sign-in then writes no `auth.ad_roles_resynced` row and raises no second - `administrator_granted`. That section lists when a pass revokes or alerts nothing. + `administrator_granted`. A sign-in that runs at the same time as the pass can still raise a + second one, since each judges against its own read of the roles; a duplicate page is the safe + direction. That section lists when a pass revokes or alerts nothing. No pass runs at `[auth].ad_session_recheck_seconds = 0`. A pass probes at most `ad_session_recheck_max_users` accounts (200 by default), least recently probed first. When more directory accounts than that hold live sessions, an account can wait several passes. @@ -4116,7 +4118,8 @@ writes no roles, and it strikes and revokes as an absent account does, with the 0195 hold. It is never asked, so it is no part of an outage either: a pass whose every lookup fails still audits `auth.ad_reconcile_skipped`, and that row's `asked` count leaves it out. Such a pass, and one whose every lookup was referred, still strikes the row, because its answer never came from -the directory (BACKLOG #3170). A referral's `auth.ad_reconcile_referred` row and alert count only +the directory (BACKLOG #3170). Neither pass revokes it: that waits for the next pass that is not +aborted. A referral's `auth.ad_reconcile_referred` row and alert count only the lookups sent too. Its revocation does count toward the mass-revoke breaker below. Such a row cannot take a federated binding: the bind refuses it, so every binding the bind has made since BACKLOG #1143 slice C sits on a row probed by its id (ADR 0184 AC-5). A binding already on an id-less diff --git a/messagefoundry/pipeline/alerts.py b/messagefoundry/pipeline/alerts.py index b7462db6e9..a87aa46298 100644 --- a/messagefoundry/pipeline/alerts.py +++ b/messagefoundry/pipeline/alerts.py @@ -1088,7 +1088,8 @@ def no_enabled_administrator(self, name: str, *, via: str) -> None: def ad_reconcile_aborted(self, name: str, *, reason: str, probed: int, detail: str) -> None: log.warning( - "ALERT ad_reconcile_aborted: %r left accounts unrevoked in a pass of %d principal(s) " + # The count is the breaker's judged total or a referral's probes asked (BACKLOG #3170). + "ALERT ad_reconcile_aborted: %r left accounts unrevoked, counting %d principal(s) " "(%s): %s", name, probed, diff --git a/tests/test_directory_last_admin_alert.py b/tests/test_directory_last_admin_alert.py index 424d7c3cd2..0631fd0f75 100644 --- a/tests/test_directory_last_admin_alert.py +++ b/tests/test_directory_last_admin_alert.py @@ -354,8 +354,11 @@ async def test_a_reconcile_pass_that_newly_grants_administrator_pages_once(rig: assert await rig.roles("jdoe") == [Role.ADMINISTRATOR.value] assert rig.grants == [("jdoe", RECONCILE_VIA)] assert rig.alerts == [] - # The pass after it finds nothing to change, so it pages nothing more. - assert (await rig.service.reconcile_directory_sessions()).revocations == () + # Why the pass must page: the account's next sign-in finds the role written and gains nothing. + outcome = await rig.service._complete_ad_login( + rig.ldap.present["jdoe"], None, mfa_verified=True + ) + assert outcome.ok and outcome.roles_gained is None assert rig.grants == [("jdoe", RECONCILE_VIA)] @@ -477,9 +480,14 @@ async def test_a_loss_after_a_reconciler_regrant_is_not_held_back_by_the_cooldow regrant = administrator_granted_alert(sink, granted_by=DIRECTORY_GRANTED_BY) lose("jdoe", "ad-reconcile") + # The control: a grant that cleared every account's cooldown would let asmith's second loss + # through and fail the count below. + lose("asmith", "ad-reconcile") regrant("jdoe", RECONCILE_VIA) lose("jdoe", "ad-reconcile") + lose("asmith", "ad-reconcile") await _drain(sink) losses = [e["connection"] for e in transport.events if e["type"] == "no_enabled_administrator"] assert losses.count("user:jdoe") == 2 + assert losses.count("user:asmith") == 1 From ed6efa27d6982a8a00a1f198a30e63cd543498c7 Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Fri, 16 Oct 2026 23:11:51 -0500 Subject: [PATCH 27/57] Review round two: the docs stop over-claiming what an aborted pass does (BACKLOG #3141 #3170) Docs and docstrings only, from the second code-review round: - SECURITY.md's Fail-OPEN bullet and the referring-base paragraph, and reconcile.py's module and judged_nothing docstrings, no longer say an outage or referral-only pass strikes or judges nothing: it strikes an id-less row (BACKLOG #3170). - SECURITY.md and changelog.d/3170: an id-less row is revoked only by a pass that is not aborted, once its strikes reach the threshold and it is in that pass's probe budget. - SECURITY.md "A directory grant": the cases that raise no administrator_granted are "at least" these, adding a breaker-aborted pass and recheck_seconds = 0. Proposed PR title: Directory reconcile follow-ups: a reconciler grant pages administrator_granted, UNKEYED referral counts and outage strikes (BACKLOG #3141 #3170) Proposed vault banner, #3141: BUILT on engine branch b257-d3 (batch 257, D3). A reconcile pass whose role write newly gives an account Administrator raises administrator_granted (via directory_reconcile, granted_by ) right after the write, through an injected callable, judged on the sealed apply-time roles; an aborted pass writes nothing and pages nothing. It pops the no_enabled_administrator cooldown as intended. changelog.d/2610 now states the server-error-after-role-write limit. Item 3: the two low test findings fixed; concurrent first sign-ins paging twice and the future-route reminder declined in writing. Vault #3602 item 4 closes with this; item 8 (the reconciler-first timing race) closes too, since the reconciler-first case now pages. Proposed vault banner, #3170: BUILT on engine branch b257-d3 (batch 257, D3). Items 1, 3, 4, 5, 6, 7 fixed: the referral's message, audit row (field renamed to asked) and alert count plan.asked, the breaker alert keeps probed; outage and referral-only passes strike UNKEYED rows (strikes only); changelog 2434 names the rename; PHI.md lists directory_object_id_missing; the _directory_presence comment is corrected; the inline id checks read _directory_key/_holds_directory_key with reason strings unchanged. Item 2 DECLINED by Manager ruling: exempting UNKEYED revocations would change what the mass-revoke breaker counts. Items 1 and 3 have tests. --- changelog.d/3170.security.md | 6 +++--- docs/SECURITY.md | 19 +++++++++++-------- messagefoundry/auth/reconcile.py | 6 ++++-- 3 files changed, 18 insertions(+), 13 deletions(-) diff --git a/changelog.d/3170.security.md b/changelog.d/3170.security.md index 2795c080d7..44397d0f6f 100644 --- a/changelog.d/3170.security.md +++ b/changelog.d/3170.security.md @@ -4,7 +4,7 @@ field in place of `probed`, and that row, the referral's latched message and its `ad_reconcile_aborted` alert count only the lookups the pass sent. The breaker's own alert still counts every account it judged. Such a row is also struck now on a pass that is a directory - outage or got only referrals, as on any other pass. Such a pass still revokes nothing, so the - row's sessions end at the first pass after it that is not aborted, and only at their own expiry - while every pass aborts. Its revocation still counts toward the mass-revoke breaker, as before. + outage or got only referrals, as on any other pass. Such a pass still revokes nothing. Only a + pass that is not aborted can, once the row's strikes reach the threshold and the row is in that + pass's probe budget, so while every pass aborts its sessions last to their own expiry. Its revocation still counts toward the mass-revoke breaker, as before. (`vault BACKLOG #3170`) diff --git a/docs/SECURITY.md b/docs/SECURITY.md index b042428a3d..731c76e824 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -2425,10 +2425,12 @@ with no page. `account_created` notice. - **A directory grant.** An account can get Administrator because the *directory* added it to a group already mapped to Administrator. The engine sees that grant in one of the two places below, - whichever comes first. Either one raises `administrator_granted`, unless a sign-in that sees it - first fails with a server error after the role write; the retry then gains nothing. An account - that never signs in again and holds no live session raises nothing. So watch membership of the - mapped group in the directory itself. + whichever comes first. Either one raises `administrator_granted`, but at least these cases raise + none. A sign-in that sees it first and fails with a server error after the role write loses it; + the retry then gains nothing. A reconciler pass the mass-revoke breaker aborts writes no roles. + An account that never signs in again raises nothing when no pass reaches it: it holds no live + session, or `[auth].ad_session_recheck_seconds` is 0. So watch membership of the mapped group in + the directory itself. **CORRECTED 2026-10-01:** this read "raises no alert. The engine never sees that grant." **CORRECTED 2026-10-06:** this read "raises no `administrator_granted` alert", and the sign-in case below read "It raises no alert." Vault BACKLOG #2610 added the sign-in alert. @@ -4118,8 +4120,8 @@ writes no roles, and it strikes and revokes as an absent account does, with the 0195 hold. It is never asked, so it is no part of an outage either: a pass whose every lookup fails still audits `auth.ad_reconcile_skipped`, and that row's `asked` count leaves it out. Such a pass, and one whose every lookup was referred, still strikes the row, because its answer never came from -the directory (BACKLOG #3170). Neither pass revokes it: that waits for the next pass that is not -aborted. A referral's `auth.ad_reconcile_referred` row and alert count only +the directory (BACKLOG #3170). Neither pass revokes it: only a pass that is not aborted can, once +the row's strikes reach the threshold and the row is in that pass's probe budget. A referral's `auth.ad_reconcile_referred` row and alert count only the lookups sent too. Its revocation does count toward the mass-revoke breaker below. Such a row cannot take a federated binding: the bind refuses it, so every binding the bind has made since BACKLOG #1143 slice C sits on a row probed by its id (ADR 0184 AC-5). A binding already on an id-less @@ -4134,7 +4136,8 @@ Removing the binding (`DELETE /users/{user_id}/federated-identity`) returns the The pass has these safety properties, because the lookup still returns one indistinguishable "not found" for *deleted*, *moved out of the search base* and *the search base was never right*: -- **Fail-OPEN.** An unreachable domain controller revokes **nothing** and does not even accrue a strike. +- **Fail-OPEN.** An unreachable domain controller revokes **nothing**, and an account the pass asked + about accrues no strike. An id-less row, never asked, is still struck (BACKLOG #3170). A fail-closed re-check would turn a directory blip into a total console outage during exactly the incident when operators need the console. - **Two strikes** (`ad_session_recheck_strikes`, default 2) before any revocation. @@ -4192,7 +4195,7 @@ refers on every pass. Read as an outage, it would stop revocation on a first dep page. The two bases fail differently. A referring user search base refers every probe that searches, -so the pass judges nothing and revokes nothing, like an outage, but it still pages. A referring +so the pass judges no account it asked about and revokes nothing, like an outage, but it still pages. A referring group search base refers only the accounts the user search found enabled. Disables and deletions still revoke, but a role or channel-scope change in the directory does not reach those accounts' sessions while it diff --git a/messagefoundry/auth/reconcile.py b/messagefoundry/auth/reconcile.py index 230586d7b3..d32146136c 100644 --- a/messagefoundry/auth/reconcile.py +++ b/messagefoundry/auth/reconcile.py @@ -33,7 +33,8 @@ A referral is not an outage, and not a reason to stop the pass either (BACKLOG #2538). A :attr:`ProbeOutcome.REFERRED` account is left unjudged, like an unreachable one, and every other probe in the pass is judged as usual. Unlike an outage, a referral alerts. A pass whose only answers are -referrals and failures aborts as ``directory_referral``, which judges nothing and alerts. +referrals and failures aborts as ``directory_referral``, which judges no account it asked about +and alerts. Such a pass, like an outage, still strikes an UNKEYED row (BACKLOG #3170). The pass also re-diffs channel scope (ADR 0198, BACKLOG #1957). It revokes when the directory would withdraw or narrow a principal's scope, and it never writes the scope: the next login does, through @@ -258,7 +259,8 @@ def directory_referral(self) -> bool: def judged_nothing(self) -> bool: """True when the pass aborted before judging any account: an outage, or a pass of referrals and failures only. Such a pass leaves the hold's state, its message and every clear alone, - because it learned nothing about the accounts.""" + because it learned nothing about the accounts it asked about. It still strikes an UNKEYED + row, which was judged without the directory (BACKLOG #3170).""" return self.directory_outage or self.directory_referral @property From 71addac55377cd26607319e3eb8260b39e1579ac Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Sat, 17 Oct 2026 00:08:29 -0500 Subject: [PATCH 28/57] Review round one on the b257-dchain merge: count and strike wording Applies round-one findings that concern text and tests the merge itself wrote; nothing here changes behaviour. - pipeline/alerts.py, docs/PHI.md: the ad_reconcile_aborted count for a probe fault is no longer called "judged". It is the same total as the breaker's, the candidates the pass took up, asked or not, which counts the faulted accounts left unjudged. - docs/SECURITY.md, changelog.d/3170.security.md: an id-less row is struck on any pass that aborts on referrals, probe faults or failures, not only on one whose every lookup was referred or faulted. - tests/test_ad_session_reconcile.py: the unkeyed-strike cases pin the abort slug, so a faults-only pass cannot pass as an outage. Left open, for the Manager: the probe fault's alert and its auth.ad_reconcile_faulted row count plan.probed while its latched message counts plan.asked (main's code, PR 2342); #3170's rule for the referral would apply to it unchanged. Proposed PR title: b257 D1-D3 auth follow-ups: TOTP refusal row, directory notices, revoke-miss audit, OIDC step-up follow-ups, sign-in timeout, reconcile admin grant and unkeyed counts --- changelog.d/3170.security.md | 3 ++- docs/PHI.md | 2 +- docs/SECURITY.md | 4 ++-- messagefoundry/pipeline/alerts.py | 13 ++++++----- tests/test_ad_session_reconcile.py | 36 ++++++++++++++++++------------ 5 files changed, 34 insertions(+), 24 deletions(-) diff --git a/changelog.d/3170.security.md b/changelog.d/3170.security.md index 7a0474d49c..9008375017 100644 --- a/changelog.d/3170.security.md +++ b/changelog.d/3170.security.md @@ -4,7 +4,8 @@ field in place of `probed`, and that row, the referral's latched message and its `ad_reconcile_aborted` alert count only the lookups the pass sent. The breaker's own alert still counts every account it judged. Such a row is also struck now on a pass that is a directory - outage or got only referrals or probe faults, as on any other pass. Such a pass still revokes nothing. Only a + outage or aborted on referrals or probe faults, as on any other pass. Such a pass still revokes + nothing. Only a pass that is not aborted can, once the row's strikes reach the threshold and the row is in that pass's probe budget, so while every pass aborts its sessions last to their own expiry. Its revocation still counts toward the mass-revoke breaker, as before. (`vault BACKLOG #3170`) diff --git a/docs/PHI.md b/docs/PHI.md index 4327544ea5..c44aa755b5 100644 --- a/docs/PHI.md +++ b/docs/PHI.md @@ -1808,7 +1808,7 @@ with materially different PHI profiles, so they get their own rows; stream 4 is | **5. `audit_log` table** (SQLite, Postgres, SQL Server) | who / what / **where-from** / when of auth + PHI *access* and admin actions — plus, while `[security].audit_all_authorization_decisions` is on (**default `true`** since BACKLOG #1277, 2026-09-02, which reversed the ADR 0118 §5 `false`; the internal field it desugars to is `audit_all_authz`, whose old `[diagnostics]` TOML spelling is **refused at load** — ADR 0118), an `authz` row for **every** authorization decision including successes. That is the shipped volume of this stream, not an opt-in addition to it: one row per authenticated request on each `require()`-gated route, and this row's retention cell records that **nothing prunes the table** — `actor`, `action`, `channel_id`, `client`, `detail`, `row_hash` | JSON `detail`; **tamper-evident hash chain** over `prev_hash` + the row (the `client` address is **inside** the chained payload — ADR 0150) | the store database | HIPAA §164.312(b) audit controls; incident response; `verify_audit_chain` integrity checks | `GET /audit` requires **`audit:read`**; `GET /audit/export` requires the separate **`audit:export`** and streams CSV with formula-injection neutralisation, recording its own `audit.export` row *before* streaming; `GET /me/security-events` is a per-user view of the same table | **`[retention].audit_days` is reserved and NOT enforced — keep-forever by design** (the audit-retention requirement, ~6 years — **not** chain-breakage; [§8](#8-retention--purge) states the position and cites its source of record) | `detail` is stored **in the clear** (it is not a cipher-covered column): its protection is that writers store at least filter shapes, counts, ids, keyed digests, bounded engine-authored text and client addresses, the `audit_log.detail` row above says which — never bodies or credentials — plus the store ACL and the volume layer. *Corrected 2026-10-16 (vault BACKLOG #3145):* this said writers "only ever store filter shapes, counts and ids" | | **6. `message_events` table** | the per-message disposition timeline — the **complete** vocabulary is `received`, `routed`, `unrouted`, `filtered`, `transformed`, `delivered`, `failed`, `dead`, `error`, `replayed`, `resent`, `reingressed`, `passthrough`, `passthrough_dropped`, `cancelled`, `edit_resend`, `edit_resubmit`, `viewed`, `not_deployed`, the ADR 0154 synchronous-reply pair `reply_returned` / `reply_timeout` (names, counts and `waited_ms` only — **never** a fragment of the partner's reply body), and `reply_dropped`, a delivery whose 2xx reply body was over the byte bound and dropped (the status and the bound only, vault BACKLOG #3040) (CI asserts this list against the engine's own `MESSAGE_EVENT_KINDS`). `[diagnostics].message_events` can thin the set, but never below the compliance floor `viewed` / `dead` / `error` / `failed` / `not_deployed` / `reply_timeout` / `reply_dropped` | rows: `message_id`, `ts`, `event`, `destination`, `detail` | the store database | operator timeline on the message-detail view; the `viewed` row is the HIPAA PHI-access record | `GET /messages/{id}` under **`messages:view_raw`** + `require_phi_read`; the read itself writes a `viewed` event **and** a `message_view` audit row | no dedicated window — `purge_message_bodies` sets `message_events.detail` to `NULL` in the same transaction that blanks the body, so it inherits `[retention].messages_days` | `detail` is `safe_text()`-scrubbed **then** cipher-encrypted (AAD `("message_events","detail",message_id,ts,event)`). Verbosity gate `[diagnostics].message_events` = `all` (default) / `errors` / `off`, with a **compliance floor that can never be thinned**: `viewed`, `dead`, `error`, `failed`, `not_deployed`, `reply_timeout`, `reply_dropped` are retained at every level (`reply_timeout` is the one row that explains a "we called you and got a 504" complaint, so an instance that thinned its logs would lose exactly the record it is later asked for; `reply_dropped` is the one row that answers "you ignored our reply", for the same reason) | | **7. `connection_event` table — DEFAULT ON** (`[diagnostics].connection_events = true`) | transport/lifecycle events per connection: `established`, `closed` (reason `eof`, `idle_timeout` or `frame_deadline` — a fixed vocabulary chosen by the listener, never free text), `idle_timeout`, `at_capacity`, `peer_not_allowlisted`, `frame_oversize`, `framing_error`, `peer_reset`, the MLLP listener's `handler_error` (BACKLOG #1619 -- the inbound handler faulted on a frame the listener read cleanly, such as a store outage at the ingress commit; the reason is scrubbed at the source like every other), the MLLP listener's `tls_handshake_failed` (vault BACKLOG #2613 -- a TLS handshake on a listener that runs its own handshakes (the standard asyncio loop, not uvloop) failed; the first at once, then one per listener per minute carrying the count since the last, and the reason is a count and an exception class name, never the peer's bytes), the DATABASE poll source's `row_undecodable` (BACKLOG #1662 — a polled row the source cannot turn into a body; the reason is scrubbed at the source and names the exception type or the operator's own configured column, never a row value), the FILE poll source's quarantine kinds `file_oversize` / `file_decompress_failed` / `file_content_mismatch` / `file_scan_rejected` (BACKLOG #1621 — a drop moved to `.error`; the reason is a size, a content type or a scrubbed codec or scanner message, never the file name), the inbound-HTTP intake-auth refusals `intake_auth_failed` / `auth_subject_denied` / `auth_rate_limited` (ADR 0154 D6 — peer address and mode only; **never** the credential, a prefix of it, or its length. Each of these also writes a tamper-evident audit-log row — the copy that survives an operator turning this diagnostics stream off), plus the runner's `connection_lost` / `connection_restored`. That is the whole vocabulary, asserted in CI against the literal emit call sites in `transports/` and the pipeline runner **and** cross-checked against the console's own filter tuple. The MLLP, raw-TCP, X12 and HTTP listeners emit these; the **DATABASE poll source** emits `row_undecodable` and nothing else (it dials out, so there is no lifecycle to report and its rows carry a **NULL peer column**); the **FILE poll source** emits only its four quarantine kinds, also with a NULL peer column; the **DICOM inbound C-STORE SCP** emits none — the runner injects the sink onto **every** source (`wiring_runner.py`, over the base-class `on_connection_event` field), so that connector *has* the wiring and simply never calls it — so this stream covers those four listeners plus those poll-source events and the runner's outbound-lane transitions — not literally every connection. A DICOM association's connects and refusals are therefore **absent** from this stream. The **`ISA`/`IEA`-framed X12 inbound** was equally absent until BACKLOG #1665 and is not any more: it emits the same seven kinds as the raw-TCP listener | rows: `ts`, `connection`, `transport`, `direction`, `kind`, `peer_host`, `message_id` (correlation hint), `reason` | the store database, **all three backends** | Corepoint-style transport diagnostics — "did the sender connect, and why did it drop" | `GET /events` and `GET /connections/{name}/events` under **`monitoring:read`** (**not** a PHI permission) with per-channel RBAC — an out-of-scope `connection=` is 403'd *and* audited — server-clamped to ≤1000 rows. `reason` alone is gated on `messages:view_summary` and masked as `****` until the audited per-event `reveal=` act (BACKLOG #2443; the `connection_event.reason` row of §3's PL-2 block says how) | `[retention].connection_event_retention_hours` (its own **hours** window); 0 inherits `[retention].messages_days`; both 0 = keep forever. Plain age `DELETE` (metadata-only) | **`reason` is free text that can carry sensitive fragments.** Defended twice — `safe_exc()` at the source, `safe_text(reason, limit=200)` at the store (the bound §3's PL-2 block states) — then cipher-encrypted (AAD `("connection_event","reason",connection,ts,kind)`). Every other column is config metadata; the table is documented **metadata-only** — never a frame, body or HL7 field value. Writes are a pure side observer: a bounded in-memory queue drained by a background task outside any handoff transaction, so a flood can never block a listener or pin a message disposition | -| **8. `alert_instance` table — default on wherever an `[alerts]` notifier exists** | resolvable operator alerts: `connection_stopped`, `queue_buildup`, `lane_stuck`, `message_stall`, `saturation`, `connection_error`, `storage_threshold`, `cert_expiry`, `secret_rotation`, `initial_credential_expiring` (an UNCLAIMED admin-issued temporary password nearing the instant the login gate stops accepting it -- ASVS 6.4.5, BACKLOG #1141; keyed on `user:`, and the payload is the ISO deadline plus whole hours remaining, repeated in the reason column, never the password), `integrity_drift`, `update_available`, `backup_failed`, `store_privilege_warning` (the store privilege preflight found the store principal over-granted or could not read it -- BACKLOG #305; the payload is the finding, a count and the preflight's summary line, which names the login, the database and role names, never a password or message content), `leadership_acquired`, `dr_activated`, `log_write_failed` (an application-log sink was rolled after a write failure, or is UNWRITABLE and this process's connections were stopped -- BACKLOG #122, ADR 0162; the payload is the sink LABEL, the stage, a `safe_exc` reason and a count of connections stopped, never the record whose write failed), `log_forward_failed` (the off-box log forwarder is absent, losing records or not sending -- BACKLOG #2612; keyed on `forwarder:`, with `@node:` or `@shard:` appended when several engine processes share the store, and the payload is that key, fixed reason words and a count, never a record, the collector's address or an error text; an unpinned cluster node's `node_id` holds this engine's own host name and process id, so the key carries them), `gcm_invocations` (the per-key AES-GCM invocation bound crossing its 2^31 soft warn — ASVS 11.3.4; its payload carries a one-way `key_id` fingerprint plus counters, never key bytes), `approval_stale_requester` (a dual-control release refused because the requester no longer holds the authority the operation needs -- ASVS 8.3.2, BACKLOG #289; keyed on the approval id, and the payload is that id, the operation key and a closed-set reason slug, never the requester's name or the captured params), `approval_too_early` (a dual-control release refused because it arrived before `[approvals].min_dwell_seconds` -- ASVS 2.4.2, BACKLOG #287; keyed on `approval:`, and the payload is that key, the operation key and a fixed reason string, never the approver's or requester's name or the captured params), `audit_write_failed` (the approval gate could not write one of its audit rows -- vault BACKLOG #2255; keyed on `approval:`, or on `config_reload:inline` when an ungated config reload could not write its `config_reload_attempted` row -- vault BACKLOG #2254; the payload is that key and the lost row's audit action name, never the row's detail, a username or the captured params; the ERROR log line beside it holds the detail on some paths and only the exception on others), `ad_reconcile_aborted` (a directory reconciler pass left signed-in accounts unrevoked and needs an operator: the mass-revoke breaker tripped and nothing was revoked, or one or more probes were answered with an LDAP referral and those accounts were left unjudged while the rest of the pass was applied, BACKLOG #2538, or one or more probes raised a fault the engine could not read and those accounts were left unjudged the same way, BACKLOG #3100; the payload is a fixed source label, `directory-reconciler` for the breaker, `directory-reconciler-referral` for a referral and `directory-reconciler-probe-fault` for a probe fault so each is its own instance, the closed-set reason slug `mass_revoke_breaker`, `directory_referral` or `probe_fault`, a count (the accounts the breaker or the probe fault judged, or for a referral the lookups the pass sent, which leaves out an id-less row -- vault BACKLOG #3170) and the latched operator explanation, never a username), `ad_reconcile_held` (the reconciler is holding accounts whose `userAccountControl` it cannot read, ADR 0195; the payload is the same fixed source label, a closed-set reason slug, the count of undetermined accounts and the latched operator explanation, never a username), `ad_session_revoked` (the reconciler revoked a directory principal's sessions; keyed on that operator account's username, with at least a `directory_absent`, `directory_disabled`, `directory_undetermined`, `directory_object_id_missing`, `roles_changed` or `scope_changed` reason), `approval_approver_provenance` (a dual-control release, or the resolution of an interrupted one, went ahead with an approver or resolver account created, re-passworded or TOTP-enrolled after the request -- BACKLOG #315, #2125; keyed on `approval:`, or `approval::resolve` for a resolution, and the payload is the operation key and the closed-set change slugs, never a username or the captured params), `administrator_granted` (the built-in Administrator role was granted by account creation, a role change, a directory group newly mapped to it -- BACKLOG #315 --, a directory sign-in whose role sync newly gave the account the role -- vault BACKLOG #2610 -- or a directory reconcile pass whose role re-diff did -- vault BACKLOG #3141; keyed on `user:` or `ad-group:`, with the grant route and the granting administrator's username, or `` for a directory sign-in or reconcile pass), `no_enabled_administrator` (a directory sign-in's role sync or the session reconciler took Administrator from its last enabled holder -- vault BACKLOG #3229; keyed on `user:`, and the payload is `via`, `ad-sync` or `ad-reconcile`, and a fixed reason sentence, never message content), `intake_paused` (the engine paused intake because the staged backlog went over `[inbound].max_staged_depth` or the SQLite volume fell below `[retention].min_free_disk_mb` -- BACKLOG #290; keyed on `intake:staged_depth` or `intake:disk_floor`, with `@node:` or `@shard:` appended when several engine processes share the store (BACKLOG #2272), and the payload is the reason, a count or a MiB figure, its limit and the store backend's name, never message content), `config_changed` (a start loaded a config whose fingerprint differs from the store's baseline -- vault BACKLOG #2597; keyed on `config:`, and the payload is the two digests, the node and engine shard labels, and the action, actor username and time of the row the older digest came from, never the config directory, a git commit or message content), `crl_reload_refused` (the CRL reload refused a replaced CRL file, so running TLS contexts keep the copy they hold -- vault BACKLOG #2904; keyed on `crl_reload:`, with `@node:` or `@shard:` appended when several engine processes share the store, and the payload is that key, the file's path, a short class token for the remedy, a count of contexts, the engine's own fixed remedy sentence and, when a start would refuse the file too, a `start_refuses` flag, never the refusal's reason, which quotes CRL issuer names and dates; that reason is in the application log, and in the `detail` of the CRL expiry alert once the held copy nears its `nextUpdate`), `remote_debug_refused` (the engine's remote-debugging guard refused scripts that another process injected into the engine process -- vault BACKLOG #2742; keyed on `remote_debug:refused`, with `@node:` or `@shard:` appended when several engine processes share the store, and the payload is that key, the count of scripts refused since the alert was last raised, the count since the process started and a fixed sentence built from the two, never a script's file name, which the injecting process chooses; that name is in the application log, in the guard's WARNING line), and the six security signals of the audit-stream rule layer -- vault BACKLOG #2613: `signin_failure_burst` (keyed on `signin:`, never the typed username), `access_denied_burst`, `body_view_burst` and `bulk_export` (each keyed on `account:`), `log_level_debug` (keyed on `logging:debug`) and `posture_loosened` (keyed on `posture:start`); the payload is a count and a fixed sentence holding a level name, switch names or a username, never a message body, a message id or an audit row's detail. The reachable **inverse** signals — `connection_restored`, `leadership_lost`, `dr_released`, `store_privilege_clean`, `intake_resumed`, `crl_reload_applied` (it resolves `crl_reload_refused` for the same file on the same engine process, once the reload's record no longer holds the refusal, and when the reload stops -- vault BACKLOG #2904), `ad_reconcile_breaker_cleared`, `ad_reconcile_hold_released` (the last two resolve `ad_reconcile_aborted`, under the breaker's or the referral's source but never the probe fault's, which has no inverse, and `ad_reconcile_held`; when they do, and where they never do, is stated once in [CONFIGURATION.md](CONFIGURATION.md#when-the-reconcilers-two-alerts-resolve-themselves) -- BACKLOG #2136) — are never rows here: `_record_state` routes an inverse through `_AUTO_RESOLVE` to `resolve_alert_instances_for`, never to `upsert_alert_instance`. (One more mapped key, `connection_started`, is emitted by no code path today.) | rows: `event_type`, `connection`, `severity`, `status`, `first_seen`, `last_seen`, `count`, `reason`, `acked_by`, `acked_at`, `resolved_at`, `suspended_until`, `escalation_tier` | the store database, **all three backends** | the operator alert list — acknowledge / resolve / suspend. Durable state is recorded **before** any suppression or throttle return, so a muted alert still leaves a record | `GET /alerts/active` under **`monitoring:diagnose`** (**not** a PHI permission) with the same per-channel scope, and `reason` alone gated on `messages:view_summary` and masked as `****` until the audited per-alert `reveal=` act (BACKLOG #2443; the `alert_instance.reason` row of §3's PL-2 block says how); ack/resolve/suspend/**resume** are POSTs on the same tier, and the separate read-only `GET /alerts/rules` view sits on its own gate | shares the connection-event window; **only RESOLVED instances are DELETEd**, by `resolved_at` — an open or acknowledged condition is never aged out from under an operator | **`reason` is free text** taken from the event's `detail`/`reason`: `safe_text(reason, limit=200)` (the bound §3's PL-2 block states) then cipher-encrypted (AAD `("alert_instance","reason",event_type,connection)` — the de-dup grain, so one AAD covers both the INSERT and the re-fire UPDATE). | +| **8. `alert_instance` table — default on wherever an `[alerts]` notifier exists** | resolvable operator alerts: `connection_stopped`, `queue_buildup`, `lane_stuck`, `message_stall`, `saturation`, `connection_error`, `storage_threshold`, `cert_expiry`, `secret_rotation`, `initial_credential_expiring` (an UNCLAIMED admin-issued temporary password nearing the instant the login gate stops accepting it -- ASVS 6.4.5, BACKLOG #1141; keyed on `user:`, and the payload is the ISO deadline plus whole hours remaining, repeated in the reason column, never the password), `integrity_drift`, `update_available`, `backup_failed`, `store_privilege_warning` (the store privilege preflight found the store principal over-granted or could not read it -- BACKLOG #305; the payload is the finding, a count and the preflight's summary line, which names the login, the database and role names, never a password or message content), `leadership_acquired`, `dr_activated`, `log_write_failed` (an application-log sink was rolled after a write failure, or is UNWRITABLE and this process's connections were stopped -- BACKLOG #122, ADR 0162; the payload is the sink LABEL, the stage, a `safe_exc` reason and a count of connections stopped, never the record whose write failed), `log_forward_failed` (the off-box log forwarder is absent, losing records or not sending -- BACKLOG #2612; keyed on `forwarder:`, with `@node:` or `@shard:` appended when several engine processes share the store, and the payload is that key, fixed reason words and a count, never a record, the collector's address or an error text; an unpinned cluster node's `node_id` holds this engine's own host name and process id, so the key carries them), `gcm_invocations` (the per-key AES-GCM invocation bound crossing its 2^31 soft warn — ASVS 11.3.4; its payload carries a one-way `key_id` fingerprint plus counters, never key bytes), `approval_stale_requester` (a dual-control release refused because the requester no longer holds the authority the operation needs -- ASVS 8.3.2, BACKLOG #289; keyed on the approval id, and the payload is that id, the operation key and a closed-set reason slug, never the requester's name or the captured params), `approval_too_early` (a dual-control release refused because it arrived before `[approvals].min_dwell_seconds` -- ASVS 2.4.2, BACKLOG #287; keyed on `approval:`, and the payload is that key, the operation key and a fixed reason string, never the approver's or requester's name or the captured params), `audit_write_failed` (the approval gate could not write one of its audit rows -- vault BACKLOG #2255; keyed on `approval:`, or on `config_reload:inline` when an ungated config reload could not write its `config_reload_attempted` row -- vault BACKLOG #2254; the payload is that key and the lost row's audit action name, never the row's detail, a username or the captured params; the ERROR log line beside it holds the detail on some paths and only the exception on others), `ad_reconcile_aborted` (a directory reconciler pass left signed-in accounts unrevoked and needs an operator: the mass-revoke breaker tripped and nothing was revoked, or one or more probes were answered with an LDAP referral and those accounts were left unjudged while the rest of the pass was applied, BACKLOG #2538, or one or more probes raised a fault the engine could not read and those accounts were left unjudged the same way, BACKLOG #3100; the payload is a fixed source label, `directory-reconciler` for the breaker, `directory-reconciler-referral` for a referral and `directory-reconciler-probe-fault` for a probe fault so each is its own instance, the closed-set reason slug `mass_revoke_breaker`, `directory_referral` or `probe_fault`, a count (the accounts the breaker judged; for a probe fault the same total of candidates the pass took up, asked or not, faulted ones included; or for a referral the lookups the pass sent, which leaves out an id-less row -- vault BACKLOG #3170) and the latched operator explanation, never a username), `ad_reconcile_held` (the reconciler is holding accounts whose `userAccountControl` it cannot read, ADR 0195; the payload is the same fixed source label, a closed-set reason slug, the count of undetermined accounts and the latched operator explanation, never a username), `ad_session_revoked` (the reconciler revoked a directory principal's sessions; keyed on that operator account's username, with at least a `directory_absent`, `directory_disabled`, `directory_undetermined`, `directory_object_id_missing`, `roles_changed` or `scope_changed` reason), `approval_approver_provenance` (a dual-control release, or the resolution of an interrupted one, went ahead with an approver or resolver account created, re-passworded or TOTP-enrolled after the request -- BACKLOG #315, #2125; keyed on `approval:`, or `approval::resolve` for a resolution, and the payload is the operation key and the closed-set change slugs, never a username or the captured params), `administrator_granted` (the built-in Administrator role was granted by account creation, a role change, a directory group newly mapped to it -- BACKLOG #315 --, a directory sign-in whose role sync newly gave the account the role -- vault BACKLOG #2610 -- or a directory reconcile pass whose role re-diff did -- vault BACKLOG #3141; keyed on `user:` or `ad-group:`, with the grant route and the granting administrator's username, or `` for a directory sign-in or reconcile pass), `no_enabled_administrator` (a directory sign-in's role sync or the session reconciler took Administrator from its last enabled holder -- vault BACKLOG #3229; keyed on `user:`, and the payload is `via`, `ad-sync` or `ad-reconcile`, and a fixed reason sentence, never message content), `intake_paused` (the engine paused intake because the staged backlog went over `[inbound].max_staged_depth` or the SQLite volume fell below `[retention].min_free_disk_mb` -- BACKLOG #290; keyed on `intake:staged_depth` or `intake:disk_floor`, with `@node:` or `@shard:` appended when several engine processes share the store (BACKLOG #2272), and the payload is the reason, a count or a MiB figure, its limit and the store backend's name, never message content), `config_changed` (a start loaded a config whose fingerprint differs from the store's baseline -- vault BACKLOG #2597; keyed on `config:`, and the payload is the two digests, the node and engine shard labels, and the action, actor username and time of the row the older digest came from, never the config directory, a git commit or message content), `crl_reload_refused` (the CRL reload refused a replaced CRL file, so running TLS contexts keep the copy they hold -- vault BACKLOG #2904; keyed on `crl_reload:`, with `@node:` or `@shard:` appended when several engine processes share the store, and the payload is that key, the file's path, a short class token for the remedy, a count of contexts, the engine's own fixed remedy sentence and, when a start would refuse the file too, a `start_refuses` flag, never the refusal's reason, which quotes CRL issuer names and dates; that reason is in the application log, and in the `detail` of the CRL expiry alert once the held copy nears its `nextUpdate`), `remote_debug_refused` (the engine's remote-debugging guard refused scripts that another process injected into the engine process -- vault BACKLOG #2742; keyed on `remote_debug:refused`, with `@node:` or `@shard:` appended when several engine processes share the store, and the payload is that key, the count of scripts refused since the alert was last raised, the count since the process started and a fixed sentence built from the two, never a script's file name, which the injecting process chooses; that name is in the application log, in the guard's WARNING line), and the six security signals of the audit-stream rule layer -- vault BACKLOG #2613: `signin_failure_burst` (keyed on `signin:`, never the typed username), `access_denied_burst`, `body_view_burst` and `bulk_export` (each keyed on `account:`), `log_level_debug` (keyed on `logging:debug`) and `posture_loosened` (keyed on `posture:start`); the payload is a count and a fixed sentence holding a level name, switch names or a username, never a message body, a message id or an audit row's detail. The reachable **inverse** signals — `connection_restored`, `leadership_lost`, `dr_released`, `store_privilege_clean`, `intake_resumed`, `crl_reload_applied` (it resolves `crl_reload_refused` for the same file on the same engine process, once the reload's record no longer holds the refusal, and when the reload stops -- vault BACKLOG #2904), `ad_reconcile_breaker_cleared`, `ad_reconcile_hold_released` (the last two resolve `ad_reconcile_aborted`, under the breaker's or the referral's source but never the probe fault's, which has no inverse, and `ad_reconcile_held`; when they do, and where they never do, is stated once in [CONFIGURATION.md](CONFIGURATION.md#when-the-reconcilers-two-alerts-resolve-themselves) -- BACKLOG #2136) — are never rows here: `_record_state` routes an inverse through `_AUTO_RESOLVE` to `resolve_alert_instances_for`, never to `upsert_alert_instance`. (One more mapped key, `connection_started`, is emitted by no code path today.) | rows: `event_type`, `connection`, `severity`, `status`, `first_seen`, `last_seen`, `count`, `reason`, `acked_by`, `acked_at`, `resolved_at`, `suspended_until`, `escalation_tier` | the store database, **all three backends** | the operator alert list — acknowledge / resolve / suspend. Durable state is recorded **before** any suppression or throttle return, so a muted alert still leaves a record | `GET /alerts/active` under **`monitoring:diagnose`** (**not** a PHI permission) with the same per-channel scope, and `reason` alone gated on `messages:view_summary` and masked as `****` until the audited per-alert `reveal=` act (BACKLOG #2443; the `alert_instance.reason` row of §3's PL-2 block says how); ack/resolve/suspend/**resume** are POSTs on the same tier, and the separate read-only `GET /alerts/rules` view sits on its own gate | shares the connection-event window; **only RESOLVED instances are DELETEd**, by `resolved_at` — an open or acknowledged condition is never aged out from under an operator | **`reason` is free text** taken from the event's `detail`/`reason`: `safe_text(reason, limit=200)` (the bound §3's PL-2 block states) then cipher-encrypted (AAD `("alert_instance","reason",event_type,connection)` — the de-dup grain, so one AAD covers both the INSERT and the re-fire UPDATE). | | **9. `response` rows with `kind='ack_sent'` — DEFAULT ON** (`[diagnostics].response_sent = true`) | the ACK/NAK the engine returned to an inbound sender, under a sentinel destination `\x1fack:` | rows: `ack_code` (`AA`/`AE`/`AR`/`CA`/`CE`/`CR`), `ack_phase` (`decode`/`parse`/`strict`/`ingest`), `outcome`, `body`, `detail` | the store database | "what did we actually reply, and why" — the operator's answer to a sender disputing an ACK | `GET /messages/{id}/responses` under `messages:read` + `require_phi_read`; the `body` only for a caller who also holds `messages:view_raw` and `messages:view_summary`; every read writes a `response.read` audit row | `body`, `detail` and `resp_headers` are set to `NULL` in place by `purge_message_bodies` on the message-body window, on all three backends | **PHI fail-safe:** the ACK **body** is stored **only when the store cipher is active** — on a keyless store it is `NULL` rather than plaintext — and every NAK passes no body at all, so the offending field value is never persisted. The disposition metadata (`ack_code`/`ack_phase`/`outcome`) is non-PHI and always captured; `detail` is `safe_text`-scrubbed, held to the 200-character bound §3's PL-2 block states, and encrypted | | **10. `[alerts]` webhook transport** (off by default — `webhook_url` unset) | one HTTPS POST per alert, carrying every non-underscore event key as JSON | JSON | the operator's webhook endpoint (Slack/Teams/PagerDuty/custom) | operator notification | **`https` only** — a plaintext `http://` webhook URL is refused at construction unless the `MEFOR_ALLOW_INSECURE_TLS` escape is set (and then a warning is logged); since #329 this path routes that escape through the clamped `weakened_tls_escape_permitted(posture)` (the instance posture threaded from the API lifespan), so on an enforcing-PHI instance the escape is inert and a cleartext webhook POST stays refused — the same clamp as the connectors, no longer the raw escape. Redirects are refused; an optional `webhook_allowed_hosts` egress allowlist gates the host | the endpoint's | **carries the alert's `detail`/`reason` free text** (`safe_exc()`-scrubbed at the emit sites, but **not** re-run through `safe_text` on this path). Internal `_`-prefixed keys (per-rule recipients, rule id, cooldown) are stripped before send, so recipient addresses never cross the wire | | **11. `[alerts]` SMTP transport — operator alert list** (off unless `email_smtp_host` + `email_from` + ≥1 `email_to`) | one email per alert; default subject `[MessageFoundry] — `, default body every non-underscore event key as `k: v` | plain text (always kept — never HTML-only); optional HTML alternative | the operators' mailboxes | operator notification | `smtp_allowed_hosts` egress allowlist; the SMTP password comes from `MEFOR_ALERTS_EMAIL_PASSWORD` or a `[secrets]` provider, never the config file; per-send timeout `email_timeout` | the mail system's | carries the same `detail`/`reason` free text as the webhook. #138 operator templates are constrained to a **closed non-PHI variable allowlist** validated fail-closed at config load. **Transport posture:** `send_plain_email` builds an explicit **verifying** context (chain + hostname + strict RFC 5280, TLS 1.2 floor) via `tls_policy.build_smtp_tls_context()` and passes it to `starttls()`, anchored to the OS roots, `[alerts].email_tls_ca_file`, or `[tls].internal_ca_file` — the same factory the EMAIL and DIRECT *message destinations* use, so all three SMTP cells now share one policy ([#323](BACKLOG.md), closed 2026-08-02). Before that this call passed **no** context and Python's stdlib default applied (`ssl._create_stdlib_context` **is** `ssl._create_unverified_context` — `CERT_NONE`, `check_hostname = False`), leaving the hop encrypted but unauthenticated. There is still **no hop gradient or attestation on this path** — unlike the connectors, this cell is constructed outside the `active_hop_posture` scope, so its deviations (`email_use_tls = false`, or `email_tls_verify = false`) are gated by a `[security].allow_unverified_alert_smtp_tls` **acknowledgment switch at the serve gate** rather than by the clamped escape: on an enforcing PHI instance `serve` refuses to start without it, and permits + `AUDIT`-logs the start with it. Both deviations are named by `security_loosenings()` and reported by `messagefoundry check`'s `alert-smtp-tls` advisory | diff --git a/docs/SECURITY.md b/docs/SECURITY.md index 3dfb569571..cc5008b5ea 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -4143,8 +4143,8 @@ writes no roles, and it strikes and revokes as an absent account does, with the `directory_object_id_missing`. It is not a `userAccountControl` answer, so it never moves the ADR 0195 hold. It is never asked, so it is no part of an outage either: a pass whose every lookup fails still audits `auth.ad_reconcile_skipped`, and that row's `asked` count leaves it out. Such a pass, -and one whose every lookup was referred or faulted, still strikes the row, because its answer never came from -the directory (BACKLOG #3170). Neither pass revokes it: only a pass that is not aborted can, once +and one that aborts because its lookups were only referred, faulted or failed, still strikes the +row, because its answer never came from the directory (BACKLOG #3170). No such pass revokes it: only a pass that is not aborted can, once the row's strikes reach the threshold and the row is in that pass's probe budget. A referral's `auth.ad_reconcile_referred` row and alert count only the lookups sent too. Its revocation does count toward the mass-revoke breaker below. Such a row cannot take a federated binding: the bind refuses it, so every binding the bind has made since diff --git a/messagefoundry/pipeline/alerts.py b/messagefoundry/pipeline/alerts.py index ffa60a9db5..8db451c29d 100644 --- a/messagefoundry/pipeline/alerts.py +++ b/messagefoundry/pipeline/alerts.py @@ -546,10 +546,11 @@ def ad_reconcile_aborted(self, name: str, *, reason: str, probed: int, detail: s ``name`` labels the source: ``"directory-reconciler"`` for the breaker, ``"directory-reconciler-referral"`` for a referral, and ``"directory-reconciler-probe-fault"`` for a probe fault, so each is its own instance and - throttle. ``reason`` is one of those three closed-set slugs. ``probed`` is, for the breaker - and for a probe fault, how many principals the pass judged, the breaker's denominator, and - for a referral how many probes the pass sent to the directory, which leaves out an id-less - row (BACKLOG #3170). ``detail`` is the operator-facing explanation the auth service + throttle. ``reason`` is one of those three closed-set slugs. ``probed`` is, for the breaker, + how many principals the pass judged, its denominator; for a probe fault, the same total of + candidates the pass took up, asked or not, which counts the faulted accounts it left + unjudged and any id-less row; and for a referral how many probes the pass sent to the + directory, which leaves out an id-less row (BACKLOG #3170). ``detail`` is the operator-facing explanation the auth service latches. No PHI. Emitted by the API-lifespan reconciler task, never from ``auth/``. A whole-directory outage is NOT this event: it is audited as ``auth.ad_reconcile_skipped`` and pages nothing, because the accounts are fine.""" @@ -1093,8 +1094,8 @@ def no_enabled_administrator(self, name: str, *, via: str) -> None: def ad_reconcile_aborted(self, name: str, *, reason: str, probed: int, detail: str) -> None: log.warning( - # The count is the judged total for the breaker and a probe fault, or a referral's probes - # asked (BACKLOG #3170). + # The count is the candidates the pass took up for the breaker and a probe fault, or a + # referral's probes asked (BACKLOG #3170); see AlertSink.ad_reconcile_aborted. "ALERT ad_reconcile_aborted: %r left accounts unrevoked, counting %d principal(s) " "(%s): %s", name, diff --git a/tests/test_ad_session_reconcile.py b/tests/test_ad_session_reconcile.py index 394d85806c..3fdac67123 100644 --- a/tests/test_ad_session_reconcile.py +++ b/tests/test_ad_session_reconcile.py @@ -664,20 +664,26 @@ def test_an_unkeyed_probe_does_not_hide_an_outage_or_a_referral_only_pass() -> N @pytest.mark.parametrize( - "asked", + ("asked", "aborted"), [ - [reconcile.Probe("a", "a", reconcile.ProbeOutcome.UNAVAILABLE)], - [reconcile.Probe("r", "r", reconcile.ProbeOutcome.REFERRED)], - [ - reconcile.Probe("a", "a", reconcile.ProbeOutcome.UNAVAILABLE), - reconcile.Probe("r", "r", reconcile.ProbeOutcome.REFERRED), - ], + ([reconcile.Probe("a", "a", reconcile.ProbeOutcome.UNAVAILABLE)], "directory_unavailable"), + ([reconcile.Probe("r", "r", reconcile.ProbeOutcome.REFERRED)], reconcile.REFERRAL_ABORT), + ( + [ + reconcile.Probe("a", "a", reconcile.ProbeOutcome.UNAVAILABLE), + reconcile.Probe("r", "r", reconcile.ProbeOutcome.REFERRED), + ], + reconcile.REFERRAL_ABORT, + ), # BACKLOG #3100 shares the referral's early return, so a pass of faults only is one too. - [reconcile.Probe("f", "f", reconcile.ProbeOutcome.FAULT)], - [ - reconcile.Probe("a", "a", reconcile.ProbeOutcome.UNAVAILABLE), - reconcile.Probe("f", "f", reconcile.ProbeOutcome.FAULT), - ], + ([reconcile.Probe("f", "f", reconcile.ProbeOutcome.FAULT)], reconcile.PROBE_FAULT_ABORT), + ( + [ + reconcile.Probe("a", "a", reconcile.ProbeOutcome.UNAVAILABLE), + reconcile.Probe("f", "f", reconcile.ProbeOutcome.FAULT), + ], + reconcile.PROBE_FAULT_ABORT, + ), ], ids=[ "outage", @@ -688,15 +694,17 @@ def test_an_unkeyed_probe_does_not_hide_an_outage_or_a_referral_only_pass() -> N ], ) def test_an_unkeyed_row_is_struck_on_a_pass_that_judged_nothing( - asked: list[reconcile.Probe], + asked: list[reconcile.Probe], aborted: str ) -> None: """BACKLOG #3170 item 3. An UNKEYED row is judged without the directory, so a pass that learned nothing from it still strikes the row. Only the strike: the asked probes' strikes, the outcome - record the hold reads, and the hold's latch all stay as they were. + record the hold reads, and the hold's latch all stay as they were. The abort slug is pinned + too, so an id-less row cannot hide which kind of pass this was (BACKLOG #2434, #3100). RED before #3170: both early returns carried no strikes, so such a row went unstruck.""" unkeyed = reconcile.Probe("k", "k", reconcile.ProbeOutcome.UNKEYED) plan = _plan([*asked, unkeyed], prior_strikes={"k": 1, "a": 1, "r": 1, "f": 1}, latched=True) + assert plan.aborted == aborted assert plan.judged_nothing assert plan.strikes == {"k": 2} assert plan.outcomes == {} and plan.latched and not plan.hold From 42dd6ebbef3f1bcac9ea38575963d24c58e3510d Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Sat, 17 Oct 2026 01:17:11 -0500 Subject: [PATCH 29/57] Reconciler: configuration refusals page, bind referrals refer, fault counts over asked (BACKLOG #3170) D3b, batch 257. Follow-ups of #3010 limb 5 and the #3170 merge. The #3010 claim is held by the batch Manager's PR worktree, so the subject names only #3170; this commit carries #3010 work too. #3010 A: noSuchObject, invalidDNSyntax and insufficientAccessRights (LdapUnansweredError.configuration) read as ProbeOutcome.MISCONFIGURED. Unjudged, never revokes. A pass whose every answer went unjudged, at least one refused and none referred, aborts as directory_misconfigured. Each such pass latches a message, logs one ERROR, audits auth.ad_reconcile_misconfigured and pages ad_reconcile_aborted under directory-reconciler-misconfigured, counted over asked. No inverse. #3010 B: a bind response carrying a referral (ldap3 KeyError(3) in decode_sequence with BIND_RESPONSE_CONTEXT) is LdapReferralError. #3010 D: authenticate and probe_principal unbind the service connection through _released. #3170: the fault's alert and auth.ad_reconcile_faulted row count asked. --- messagefoundry/api/app.py | 41 +++- messagefoundry/auth/ldap.py | 118 ++++++++--- messagefoundry/auth/reconcile.py | 175 ++++++++++++----- messagefoundry/auth/service.py | 195 +++++++++++++++---- messagefoundry/pipeline/alerts.py | 27 +-- tests/test_ad_directory_identity.py | 3 + tests/test_ad_session_reconcile.py | 259 ++++++++++++++++++++++++- tests/test_ad_user_account_control.py | 71 +++++++ tests/test_ldap_decode_boundary.py | 166 +++++++++++++++- tests/test_ldap_search_result_codes.py | 92 +++++++-- tests/test_ldap_timeouts.py | 11 +- tests/test_mfa_directory_recheck.py | 9 +- 12 files changed, 1009 insertions(+), 158 deletions(-) diff --git a/messagefoundry/api/app.py b/messagefoundry/api/app.py index 31c81b8862..473de2ac18 100644 --- a/messagefoundry/api/app.py +++ b/messagefoundry/api/app.py @@ -277,6 +277,7 @@ from messagefoundry.auth.audit_visibility import reads_audit_copies_in_the_log from messagefoundry.auth.reconcile import ( HOLD_REASON, + MISCONFIGURED_ABORT, PROBE_FAULT_ABORT, REFERRAL_ABORT, ReconcilePlan, @@ -9465,6 +9466,9 @@ async def _directory_reconciler( #: The source label of the probe fault's own ``ad_reconcile_aborted`` instance (BACKLOG #3100), apart #: from the breaker's and the referral's for the same reasons. _PROBE_FAULT_ALERT_SOURCE = "directory-reconciler-probe-fault" +#: The source label of the configuration refusal's own ``ad_reconcile_aborted`` instance (BACKLOG +#: #3010), apart from the others for the same reasons. +_MISCONFIGURED_ALERT_SOURCE = "directory-reconciler-misconfigured" def _alert_reconcile_plan(plan: ReconcilePlan, auth: AuthService, sink: AlertSink) -> None: @@ -9491,8 +9495,22 @@ def _alert_reconcile_plan(plan: ReconcilePlan, auth: AuthService, sink: AlertSin alert with reason ``probe_fault``, under its own source label (:data:`_PROBE_FAULT_ALERT_SOURCE`). It has no inverse: an operator resolves it. + A search the directory refused for its configuration (BACKLOG #3010) is not an outage either: a + search base that was deleted or moved, or that the service account may not read, ends every + search with noSuchObject, invalidDNSyntax or insufficientAccessRights on every pass. Any pass + with such a refusal is ``auth.ad_reconcile_misconfigured`` and becomes one + ``ad_reconcile_aborted`` alert with reason ``directory_misconfigured``, under its own source + label (:data:`_MISCONFIGURED_ALERT_SOURCE`). Like the fault's, it has no inverse. Before #3010 + such a base read as "not found" for every account, so it tripped the breaker on an estate + large enough; read as an outage, it would page nobody. + + The breaker's alert counts the accounts the pass judged, its denominator. The referral's, the + refusal's and the fault's count the probes the pass asked, which leaves out an id-less row + (BACKLOG #3170): only a probe that was sent can be referred, refused or raise a fault. + Reads the RETURNED plan, so it sees only a pass that finished. A failed write of the pass's own - held, aborted, skipped, referred or unkeyed-binding row no longer ends the pass (BACKLOG #2137): + held, aborted, skipped, referred, misconfigured, faulted or unkeyed-binding row no longer ends + the pass (BACKLOG #2137): the service logs it at ERROR and returns the plan, so these alerts still fire. A pass that raised part-way for another reason, such as inside one revocation, has already audited the revocations it applied before that point; those rows stand, and no alert is raised for them. @@ -9510,9 +9528,9 @@ def _alert_reconcile_plan(plan: ReconcilePlan, auth: AuthService, sink: AlertSin ``_mark_reconcile_clears`` states that rule once.""" if plan.directory_outage: return - if plan.aborted is not None and not plan.directory_referral and not plan.probe_fault: - # A breaker trip. A pass of referrals only is not one: its one alert is the referral's below. - # Nor is a pass of faults only (BACKLOG #3100): its one alert is the fault's below. + if plan.aborted is not None and not plan.judged_nothing: + # A breaker trip. A pass that judged nothing is not one: its alerts are the referral's, the + # refusal's or the fault's below (BACKLOG #2538, #3010, #3100), and an outage has none. sink.ad_reconcile_aborted( "directory-reconciler", reason=plan.aborted, @@ -9527,17 +9545,24 @@ def _alert_reconcile_plan(plan: ReconcilePlan, auth: AuthService, sink: AlertSin sink.ad_reconcile_aborted( _REFERRAL_ALERT_SOURCE, reason=REFERRAL_ABORT, - # The probes ASKED (BACKLOG #3170): an UNKEYED row was never sent, so no referral - # could answer it. The breaker's count above stays its judged denominator. + # The probes ASKED, as the refusal's and the fault's below (BACKLOG #3170). probed=plan.asked, detail=auth.directory_reconcile_referral or REFERRAL_ABORT, ) + if plan.misconfigured: + # BACKLOG #3010. Beside the referral's, in the same order the service records them. + sink.ad_reconcile_aborted( + _MISCONFIGURED_ALERT_SOURCE, + reason=MISCONFIGURED_ABORT, + probed=plan.asked, + detail=auth.directory_reconcile_misconfigured or MISCONFIGURED_ABORT, + ) if plan.faulted: - # BACKLOG #3100. Beside the referral's, in the same order the service records them. + # BACKLOG #3100. sink.ad_reconcile_aborted( _PROBE_FAULT_ALERT_SOURCE, reason=PROBE_FAULT_ABORT, - probed=plan.probed, + probed=plan.asked, detail=auth.directory_reconcile_fault or PROBE_FAULT_ABORT, ) if plan.hold: diff --git a/messagefoundry/auth/ldap.py b/messagefoundry/auth/ldap.py index cc33ae8f32..920b5bff45 100644 --- a/messagefoundry/auth/ldap.py +++ b/messagefoundry/auth/ldap.py @@ -137,6 +137,15 @@ def __init__( self.code = code self.code_name = code_name + @property + def configuration(self) -> bool: + """Whether the code points at the configuration rather than at a passing load + (:data:`_CONFIGURATION_CODES`): noSuchObject (32), invalidDNSyntax (34) and + insufficientAccessRights (50). Such a code recurs on every attempt until the configuration + changes, so the session reconciler reads it as misconfigured, which alerts, and not as + unavailable, which pages nobody (BACKLOG #3010).""" + return self.code in _CONFIGURATION_CODES + class LdapUndecodableError(LdapError): """ldap3 could not decode the directory's response to an operation (BACKLOG #3010). @@ -768,8 +777,14 @@ def _refuse_referral(conn: Any, operation: str) -> None: "the forest; use a base in this domain controller's own domain, or a global catalog, " "which carries universal-group membership only (ADR 0180 Amendment F)." ) + raise _referral_error(operation, message) + + +def _referral_error(operation: str, message: str) -> LdapReferralError: + """The referral refusal for ``operation``, warned once per operation, whether ldap3 decoded the + referral (:func:`_refuse_referral`) or could not (:func:`_referred_bind`).""" _warn_once(f"referral: {operation}", "%s Reported once per operation.", message) - raise LdapReferralError(message) + return LdapReferralError(message) #: What a refused code most likely means, where the code points at the configuration rather than at @@ -783,6 +798,11 @@ def _refuse_referral(conn: Any, operation: str) -> None: 50: " insufficientAccessRights means the service account may not read it.", } +#: The codes that point at the configuration rather than at a passing load +#: (:attr:`LdapUnansweredError.configuration`). Kept apart from the hints on purpose: a hint added +#: for a transient code must not make it page as a misconfiguration. +_CONFIGURATION_CODES = frozenset({32, 34, 50}) + def _refuse_unanswered(conn: Any, operation: str, *, answers: frozenset[int] | None = None) -> None: """Raise :class:`LdapReferralError` for a referral, and otherwise :class:`LdapUnansweredError` @@ -807,18 +827,17 @@ def _refuse_unanswered(conn: Any, operation: str, *, answers: frozenset[int] | N read as absence it would read every account absent at once, which the reconciler's mass-revoke breaker does not stop at or under its floor. - **What each caller makes of it.** The session reconciler reads an :class:`LdapError` as - unavailable, which never strikes; a sign-in writes ``auth.login_error``; the nested group search - raising means no caller sees a shorter group list. + **What each caller makes of it.** The session reconciler reads one of the configuration codes + (:attr:`LdapUnansweredError.configuration`) as misconfigured and any other as unavailable. + Neither strikes. A sign-in writes ``auth.login_error``; the nested group search raising means + no caller sees a shorter group list. - **The cost, named.** Unavailable never revokes, so a code that recurs on every pass, such as a - deleted search base, stops offboarding and the role re-diff for as long as it lasts, and nothing - latches an alert for it. Making a standing one visible is #3010's item 4, which the batch - deferred to vault BACKLOG #3100's item 3. As filed, that item names the reconciler's broad - ``except Exception`` arm, which this error does not reach, so it must widen to cover the - ``except LdapError`` arm before the deferral holds. This warns once per operation and code, and - the text carries no directory text: the code, ldap3's name for it and the operation, never the - diagnostic message the directory sent. + **The cost, named.** Neither reading revokes, so a code that recurs on every pass, such as a + deleted search base, stops offboarding and the role re-diff for as long as it lasts. A + configuration code alerts on every pass it answers, and a pass where nothing else answered + aborts as ``directory_misconfigured``. Any other code reads as an outage, which pages nobody. + This warns once per operation and code, and the text carries no directory text: the code, + ldap3's name for it and the operation, never the diagnostic message the directory sent. **A result that is not a dict is let through.** ldap3's synchronous strategy, the only one built here, reads ``self.result['type']`` straight after every search, so a real search cannot return @@ -876,7 +895,8 @@ def _search(conn: Any, operation: str, **kwargs: Any) -> None: #: What ldap3 raises, other than its own ``LDAPException``, when a response it reads is malformed. #: ldap3 2.9.1's default fast decoder indexes the BER it reads without checking it: a context tag a #: response may not carry is ``KeyError(tag)`` (``utils/asn1.py``, ``decode_sequence``), so a bind -#: response carrying a referral raises ``KeyError(3)``; a result code outside its table is +#: response carrying a referral raises ``KeyError(3)``, which :func:`_ldap3_call` refuses as a +#: referral (:func:`_is_bind_referral`); a result code outside its table is #: ``KeyError(code)`` (``ldap_result_to_dict_fast``, ``operation/bind.py``, #: ``operation/extended.py``); a paged results control with no value is ``TypeError`` and one with no #: cookie is ``IndexError`` (``strategy/base.py``, ``decode_control_fast``); a bind response whose @@ -945,15 +965,20 @@ def _ldap3_call(operation: str, *, close: bool = False) -> Iterator[None]: debug level (:func:`_reading_connection` says how the two are told apart). An ``LDAPException`` passes through untouched, for its own handler. + **One fault is a referral, not an undecodable response.** ldap3's decoder has no field for the + referral a bind response may carry, so it raises ``KeyError(3)`` on one. That becomes + :class:`LdapReferralError`, as a referral ldap3 could decode does (:func:`_refuse_referral`), + so the session reconciler reads it as referred and alerts (BACKLOG #2538). + :func:`_is_bind_referral` says how it is told apart, again by origin. + ``close`` closes the socket the failed read was on, where nothing else will. Only the service account's bind passes it: its ``auto_bind`` runs inside ldap3's ``Connection`` constructor, so - when it raises no caller holds the connection to unbind. A search does not (BACKLOG #3010). - ``authenticate`` and ``probe_principal`` hold the service connection in ldap3's own ``with``, - and ldap3 2.9.1's ``Connection.__exit__`` reopens a connection that was open on entry and is - closed on exit. So a close there shut the failed socket and then cost one more connect for a - fresh, unbound one. The cost of not closing: after an undecodable search response, those two - callers' connection stays open and bound, on a stream that may be out of step, until it is - collected. ``read_bind_account`` unbinds its own in a ``finally``. + when it raises no caller holds the connection to unbind. A search does not (BACKLOG #3010): its + caller holds the connection, and every caller here releases it, an undecodable response + included. ``authenticate`` and ``probe_principal`` release theirs through :func:`_released`, and + ``read_bind_account`` in a ``finally``. None relies on ldap3's own ``with``, whose 2.9.1 + ``Connection.__exit__`` neither unbinds a connection that was bound on entry, as ``auto_bind`` + makes every one here, nor leaves one closed that was open on entry. """ import ldap3 @@ -974,9 +999,54 @@ def _ldap3_call(operation: str, *, close: bool = False) -> Iterator[None]: logger.debug( "AD socket close after an undecodable response: %s", _type_name(failed) ) + if _is_bind_referral(exc): + raise _referred_bind(operation) from exc raise _undecodable(operation, exc) from exc +#: The context tag of the ``referral`` field of an LDAPResult (RFC 4511 section 4.1.9). +_REFERRAL_TAG = 3 + + +def _is_bind_referral(exc: BaseException) -> bool: + """Whether ``exc`` is ldap3's decoder meeting the referral field of a bind response. + + Decided by ORIGIN and decoder state, never by the exception's value alone: the innermost frame is + ldap3's ``decode_sequence`` reading a bind response's fields (its ``context_decoders`` is + ``BIND_RESPONSE_CONTEXT``, which has no entry for the field) at the referral's tag. A + ``KeyError(3)`` from anywhere else, such as a stray tag [3] in some other response or engine + code, is not this. A test pins the ldap3 names this reads. + """ + if not isinstance(exc, KeyError): + return False + from ldap3.utils import asn1 # lazy, like every ldap3 import here + + tb = exc.__traceback__ + if tb is None: + return False + while tb.tb_next is not None: + tb = tb.tb_next + frame = tb.tb_frame + return ( + frame.f_globals.get("__name__") == asn1.__name__ + and frame.f_code.co_name == "decode_sequence" + and frame.f_locals.get("context_decoders") is asn1.BIND_RESPONSE_CONTEXT + and frame.f_locals.get("ber_type") == _REFERRAL_TAG + ) + + +def _referred_bind(operation: str) -> LdapReferralError: + """The refusal for a bind answered with a referral ldap3 could not decode, warned once per + operation. The referred hosts were in the part that could not be read, so none is named.""" + message = ( + f"AD answered the {operation} with a referral, whose hosts ldap3 cannot decode in a bind " + "response; the engine does not follow referrals (BACKLOG #2530, #3010). This usually " + "means a configured DN or search base lies in another domain of the forest; use one in " + "this domain controller's own domain, or a global catalog." + ) + return _referral_error(operation, message) + + def _undecodable(operation: str, exc: BaseException) -> LdapUndecodableError: """The refusal for ``operation`` whose response ldap3 could not decode, warned once per operation and fault type. The text names the type only: neither a value the directory sent @@ -1624,7 +1694,10 @@ def authenticate( if not password: # never allow an empty password (it triggers an anonymous bind) return DirectoryBind(None) try: - with self._service_conn() as svc: + # Released explicitly (BACKLOG #3010): ldap3's own ``with`` never unbinds a connection + # that was bound on entry, so the session stayed open until it was collected. + svc = self._service_conn() + with _released(svc): found = ( self._lookup_by_object_id(svc, object_id, fallback_username=username) if object_id is not None @@ -1739,7 +1812,8 @@ def probe_principal( if object_id is not None and tie_break_object_id is not None: raise ValueError("pass object_id or tie_break_object_id, not both") try: - with self._service_conn() as svc: + svc = self._service_conn() + with _released(svc): # not ldap3's own ``with``: see ``authenticate`` found = ( self._lookup_by_object_id(svc, object_id, fallback_username=username) if object_id is not None diff --git a/messagefoundry/auth/reconcile.py b/messagefoundry/auth/reconcile.py index a658eedfba..c310e2e824 100644 --- a/messagefoundry/auth/reconcile.py +++ b/messagefoundry/auth/reconcile.py @@ -24,24 +24,34 @@ at once. :func:`hold_engaged` holds those accounts, and only those: none of them is revoked, and the rest of the estate is reconciled as usual. The mass-revoke breaker below cannot do this on a small estate, because its absolute floor lets five or fewer revocations through. -4. **A mass-revoke circuit breaker.** A misconfigured search base or a moved OU returns "not found" - for **every** user, which looks the same as "everyone was deleted". :func:`breaker_tripped` - aborts such a pass wholesale rather than signing out the estate. This is why the pass is - planned in full before anything is written: an abort must leave the store byte-identical, - including the role re-diff. +4. **A mass-revoke circuit breaker.** Users moved out of the search base, or a base that exists + but is the wrong one, return "not found" for **every** user, which looks the same as "everyone + was deleted". :func:`breaker_tripped` aborts such a pass wholesale rather than signing out the + estate. This is why the pass is planned in full before anything is written: an abort must leave + the store byte-identical, including the role re-diff. A base that was deleted or moved, or that + the service account may not read, is not "not found": the search ends with a result code, which + reads :attr:`ProbeOutcome.MISCONFIGURED` (BACKLOG #3010). A referral is not an outage, and not a reason to stop the pass either (BACKLOG #2538). A :attr:`ProbeOutcome.REFERRED` account is left unjudged, like an unreachable one, and every other -probe in the pass is judged as usual. Unlike an outage, a referral alerts. A pass whose only answers are -referrals and failures aborts as ``directory_referral``, which judges no account it asked about -and alerts. Such a pass, like an outage, still strikes an UNKEYED row (BACKLOG #3170). +probe in the pass is judged as usual. Unlike an outage, a referral alerts. A pass whose every answer +went unjudged, at least one a referral, aborts as ``directory_referral``, which judges no account it +asked about and alerts. Such a pass, like an outage, still strikes an UNKEYED row (BACKLOG #3170). A probe fault is not an outage either (BACKLOG #3100). A :attr:`ProbeOutcome.FAULT` account, whose probe raised something that is not an LDAP error, is left unjudged like an unreachable one, and unlike one it alerts on every pass that probes it. A pass whose only answers are faults and failures, at least one a fault, aborts as ``probe_fault``, which judges no account it asked about and alerts. +A pass where every probe fails and none faults is a plain outage, which pages nobody. It too still strikes an UNKEYED row (BACKLOG #3170). +A search the directory refused for a reason that points at the configuration is not an outage +either (BACKLOG #3010). A :attr:`ProbeOutcome.MISCONFIGURED` account is left unjudged like an +unreachable one, and unlike one it alerts on every pass that probes it. A pass whose only answers +are those refusals, faults and failures, at least one such refusal and no referral, aborts as +``directory_misconfigured``, which judges no account it asked about, alerts, and still strikes an +UNKEYED row. + The pass also re-diffs channel scope (ADR 0198, BACKLOG #1957). It revokes when the directory would withdraw or narrow a principal's scope, and it never writes the scope: the next login does, through the same :func:`~messagefoundry.auth.channel_scope.decide_ad_channel_scope` this module plans with. @@ -76,17 +86,19 @@ class ProbeOutcome(Enum): #: (BACKLOG #1639). A single one beside readable answers strikes and revokes like :attr:`ABSENT`; #: a wave of them is held (ADR 0195, :func:`hold_engaged`). UNDETERMINED = "undetermined" - #: The lookup matched nothing, or an id-keyed probe held a stored ``objectGUID`` that could not - #: be parsed, so no search ran, or an id-keyed search's entry did not read back that id (BACKLOG - #: #2027). **Ambiguous**: deleted, moved out of the search base, or a search - #: base that was never right. Strikes, never revokes on its own. Since ADR 0195 a disabled entry, + #: The lookup ended with success and matched nothing, or an id-keyed probe held a stored + #: ``objectGUID`` that could not be parsed, so no search ran, or an id-keyed search's entry did + #: not read back that id (BACKLOG #2027). **Ambiguous**: deleted, moved out of the search base, + #: or a search base that exists but was never the right one. A base that does not exist, or that + #: may not be read, is not this: the search ends with a result code, which is + #: :attr:`MISCONFIGURED` (BACKLOG #3010). Strikes, never revokes on its own. Since ADR 0195 a disabled entry, #: or one whose ``userAccountControl`` is unreadable, is NOT this outcome, with one exception #: (BACKLOG #2027): an entry an id-keyed search found that does not read back that id is this #: outcome whatever its account state, because it is not this account's entry. ABSENT = "absent" #: The directory could not be consulted (``LdapError`` — connectivity/bind/config — other than a - #: referral), or a network-class ``OSError`` escaped the directory layer (BACKLOG #3100). - #: Contributes nothing: no strike, no revocation, no strike reset. + #: referral or a configuration code), or a network-class ``OSError`` escaped the directory layer + #: (BACKLOG #3100). Contributes nothing: no strike, no revocation, no strike reset. UNAVAILABLE = "unavailable" #: The directory answered with a referral, which the engine never follows (``LdapReferralError``, #: BACKLOG #2538). Contributes nothing for this account, like :attr:`UNAVAILABLE`: no strike, @@ -105,6 +117,16 @@ class ProbeOutcome(Enum): #: would leave that account unrevoked without a page. The value is the certificate path's #: ``probe_fault`` refusal reason too. FAULT = "probe_fault" + #: The directory refused the search with a result code that points at the configuration rather + #: than at a passing load: noSuchObject, invalidDNSyntax or insufficientAccessRights + #: (``LdapUnansweredError.configuration``, BACKLOG #3010). A deleted or moved search base, or one + #: the service account may not read, answers so for every account. Contributes nothing for this + #: account, like :attr:`UNAVAILABLE`: no strike, no revocation, no strike reset, no recorded + #: outcome, and it never moves the ADR 0195 hold. Unlike it, it alerts, because it recurs on + #: every pass until the configuration is fixed: read as unavailable, it would stop revocation + #: without a page. Before BACKLOG #3010 such a search read as "not found", so a deleted base + #: struck every account and could trip the breaker. + MISCONFIGURED = "misconfigured" #: The row carries no ``directory_object_id``, so it was NOT probed (BACKLOG #2434). A name #: probe could read another account's entry. No sign-in or step-up admits such a row, so a #: session it holds is anomalous. Plans exactly as :attr:`ABSENT` does: it strikes, and revokes @@ -131,7 +153,9 @@ class Probe: #: For :attr:`ProbeOutcome.REFERRED`: the refusal's text, which names the referred search and #: the referred hosts and nothing else (BACKLOG #2530). The pass logs the first one. For #: :attr:`ProbeOutcome.FAULT`: the exception's type name, never its message (BACKLOG #3100). - #: None for every other outcome. + #: For :attr:`ProbeOutcome.MISCONFIGURED`: the refusal's text, which names the operation and the + #: result code and never the directory's diagnostic message (BACKLOG #3010). None for every + #: other outcome. detail: str | None = None @@ -170,8 +194,10 @@ class UsernameRefresh: #: The closed-set slug the held audit row and alert carry (ADR 0195 rule item 9). HOLD_REASON = "user_account_control_undetermined" -#: The ``aborted`` slug of a pass whose only answers were referrals and failures (BACKLOG #2538). -#: Also the reason slug of the referral's own audit row and alert, on any pass with a referral. +#: The ``aborted`` slug of a pass whose every answer went unjudged, at least one a referral (BACKLOG +#: #2538); the others may be failures, configuration refusals or faults (BACKLOG #3010, #3100), +#: which are then recorded under their own alerts as well. Also the reason slug of the referral's +#: own audit row and alert, on any pass with a referral. REFERRAL_ABORT = "directory_referral" #: The ``aborted`` slug of a pass whose only answers were probe faults and failures, at least one a @@ -179,8 +205,20 @@ class UsernameRefresh: #: with a fault. The same string as :attr:`ProbeOutcome.FAULT`'s value. PROBE_FAULT_ABORT = ProbeOutcome.FAULT.value +#: The ``aborted`` slug of a pass whose only answers were configuration refusals, faults and +#: failures, at least one a configuration refusal and none a referral (BACKLOG #3010). Also the +#: reason slug of that refusal's own audit row and alert, on any pass with one. +MISCONFIGURED_ABORT = "directory_misconfigured" + #: The outcomes that judge nothing about the account: no strike, no recorded outcome (rule item 4). -_UNJUDGED = frozenset({ProbeOutcome.UNAVAILABLE, ProbeOutcome.REFERRED, ProbeOutcome.FAULT}) +_UNJUDGED = frozenset( + { + ProbeOutcome.UNAVAILABLE, + ProbeOutcome.REFERRED, + ProbeOutcome.FAULT, + ProbeOutcome.MISCONFIGURED, + } +) @dataclass(frozen=True) @@ -214,21 +252,21 @@ class ReconcilePlan: renames: tuple[UsernameRefresh, ...] = () #: Strikes to record: ``user_id -> consecutive ABSENT, DISABLED, UNDETERMINED or UNKEYED #: count``. A PRESENT probe maps to 0 (reset), and so does a HELD one (ADR 0195 rule item 7; - #: ``held`` tells the two apart). An UNAVAILABLE, REFERRED or FAULT probe is absent from this mapping, - #: leaving whatever strike the user already carried untouched. Populated even on a breaker - #: abort — see :func:`plan_pass`. On an outage or a pass of referrals or faults only it holds - #: the UNKEYED rows' strikes alone (BACKLOG #3170), since those were judged without the - #: directory. + #: ``held`` tells the two apart). An unjudged probe (UNAVAILABLE, REFERRED, FAULT or + #: MISCONFIGURED) is absent from this mapping, leaving whatever strike the user already carried + #: untouched. Populated even on a breaker abort — see :func:`plan_pass`. On a pass that judged + #: nothing (:attr:`judged_nothing`) it holds the UNKEYED rows' strikes alone (BACKLOG #3170), + #: since those were judged without the directory. strikes: Mapping[str, int] = field(default_factory=dict) #: Candidates this pass judged, asked or not. Excludes the per-pass budget remainder. An UNKEYED #: row counts here, because it can revoke. This is the breaker's denominator (``judged``). probed: int = 0 #: Probes this pass actually sent to the directory: ``probed`` less its UNKEYED rows (BACKLOG - #: #2434). An outage and a pass of referrals or faults only are judged over these, and the - #: outage's log line and the ``asked`` field of its ``auth.ad_reconcile_skipped`` row count - #: these. So do the referral's latched message, the ``asked`` field of its - #: ``auth.ad_reconcile_referred`` row and its alert's count (BACKLOG #3170): a referral can - #: only answer a probe that was sent. + #: #2434). Every abort but the breaker's is judged over these. The outage, the referral, the + #: configuration refusal and the probe fault each count these in their log line or latched + #: message and in the ``asked`` field of their audit row, and the last three in their alert's + #: count (BACKLOG #3170, #3010): only a probe that was sent can be referred, refused or raise a fault. + #: The breaker's alert alone counts ``probed``, its judged denominator. asked: int = 0 unavailable: int = 0 #: ``user_id`` of every probe the directory answered with a referral (BACKLOG #2538). Any one @@ -237,14 +275,18 @@ class ReconcilePlan: #: ``user_id`` of every probe that raised a fault (:attr:`ProbeOutcome.FAULT`, BACKLOG #3100). #: Any one alerts, and none of them stops the rest of the pass being judged. faulted: tuple[str, ...] = () + #: ``user_id`` of every probe the directory refused for its configuration + #: (:attr:`ProbeOutcome.MISCONFIGURED`, BACKLOG #3010). Any one alerts, and none of them stops + #: the rest of the pass being judged. + misconfigured: tuple[str, ...] = () #: Non-None when the pass ABORTED and must apply nothing: the breaker tripped, every probe the - #: pass asked failed (a whole-directory outage), every probe it asked failed or was referred, - #: at least one of them referred, or every probe it asked failed or faulted, at least one of - #: them a fault (BACKLOG #3100). An UNKEYED row was not asked, so it never stops either abort - #: (BACKLOG #2434). The value is a closed-set operator-facing slug. + #: pass asked failed (a whole-directory outage), or every probe it asked went unjudged with at + #: least one of them referred, refused for its configuration or faulted, which take the slug in + #: that order (BACKLOG #2538, #3010, #3100). An UNKEYED row was not asked, so it never stops an + #: abort (BACKLOG #2434). The value is a closed-set operator-facing slug. aborted: str | None = None #: Outcomes to record in the caller's per-candidate record (ADR 0195 rule item 4): every probe of - #: this pass except an UNAVAILABLE, REFERRED or FAULT one, which leaves the prior entry in place. + #: this pass except an unjudged one (``_UNJUDGED``), which leaves the prior entry in place. #: Populated on a breaker abort too, like ``strikes``. outcomes: Mapping[str, ProbeOutcome] = field(default_factory=dict) #: Whether this pass held its UNDETERMINED probes (:func:`hold_engaged`). @@ -277,8 +319,8 @@ def directory_outage(self) -> bool: @property def directory_referral(self) -> bool: - """True when the pass aborted because every probe it asked was referred or failed, at least - one of them referred (BACKLOG #2538). It judged nothing, like an outage, but it alerts: a + """True when the pass aborted because every probe it asked went unjudged, at least one of + them referred (BACKLOG #2538). It judged nothing, like an outage, but it alerts: a referral usually means ``ad_user_search_base`` lies in another domain of the forest, which no later pass fixes on its own. A pass with a referral beside other answers is NOT this. It judged those answers, and only ``referred`` says it saw a referral.""" @@ -289,18 +331,33 @@ def probe_fault(self) -> bool: """True when the pass aborted because every probe it asked faulted or failed, at least one of them a fault (BACKLOG #3100). It judged nothing, like an outage, but it alerts: the engine could not read what the directory returned, which no later pass fixes on its own. - A pass with a referral as well is a referral pass. A pass with a fault beside other answers - is NOT this; only ``faulted`` says it saw a fault.""" + A pass with a referral or a configuration refusal as well is one of those passes. A pass + with a fault beside other answers is NOT this; only ``faulted`` says it saw a fault.""" return self.aborted == PROBE_FAULT_ABORT + @property + def directory_misconfigured(self) -> bool: + """True when the pass aborted because every probe it asked was refused for its + configuration, faulted or failed, at least one refused for its configuration and none + referred (BACKLOG #3010). It judged nothing, like an outage, but it alerts: a deleted or + moved search base, or one the service account may not read, stays wrong until someone + fixes it. A pass with such a refusal beside other answers is NOT this; only + ``misconfigured`` says it saw one.""" + return self.aborted == MISCONFIGURED_ABORT + @property def judged_nothing(self) -> bool: - """True when the pass aborted before judging any account: an outage, a pass of referrals - and failures only, or one of faults and failures only. Such a pass leaves the hold's state, - its message and every clear alone, because it learned nothing about the accounts it asked - about. It still strikes an UNKEYED row, which was judged without the directory (BACKLOG - #3170).""" - return self.directory_outage or self.directory_referral or self.probe_fault + """True when the pass aborted before judging any account: an outage, or a pass whose every + answer went unjudged, at least one of them referred, refused for its configuration or + faulted. Such a pass leaves the hold's state, its message and every clear alone, because + it learned nothing about the accounts it asked about. It still strikes an UNKEYED row, + which was judged without the directory (BACKLOG #3170).""" + return ( + self.directory_outage + or self.directory_referral + or self.directory_misconfigured + or self.probe_fault + ) @property def judged(self) -> int: @@ -390,8 +447,8 @@ def select_candidates( def _unkeyed_strikes(probes: list[Probe], prior_strikes: Mapping[str, int]) -> dict[str, int]: """The strikes of a pass that judged nothing (BACKLOG #3170 item 3). - An UNKEYED row was judged without the directory, so an outage or a pass of referrals or faults - only (BACKLOG #3100) still strikes it, as any other pass does. Only the strike: its outcome is + An UNKEYED row was judged without the directory, so a pass that judged nothing it asked about + (:attr:`ReconcilePlan.judged_nothing`) still strikes it, as any other pass does. Only the strike: its outcome is not recorded, so the hold's record, and so its carry-over, stays as it was, and an aborted pass revokes nothing.""" return { @@ -433,16 +490,25 @@ def plan_pass( unavailable = [p for p in probes if p.outcome is ProbeOutcome.UNAVAILABLE] referred = [p for p in probes if p.outcome is ProbeOutcome.REFERRED] faulted = [p for p in probes if p.outcome is ProbeOutcome.FAULT] + misconfigured = [p for p in probes if p.outcome is ProbeOutcome.MISCONFIGURED] # BACKLOG #2434. The caller reads a row it may not ask about as UNKEYED, so that outcome is # the one mark of "never asked": it says nothing about whether the directory answered. asked = sum(1 for p in probes if p.outcome is not ProbeOutcome.UNKEYED) - if (referred or faulted) and len(unavailable) + len(referred) + len(faulted) == asked: - # BACKLOG #2538, #3100. Nothing the pass asked was answered, as on an outage, so it judges - # nothing and the hold's state carries over. Checked before the outage, so a pass of - # referrals or faults and failures is not read as a plain outage, which pages nobody. A - # user search base in another domain of the forest refers every probe, and an entry or a - # defect that raises does so on every pass. A referral wins over a fault: it names the fix. + alerting = len(referred) + len(misconfigured) + len(faulted) + if alerting and len(unavailable) + alerting == asked: + # BACKLOG #2538, #3010, #3100. Nothing the pass asked was answered, as on an outage, so it + # judges nothing and the hold's state carries over. Checked before the outage, so such a + # pass is not read as a plain outage, which pages nobody. A user search base in another + # domain of the forest refers every probe, a deleted one is refused for every probe, and an + # entry or a defect that raises does so on every pass. A referral wins, then a + # configuration refusal: each names the fix, and a fault does not. + if referred: + aborted = REFERRAL_ABORT + elif misconfigured: + aborted = MISCONFIGURED_ABORT + else: + aborted = PROBE_FAULT_ABORT return ReconcilePlan( strikes=_unkeyed_strikes(probes, prior_strikes), probed=len(probes), @@ -450,7 +516,8 @@ def plan_pass( unavailable=len(unavailable), referred=tuple(p.user_id for p in referred), faulted=tuple(p.user_id for p in faulted), - aborted=REFERRAL_ABORT if referred else PROBE_FAULT_ABORT, + misconfigured=tuple(p.user_id for p in misconfigured), + aborted=aborted, latched=latched, ) @@ -472,7 +539,8 @@ def plan_pass( # ADR 0195 rule items 4 to 6. An UNAVAILABLE probe leaves the prior entry in place, so a # directory blip on one held account does not make another look single. A REFERRED probe does # too (BACKLOG #2538): a referral says where to look, not what the account is. So does a FAULT - # probe (BACKLOG #3100): the engine could not read the answer. Every other probe + # probe (BACKLOG #3100): the engine could not read the answer. So does a MISCONFIGURED probe + # (BACKLOG #3010): the directory refused the search, not the account. Every other probe # is judged below as on any pass. A referring group search base refers only the accounts the # user search FOUND enabled, so a disabled or absent account's answer still strikes and revokes. outcomes = {p.user_id: p.outcome for p in probes if p.outcome not in _UNJUDGED} @@ -558,6 +626,7 @@ def plan_pass( unavailable=len(unavailable), referred=tuple(p.user_id for p in referred), faulted=tuple(p.user_id for p in faulted), + misconfigured=tuple(p.user_id for p in misconfigured), outcomes=outcomes, hold=hold, latched=hold_latches(undetermined=undetermined, latched=latched), diff --git a/messagefoundry/auth/service.py b/messagefoundry/auth/service.py index d4fea758f1..a815af8992 100644 --- a/messagefoundry/auth/service.py +++ b/messagefoundry/auth/service.py @@ -60,6 +60,7 @@ LdapAuthenticator, LdapError, LdapReferralError, + LdapUnansweredError, kerberos_principal, normalise_object_guid, ) @@ -296,6 +297,7 @@ def _error_if_bundled_corpus_unusable(check_breached: bool) -> None: { reconcile.ProbeOutcome.UNAVAILABLE.value, reconcile.ProbeOutcome.REFERRED.value, + reconcile.ProbeOutcome.MISCONFIGURED.value, CERT_PROBE_SATURATED, } ) @@ -2626,6 +2628,10 @@ def __init__( #: Latched probe fault (BACKLOG #3100): set on a pass in which a probe faulted, and released #: by the next pass that judged something with no fault (`_record_reconcile_fault`). self._reconcile_fault_alert: str | None = None + #: Latched configuration refusal (BACKLOG #3010), on the probe fault's rule: set on a pass in + #: which the directory refused a probe for its configuration, and released by the next pass + #: that judged something with no such refusal (`_record_reconcile_misconfigured`). + self._reconcile_misconfigured_alert: str | None = None #: user_ids a breaker trip has not yet seen read PRESENT on a pass that was not aborted #: (BACKLOG #2136). `_mark_reconcile_clears` reports no breaker clear while one is signed in. self._reconcile_unconfirmed: set[str] = set() @@ -6209,6 +6215,13 @@ def directory_reconcile_fault(self) -> str | None: pass judges something with no fault (:meth:`_record_reconcile_fault`).""" return self._reconcile_fault_alert + @property + def directory_reconcile_misconfigured(self) -> str | None: + """The standing configuration refusal's operator message, or ``None`` (BACKLOG #3010). + Latches until a pass judges something with no such refusal + (:meth:`_record_reconcile_misconfigured`).""" + return self._reconcile_misconfigured_alert + @property def directory_reconcile_hold(self) -> str | None: """The engaged undetermined-wave hold's operator message, or ``None`` (ADR 0195). Latches @@ -6269,13 +6282,27 @@ async def _probe_principal(self, user: UserRecord) -> reconcile.Probe: user.id, user.username, reconcile.ProbeOutcome.REFERRED, detail=str(exc) ) except LdapError as exc: + if isinstance(exc, LdapUnansweredError) and exc.configuration: + # BACKLOG #3010. A deleted or moved search base, or one the service account may not + # read, refuses every probe on every pass. Read as UNAVAILABLE it would stop + # revocation without a page; MISCONFIGURED alerts instead, and never strikes or + # revokes this account. DEBUG here, as for a referral: the reconciler logs one + # ERROR per pass carrying the first refusal's text. That text is the engine's own, + # naming the operation and the code, never the directory's diagnostic message. + _log.debug("directory probe of %s was refused: %s", user.username, exc) + return reconcile.Probe( + user.id, user.username, reconcile.ProbeOutcome.MISCONFIGURED, detail=str(exc) + ) # FAIL OPEN, for the reconciler. verify_mfa reads the same UNAVAILABLE as a refusal and # fails closed, because it grants rather than revokes (BACKLOG #2023). It writes an audit # row per refusal, so it adds no log line here either. # An unreachable DC must never revoke: a fail-closed re-check would turn a # directory blip into a total console outage during exactly the incident when operators # need the console. Debug-level — a flapping DC must not flood the log at one line per - # user per pass; the pass-level summary below reports the count at WARNING. + # user per pass; the pass-level summary below reports the count at WARNING. A result + # code that is not an answer, or a response ldap3 could not decode, is also reported + # at WARNING by ldap.py itself, once per operation and code or fault type (BACKLOG + # #3010), in engine text that never quotes the directory's diagnostic message. _log.debug("directory probe failed for %s: %s", user.username, exc) return reconcile.Probe(user.id, user.username, reconcile.ProbeOutcome.UNAVAILABLE) except _UNREACHED_DIRECTORY as exc: @@ -6364,7 +6391,8 @@ async def reconcile_directory_sessions(self) -> reconcile.ReconcilePlan: * a probe that could not reach the directory contributes nothing (fail-open); * a probe the directory referred contributes nothing either, and the pass alerts, while - every other account is still judged (BACKLOG #2538); + every other account is still judged (BACKLOG #2538). So does a probe the directory + refused for its configuration, such as a deleted search base (BACKLOG #3010); * a principal must come back absent, disabled, undetermined or unkeyed ``ad_session_recheck_strikes`` passes running; * a PRESENT principal whose directory groups would change its roles, or withdraw or narrow @@ -6546,18 +6574,23 @@ async def reconcile_directory_sessions(self) -> reconcile.ReconcilePlan: uid for uid, outcome in plan.outcomes.items() if outcome is present ) referral = next((p for p in probes if p.outcome is reconcile.ProbeOutcome.REFERRED), None) + refusal = next( + (p for p in probes if p.outcome is reconcile.ProbeOutcome.MISCONFIGURED), None + ) if plan.aborted is not None: - if not plan.directory_referral and not plan.probe_fault: + if plan.directory_outage or not plan.judged_nothing: + # The outage's skipped row, or the breaker's trip. A pass that judged nothing for + # any other reason is recorded only by the recorders below, as a referral (#2538), + # a configuration refusal (#3010) or a fault (#3100), and never as a trip. await self._abort_reconcile_pass(plan) if not plan.directory_outage: - # A pass of referrals only is recorded as a referral and nothing else (#2538), and - # a pass of faults only as a fault and nothing else (#3100). await self._record_reconcile_referral(plan, first=referral) + await self._record_reconcile_misconfigured(plan, first=refusal) await self._record_reconcile_fault(plan) if not plan.judged_nothing: # A held pass writes its own row even when the breaker also aborts it (ADR 0195 rule - # item 9). An outage or a pass of referrals only judged nothing, so it leaves the - # hold's message alone and marks no clear (BACKLOG #2538). + # item 9). A pass that judged nothing (an outage, or one whose every answer went + # unjudged) leaves the hold's message alone and marks no clear (BACKLOG #2538). await self._record_reconcile_hold(plan) plan = self._mark_reconcile_clears(plan, users) await self._report_unkeyed_bindings(unkeyed, still_unkeyed=still_unkeyed) @@ -6608,7 +6641,8 @@ async def reconcile_directory_sessions(self) -> reconcile.ReconcilePlan: # BACKLOG #2538. After the revocations, like the hold below: the referred accounts were left # unjudged, and every other account's revocation above has already been applied. await self._record_reconcile_referral(plan, first=referral) - # BACKLOG #3100. After the revocations too, for the same reason as the referral's. + # BACKLOG #3010 and #3100. After the revocations too, for the same reason as the referral's. + await self._record_reconcile_misconfigured(plan, first=refusal) await self._record_reconcile_fault(plan) # ADR 0195. After the revocations, so a held-row audit write that fails cannot stop a # genuine disable or demotion in the same pass from being applied. And that failure is @@ -6657,9 +6691,9 @@ def _mark_reconcile_clears( * All three need an answer on record, from this process, for every signed-in account the pass did not just revoke. A probe that could not reach the directory leaves none. - * A pass in which any probe was referred marks none of the three (BACKLOG #2538). A - referred account has no answer from this pass, and an older one on record may predate - the referral. + * A pass in which any probe was referred, faulted or refused for its configuration marks + none of the three (BACKLOG #2538, #3100, #3010). Such an account has no answer from this + pass, and an older one on record may predate the referral, fault or refusal. * The referral's own instance is clear when, on top of the first test, every account signed in at the last referral has since read PRESENT on a pass with no referral (``_reconcile_referred``), and ``_reconcile_referral_standing`` is ``"referred"``: a @@ -6720,11 +6754,18 @@ def _mark_reconcile_clears( revoked_undetermined = undetermined in (self._reconcile_outcomes.get(u) for u in revoked) ids = [uid for uid in users if uid not in revoked] outcomes = [self._reconcile_outcomes.get(uid) for uid in ids] - # A faulted account, like a referred one, keeps an older answer that may predate the fault - # (BACKLOG #3100), so a pass with either marks no clear. - covered = bool(ids) and None not in outcomes and not plan.referred and not plan.faulted + # A faulted or refused account, like a referred one, keeps an older answer that may predate + # the fault or the refusal (BACKLOG #3100, #3010), so a pass with any of them marks no clear. + covered = ( + bool(ids) + and None not in outcomes + and not plan.referred + and not plan.faulted + and not plan.misconfigured + ) settled = covered and undetermined not in outcomes - # Called only on a pass that judged something, so a pass of referrals only never gets here. + # Called only on a pass that judged something, so a pass that judged nothing never gets + # here. referral_clear = ( covered and self._reconcile_referral_standing == "referred" @@ -7290,8 +7331,8 @@ async def _abort_reconcile_pass(self, plan: reconcile.ReconcilePlan) -> None: ) return # The ceiling is taken over the breaker's own denominator, which counts held probes (ADR - # 0195 amendment 2026-10-11, BACKLOG #2138). A pass of referrals only never reaches here: it - # is not a trip (BACKLOG #2538). + # 0195 amendment 2026-10-11, BACKLOG #2138). No other pass that judged nothing reaches + # here: none of them is a trip (BACKLOG #2538, #3010, #3100). ceiling = reconcile.breaker_ceiling( probed=plan.judged, max_absolute=self._settings.ad_session_revoke_max, @@ -7324,8 +7365,9 @@ async def _record_reconcile_referral( ) -> None: """Latch, log and audit a pass in which the directory referred probes. - BACKLOG #2538. Called on every pass that judged something, and on a pass of referrals - only; an outage pass does not call it. On one with a referral, + BACKLOG #2538. Called on every pass but an outage, the same as + :meth:`_record_reconcile_misconfigured` and :meth:`_record_reconcile_fault`. On one with a + referral, whatever else the pass did, it latches the message, logs it at ERROR with the first referral's search and hosts, and writes an ``auth.ad_reconcile_referred`` row. Its own audit action, because a pass with a referral beside other answers is not an aborted one, and an @@ -7351,15 +7393,7 @@ async def _record_reconcile_referral( if plan.aborted is not None else "Accounts that answered without a referral were judged as usual." ) - if plan.faulted: - # BACKLOG #3100. Counted here too, and recorded as a fault of their own. - others = f"{len(plan.faulted)} raised a probe fault. {others}" - if plan.unavailable: - # A pass of referrals and failures is recorded here and not as an outage, so the - # failures are counted here, or a near-total outage would read as a search-base fault. - others = ( - f"{plan.unavailable} could not be read (the directory was unreachable). {others}" - ) + others = self._unjudged_others(plan, others, faults=True, refusals=True) self._reconcile_referral_alert = ( "LDAP referral: check [auth].ad_user_search_base and, for nested groups, " "[auth].ad_group_search_base; one names a base in another domain of the forest. " @@ -7408,10 +7442,12 @@ async def _record_reconcile_fault(self, plan: reconcile.ReconcilePlan) -> None: later breaker trip or referral. **Nothing resolves that instance automatically**: an operator resolves it once the fault is gone. - Called on every pass that judged something, and on a pass of referrals or faults only; an - outage pass does not call it. A pass with no fault that judged something releases the - latched message. The message and the row carry counts and no username; the per-probe - WARNING in :meth:`_probe_principal` names each account and the exception type. + Called on every pass but an outage. Whatever else a pass did, one with no fault that judged + something releases the latched message, and one that judged nothing leaves it as it was. + The message, the row and the alert count the faults over the probes ASKED, as the + referral's do (BACKLOG #3170): an UNKEYED row was never sent, so it cannot fault. The + message and the row carry counts and no username; the per-probe WARNING in + :meth:`_probe_principal` names each account and the exception type. """ if not plan.faulted: if not plan.judged_nothing: @@ -7422,12 +7458,7 @@ async def _record_reconcile_fault(self, plan: reconcile.ReconcilePlan) -> None: if plan.aborted is not None else "Accounts that answered were judged as usual." ) - if plan.unavailable: - # Counted here, as the referral's message does: a pass of faults and failures is - # recorded as a fault and not as an outage, so the outage would otherwise go unsaid. - others = ( - f"{plan.unavailable} could not be read (the directory was unreachable). {others}" - ) + others = self._unjudged_others(plan, others) # The pointer to the account comes first: the store keeps only the first 200 characters # of an alert instance's reason, as `_record_reconcile_referral` says. self._reconcile_fault_alert = ( @@ -7445,18 +7476,100 @@ async def _record_reconcile_fault(self, plan: reconcile.ReconcilePlan) -> None: "reason": reconcile.PROBE_FAULT_ABORT, "faulted": len(plan.faulted), "unavailable": plan.unavailable, - "probed": plan.probed, + "asked": plan.asked, + "aborted": plan.aborted, + } + ), + ) + + async def _record_reconcile_misconfigured( + self, plan: reconcile.ReconcilePlan, *, first: reconcile.Probe | None + ) -> None: + """Latch, log and audit a pass in which the directory refused a probe for its configuration. + + BACKLOG #3010. A search that ends with noSuchObject, invalidDNSyntax or + insufficientAccessRights says the configured base, or the service account's right to read + it, is wrong, not that the account is gone (:attr:`~reconcile.ProbeOutcome.MISCONFIGURED`). + It leaves that account unjudged: no strike and no revocation. A deleted or moved search base + refuses every account on every pass, so while it lasts nothing is revoked and the sessions + run to the absolute cap. Before #3010 such a search read as "not found", which tripped the + mass-revoke breaker on a large enough estate, and a plain outage would page nobody. So + every pass with such a refusal latches the message, logs it at ERROR with the first + refusal's text, and writes an ``auth.ad_reconcile_misconfigured`` row, and the lifespan + task raises it as ``ad_reconcile_aborted`` with reason ``directory_misconfigured`` under its + own source label, for the reasons :meth:`_record_reconcile_fault` gives. Like the fault's, + **nothing resolves that instance automatically**, and the latched message follows the + fault's rule. + + Called on every pass but an outage. The message, the row and the alert count over the + probes ASKED (BACKLOG #3170). They carry counts and no username; the refusal's text names + the operation and the code, never the directory's diagnostic message, and + :meth:`_probe_principal` names each refused account at DEBUG. + """ + if not plan.misconfigured: + if not plan.judged_nothing: + self._reconcile_misconfigured_alert = None + return + others = ( + "The pass revoked nothing." + if plan.aborted is not None + else "Accounts that answered were judged as usual." + ) + others = self._unjudged_others(plan, others) + # The settings to change come first: the store keeps only the first 200 characters of an + # alert instance's reason, as `_record_reconcile_referral` says. + self._reconcile_misconfigured_alert = ( + "LDAP configuration refused: check [auth].ad_user_search_base, " + "[auth].ad_group_search_base and the read rights of [auth].ad_bind_dn. " + f"{len(plan.misconfigured)} of {plan.asked} signed-in directory account(s) asked were " + f"refused for the configuration and left unjudged. {others} A refused account is " + "never revoked while the refusal lasts, so its sessions run to " + "[auth].session_absolute_hours. The ERROR log line for each such pass names the code." + ) + _log.error( + "directory reconcile: %s First refusal: %s", + self._reconcile_misconfigured_alert, + first.detail if first is not None else "?", + ) + await self._audit_reconciler_row( + "auth.ad_reconcile_misconfigured", + detail=_json( + { + "reason": reconcile.MISCONFIGURED_ABORT, + "misconfigured": len(plan.misconfigured), + "unavailable": plan.unavailable, + "asked": plan.asked, "aborted": plan.aborted, } ), ) + @staticmethod + def _unjudged_others( + plan: reconcile.ReconcilePlan, others: str, *, faults: bool = False, refusals: bool = False + ) -> str: + """``others`` led by the counts of the pass's other unjudged probes, for a latched message. + + A pass whose every answer went unjudged is recorded by its referral, refusal or fault and + not as an outage, so its failures are counted here, or a near-total outage would read as a + configuration fault. ``faults`` and ``refusals`` add those counts too: the referral's + message carries both, since either is recorded under its own alert as well.""" + if faults and plan.faulted: + others = f"{len(plan.faulted)} raised a probe fault. {others}" + if refusals and plan.misconfigured: + others = f"{len(plan.misconfigured)} were refused for the configuration. {others}" + if plan.unavailable: + others = ( + f"{plan.unavailable} could not be read (the directory was unreachable). {others}" + ) + return others + async def _audit_reconciler_row(self, action: str, *, detail: str) -> bool: """Write one of the reconcile pass's own rows, and say whether it was written. BACKLOG #2137. These rows are the held row, the breaker's aborted row, the outage's skipped - row, the referral's row (BACKLOG #2538), the probe fault's row (BACKLOG #3100) and the - unkeyed-binding report. Each is written + row, the referral's row (BACKLOG #2538), the probe fault's row (BACKLOG #3100), the + configuration refusal's row (BACKLOG #3010) and the unkeyed-binding report. Each is written after the pass's revocations, or on a pass that applies none. **A failed write here does not end the pass.** If it raised, the pass would return no plan, and the lifespan task would raise no alert for the revocations already applied, nor for the hold, the breaker or diff --git a/messagefoundry/pipeline/alerts.py b/messagefoundry/pipeline/alerts.py index 8db451c29d..2f13353d45 100644 --- a/messagefoundry/pipeline/alerts.py +++ b/messagefoundry/pipeline/alerts.py @@ -540,20 +540,23 @@ def ad_reconcile_aborted(self, name: str, *, reason: str, probed: int, detail: s which leaves the referred accounts unjudged and may sit beside revocations the pass applied; or one or more probes raised a fault the engine could not read (``probe_fault``, BACKLOG #3100, the ``auth.ad_reconcile_faulted`` row), which leaves those accounts unjudged the same - way. The type's name predates the second and third cases. Read ``reason`` before naming the - cause. + way; or the directory refused one or more probes with a result code that points at the + configuration, such as a deleted or moved search base (``directory_misconfigured``, BACKLOG + #3010, the ``auth.ad_reconcile_misconfigured`` row), which leaves those accounts unjudged the + same way. The type's name predates every case but the first. Read ``reason`` before naming + the cause. ``name`` labels the source: ``"directory-reconciler"`` for the breaker, - ``"directory-reconciler-referral"`` for a referral, and - ``"directory-reconciler-probe-fault"`` for a probe fault, so each is its own instance and - throttle. ``reason`` is one of those three closed-set slugs. ``probed`` is, for the breaker, - how many principals the pass judged, its denominator; for a probe fault, the same total of - candidates the pass took up, asked or not, which counts the faulted accounts it left - unjudged and any id-less row; and for a referral how many probes the pass sent to the - directory, which leaves out an id-less row (BACKLOG #3170). ``detail`` is the operator-facing explanation the auth service - latches. No PHI. Emitted by the API-lifespan reconciler task, never from ``auth/``. A - whole-directory outage is NOT this event: it is - audited as ``auth.ad_reconcile_skipped`` and pages nothing, because the accounts are fine.""" + ``"directory-reconciler-referral"`` for a referral, ``"directory-reconciler-probe-fault"`` + for a probe fault, and ``"directory-reconciler-misconfigured"`` for a configuration refusal, + so each is its own instance and throttle. ``reason`` is one of those four closed-set slugs. + ``probed`` is, for the breaker, how many principals the pass judged, its denominator; for + each of the other three, how many probes the pass sent to the directory, which leaves out + an id-less row (BACKLOG #3170). ``detail`` is the operator-facing explanation the auth + service latches. No PHI. Emitted by the API-lifespan reconciler task, never from ``auth/``. + A whole-directory outage is NOT this event: it is audited as ``auth.ad_reconcile_skipped`` + and pages nothing, because the accounts are fine. A pass where every probe fails and none is + referred, refused or faults is such an outage.""" ... def ad_reconcile_held(self, name: str, *, reason: str, undetermined: int, detail: str) -> None: diff --git a/tests/test_ad_directory_identity.py b/tests/test_ad_directory_identity.py index e51c8b32b5..fd81fe2461 100644 --- a/tests/test_ad_directory_identity.py +++ b/tests/test_ad_directory_identity.py @@ -1149,6 +1149,9 @@ def __enter__(self) -> _RecordingConn: def __exit__(self, *exc: object) -> None: return None + def unbind(self) -> None: + """The lookups release the service connection explicitly (BACKLOG #3010).""" + def search(self, **kwargs: Any) -> None: self.searches.append(kwargs) # Only the FIRST search (the user lookup) may match, and only when this double was given an diff --git a/tests/test_ad_session_reconcile.py b/tests/test_ad_session_reconcile.py index 3fdac67123..4a7c31cd13 100644 --- a/tests/test_ad_session_reconcile.py +++ b/tests/test_ad_session_reconcile.py @@ -38,6 +38,7 @@ DirectoryProbe, LdapError, LdapReferralError, + LdapUnansweredError, ) from messagefoundry.auth.notifications import ( ACCOUNT_DISABLED, @@ -684,6 +685,11 @@ def test_an_unkeyed_probe_does_not_hide_an_outage_or_a_referral_only_pass() -> N ], reconcile.PROBE_FAULT_ABORT, ), + # BACKLOG #3010 shares it too: a pass of configuration refusals only. + ( + [reconcile.Probe("m", "m", reconcile.ProbeOutcome.MISCONFIGURED)], + reconcile.MISCONFIGURED_ABORT, + ), ], ids=[ "outage", @@ -691,6 +697,7 @@ def test_an_unkeyed_probe_does_not_hide_an_outage_or_a_referral_only_pass() -> N "referrals-and-failures", "faults-only", "faults-and-failures", + "refusals-only", ], ) def test_an_unkeyed_row_is_struck_on_a_pass_that_judged_nothing( @@ -715,6 +722,61 @@ def test_an_unkeyed_row_is_struck_on_a_pass_that_judged_nothing( assert [r.reason for r in ordinary.revocations] == ["directory_object_id_missing"] +def _probes(*pairs: tuple[str, reconcile.ProbeOutcome]) -> list[reconcile.Probe]: + return [reconcile.Probe(uid, uid, outcome) for uid, outcome in pairs] + + +_M, _R, _F, _U, _A = ( + reconcile.ProbeOutcome.MISCONFIGURED, + reconcile.ProbeOutcome.REFERRED, + reconcile.ProbeOutcome.FAULT, + reconcile.ProbeOutcome.UNAVAILABLE, + reconcile.ProbeOutcome.ABSENT, +) + + +@pytest.mark.parametrize( + ("pairs", "aborted"), + [ + ((("m", _M),), reconcile.MISCONFIGURED_ABORT), + ((("m", _M), ("u", _U)), reconcile.MISCONFIGURED_ABORT), + # A referral names the fix most closely, then the refusal; a fault names none. + ((("m", _M), ("f", _F)), reconcile.MISCONFIGURED_ABORT), + ((("m", _M), ("r", _R)), reconcile.REFERRAL_ABORT), + # THE CONTROL: failures alone are an outage, which pages nobody. + ((("u", _U), ("v", _U)), "directory_unavailable"), + ], + ids=["refusals", "refusals-and-failures", "beside-a-fault", "beside-a-referral", "outage"], +) +def test_a_pass_of_configuration_refusals_aborts_on_its_own_slug( + pairs: tuple[tuple[str, reconcile.ProbeOutcome], ...], aborted: str +) -> None: + """BACKLOG #3010. A deleted or moved search base refuses every probe with noSuchObject. Read as + an outage, that pass paged nobody; before #3010 it read as "not found" and struck everyone. It + aborts as ``directory_misconfigured``, which judges nothing and alerts, and never revokes. + + RED before D3b: there is no such outcome, and the refusal read as an outage.""" + plan = _plan(_probes(*pairs), prior_strikes={"m": 1, "u": 1}) + assert plan.aborted == aborted + assert plan.judged_nothing and plan.revocations == () and plan.strikes == {} + assert plan.directory_misconfigured is (aborted == reconcile.MISCONFIGURED_ABORT) + assert plan.misconfigured == tuple(uid for uid, o in pairs if o is _M) + + +def test_a_configuration_refusal_beside_answers_leaves_only_that_account_unjudged() -> None: + """Like a referral or a fault: the refused account keeps its strike and its recorded outcome, + and the rest of the pass is judged, revocations included.""" + plan = _plan( + _probes(("m", _M), ("gone", _A)), + prior_strikes={"m": 1, "gone": 1}, + prior_outcomes={"m": reconcile.ProbeOutcome.PRESENT}, + ) + assert plan.aborted is None and not plan.judged_nothing + assert plan.misconfigured == ("m",) + assert "m" not in plan.strikes and "m" not in plan.outcomes + assert [r.user_id for r in plan.revocations] == ["gone"] + + def test_breaker_ceiling_reports_the_larger_of_the_two_thresholds() -> None: assert reconcile.breaker_ceiling(probed=10, max_absolute=5, max_fraction=0.34) == 5 assert reconcile.breaker_ceiling(probed=300, max_absolute=5, max_fraction=0.34) == 102 @@ -1315,7 +1377,7 @@ async def test_when_every_probe_raises_the_operator_sees_a_probe_fault_not_an_ou "reason": "probe_fault", "faulted": 2, "unavailable": 0, - "probed": 2, + "asked": 2, "aborted": "probe_fault", } assert _faulted_rows(audit) == [expected, expected] @@ -1745,6 +1807,201 @@ def ad_reconcile_aborted( await store.close() +def _no_such_base() -> LdapUnansweredError: + """What ``_search`` raises when the user search base was deleted: engine text, a code and its + name, never the directory's diagnostic message.""" + return LdapUnansweredError( + "AD ended the user search with result code 32 (noSuchObject), which is not an answer", + code=32, + code_name="noSuchObject", + ) + + +#: The two unjudged outcomes that count over the probes asked besides the referral: what makes +#: ``alice``'s probe one, its audit action, the row field holding its count, the plan field, the +#: latched message's property and the alert's reason. +_COUNTED_OVER_ASKED = [ + pytest.param( + lambda: KeyError("userAccountControl"), + "auth.ad_reconcile_faulted", + "faulted", + "directory_reconcile_fault", + reconcile.PROBE_FAULT_ABORT, + id="probe-fault", + ), + pytest.param( + _no_such_base, + "auth.ad_reconcile_misconfigured", + "misconfigured", + "directory_reconcile_misconfigured", + reconcile.MISCONFIGURED_ABORT, + id="configuration-refusal", + ), +] + + +@pytest.mark.parametrize(("raised", "action", "field", "latch", "reason"), _COUNTED_OVER_ASKED) +async def test_a_fault_or_refusal_is_counted_over_the_probes_asked( + raised: Any, action: str, field: str, latch: str, reason: str +) -> None: + """The twin of the referral's test above (BACKLOG #3170, Manager ruling: one event, one count). + The fault's row, message and alert count the probes asked, so the id-less row is left out, and + the row names the field ``asked``. The configuration refusal (BACKLOG #3010) does the same. + + RED before D3b: the fault's row carried ``probed`` and its alert counted the id-less row.""" + from messagefoundry.api.app import _alert_reconcile_plan + from messagefoundry.pipeline.alerts import LoggingAlertSink + + store = await MessageStore.open(":memory:") + try: + ldap = _FakeLdap( + { + "alice": _principal("alice"), + "bob": _principal("bob"), + "jdoe": replace(_principal("jdoe"), directory_object_id=None), + } + ) + service = AuthService(store, _ad_settings(), ldap=ldap) # type: ignore[arg-type] + await service.initialize() + for name in ("alice", "bob", "jdoe"): + assert await _signed_in_ad_user(service, store, name) is not None + ldap.broken["alice"] = raised() + + plan = await service.reconcile_directory_sessions() + + assert (plan.probed, plan.asked, len(getattr(plan, field))) == (3, 2, 1) + [row] = [json.loads(a["detail"]) for a in await store.list_audit() if a["action"] == action] + assert row["asked"] == 2 and "probed" not in row + message = getattr(service, latch) + assert message is not None and "1 of 2 signed-in" in message + + counts: list[tuple[str, int]] = [] + + class _Sink(LoggingAlertSink): + def ad_reconcile_aborted( + self, name: str, *, reason: str, probed: int, detail: str + ) -> None: + counts.append((reason, probed)) + + _alert_reconcile_plan(plan, service, _Sink()) + assert counts == [(reason, 2)] + # The breaker's page keeps its judged denominator, which counts the id-less row. + counts.clear() + _alert_reconcile_plan(replace(plan, aborted="mass_revoke_breaker"), service, _Sink()) + assert counts == [("mass_revoke_breaker", 3), (reason, 2)] + finally: + await store.close() + + +@pytest.mark.parametrize(("raised", "action", "field", "latch", "reason"), _COUNTED_OVER_ASKED) +async def test_a_pass_of_faults_or_refusals_only_strikes_an_id_less_row_and_never_revokes_it( + raised: Any, action: str, field: str, latch: str, reason: str +) -> None: + """BACKLOG #3170 item 3 at the service, for a pass that judged nothing the directory answered. + The id-less row is struck on every such pass, past the strike threshold, and its session is + never revoked: only a pass that is not aborted can revoke it. Then one such pass ends, and the + row is revoked on the next clean pass, which proves the strikes were counted.""" + store = await MessageStore.open(":memory:") + try: + ldap = _FakeLdap( + { + "alice": _principal("alice"), + "jdoe": replace(_principal("jdoe"), directory_object_id=None), + } + ) + service = AuthService(store, _ad_settings(), ldap=ldap) # type: ignore[arg-type] + await service.initialize() + token = await _signed_in_ad_user(service, store, "jdoe") + assert token is not None and await _signed_in_ad_user(service, store, "alice") is not None + jdoe = await store.get_user_by_username("jdoe") + assert jdoe is not None + ldap.broken["alice"] = raised() + + for strike in (1, 2, 3): # the threshold is 2 + plan = await service.reconcile_directory_sessions() + assert plan.aborted == reason and plan.judged_nothing + assert plan.revocations == () and plan.strikes == {jdoe.id: strike} + assert await service.identity_for_token(token) is not None, "an aborted pass revoked" + audit = await store.list_audit() + assert not any(a["action"] == "auth.ad_session_revoked" for a in audit) + + del ldap.broken["alice"] # THE CONTROL: the first pass that is not aborted revokes it + plan = await service.reconcile_directory_sessions() + assert [(r.user_id, r.reason) for r in plan.revocations] == [ + (jdoe.id, "directory_object_id_missing") + ] + finally: + await store.close() + + +async def test_a_deleted_search_base_aborts_alerts_and_never_revokes( + caplog: pytest.LogCaptureFixture, +) -> None: + """BACKLOG #3010 limb 5. A deleted or moved user search base answers noSuchObject for every + account on every pass. Before #3010 it read "not found", struck everyone and tripped the + breaker; with #3010 alone it read as an outage, which pages nobody. Now each pass aborts as + ``directory_misconfigured``: one ERROR naming the code and no account, a latched message + naming the settings, a misconfigured row, and no skipped or aborted row. One refusal beside a + healthy account is recorded the same way; a pass with none releases the message. + + RED before D3b: the pass is ``directory_unavailable`` and nothing latches.""" + store = await MessageStore.open(":memory:") + try: + ldap = _FakeLdap({"jdoe": _principal("jdoe"), "asmith": _principal("asmith")}) + service = AuthService(store, _ad_settings(), ldap=ldap) # type: ignore[arg-type] + await service.initialize() + tokens = [await _signed_in_ad_user(service, store, u) for u in ("jdoe", "asmith")] + for name in ("jdoe", "asmith"): + ldap.broken[name] = _no_such_base() + + for _ in range(3): # past the strike threshold + caplog.clear() + with caplog.at_level(logging.WARNING, logger="messagefoundry.auth.service"): + plan = await service.reconcile_directory_sessions() + assert plan.aborted == reconcile.MISCONFIGURED_ABORT and plan.directory_misconfigured + assert not plan.directory_outage and plan.judged_nothing + assert len(plan.misconfigured) == 2 and plan.revocations == () + assert not plan.breaker_clear and not plan.hold_clear and not plan.referral_clear + errors = [r.getMessage() for r in caplog.records if r.levelno == logging.ERROR] + assert len(errors) == 1 and "noSuchObject" in errors[0] + assert "jdoe" not in errors[0] and "asmith" not in errors[0] + for token in tokens: + assert token is not None and await service.identity_for_token(token) is not None + message = service.directory_reconcile_misconfigured + assert message is not None and "revoked nothing" in message + assert "ad_user_search_base" in message[:200] + audit = await store.list_audit() + rows = [ + json.loads(a["detail"]) + for a in audit + if a["action"] == "auth.ad_reconcile_misconfigured" + ] + expected = { + "reason": "directory_misconfigured", + "misconfigured": 2, + "unavailable": 0, + "asked": 2, + "aborted": "directory_misconfigured", + } + assert rows == [expected] * 3 + assert not any( + a["action"] in ("auth.ad_reconcile_skipped", "auth.ad_reconcile_aborted") for a in audit + ) + + del ldap.broken["asmith"] # one refusal beside a healthy account + plan = await service.reconcile_directory_sessions() + assert plan.aborted is None and plan.misconfigured != () + assert service.directory_reconcile_misconfigured is not None + assert "judged as usual" in service.directory_reconcile_misconfigured + + del ldap.broken["jdoe"] # the base is restored + plan = await service.reconcile_directory_sessions() + assert plan.aborted is None and plan.misconfigured == () + assert service.directory_reconcile_misconfigured is None + finally: + await store.close() + + async def test_a_breaker_trip_does_not_mark_an_id_less_row_or_forfeit_its_clear() -> None: """BACKLOG #2434, the breaker's half of the referral test above. A trip marks every candidate unconfirmed until a pass reads it PRESENT, and revoking a marked account forfeits the clear. An diff --git a/tests/test_ad_user_account_control.py b/tests/test_ad_user_account_control.py index b822f95f21..768f750f54 100644 --- a/tests/test_ad_user_account_control.py +++ b/tests/test_ad_user_account_control.py @@ -38,6 +38,7 @@ from messagefoundry.api import app as app_module from messagefoundry.api import create_managed_app from messagefoundry.api.app import ( + _MISCONFIGURED_ALERT_SOURCE, _PROBE_FAULT_ALERT_SOURCE, _RECONCILE_CLEAR_RESOLVES, _REFERRAL_ALERT_SOURCE, @@ -51,6 +52,7 @@ from messagefoundry.auth.permissions import Role from messagefoundry.auth.reconcile import ( HOLD_REASON, + MISCONFIGURED_ABORT, PROBE_FAULT_ABORT, REFERRAL_ABORT, ProbeOutcome, @@ -862,6 +864,7 @@ def ad_reconcile_hold_released(self, name: str) -> None: directory_reconcile_hold=None, directory_reconcile_referral=None, directory_reconcile_fault=None, + directory_reconcile_misconfigured=None, ) #: The referral's own source label (BACKLOG #2538), apart from the breaker's. @@ -996,6 +999,74 @@ def test_a_probe_fault_pages_under_its_own_source( assert faulted and all(e[2]["reason"] == PROBE_FAULT_ABORT for e in faulted) +_REFUSAL_PAGE = ("ad_reconcile_aborted", _MISCONFIGURED_ALERT_SOURCE) + + +@pytest.mark.parametrize( + ("plan", "expected"), + [ + ( + ReconcilePlan( + probed=3, + asked=3, + misconfigured=("a",), + revocations=(SessionRevocation("b", "bwong", reason="directory_absent"),), + ), + [("ad_session_revoked", "bwong"), _REFUSAL_PAGE], + ), + ( + # A pass of refusals only: the refusal's page alone, never the breaker's instance. + ReconcilePlan(probed=2, asked=2, misconfigured=("a", "b"), aborted=MISCONFIGURED_ABORT), + [_REFUSAL_PAGE], + ), + ( + # A pass of referrals and refusals only: one page on each of their instances. + ReconcilePlan( + probed=2, asked=2, referred=("a",), misconfigured=("b",), aborted=REFERRAL_ABORT + ), + [("ad_reconcile_aborted", _REFERRAL_SOURCE), _REFUSAL_PAGE], + ), + ( + ReconcilePlan( + probed=2, asked=2, misconfigured=("a",), faulted=("b",), aborted=MISCONFIGURED_ABORT + ), + [_REFUSAL_PAGE, _FAULT_PAGE], + ), + ( + ReconcilePlan(probed=12, asked=12, misconfigured=("a",), aborted="mass_revoke_breaker"), + [("ad_reconcile_aborted", "directory-reconciler"), _REFUSAL_PAGE], + ), + ( + # THE CONTROL: a plain outage pages nothing. + ReconcilePlan(probed=2, asked=2, unavailable=2, aborted="directory_unavailable"), + [], + ), + ], + ids=[ + "beside-a-revocation", + "refusals-only", + "beside-a-referral", + "beside-a-fault", + "beside-a-trip", + "outage", + ], +) +def test_a_configuration_refusal_pages_under_its_own_source( + plan: ReconcilePlan, expected: list[tuple[str, str]] +) -> None: + """BACKLOG #3010 limb 5. A deleted or moved search base read as an outage with #3010, which + pages nobody. Every pass with a configuration refusal pages under its own source, counted + over the probes asked, so an acknowledged refusal never silences a breaker trip or a referral. + + RED before D3b: a pass of refusals only pages nothing.""" + sink = _InverseSink() + _alert_reconcile_plan(plan, _AUTH, sink) # type: ignore[arg-type] + assert [(e[0], e[1]) for e in sink.events] == expected + for event in sink.events: + if (event[0], event[1]) == _REFUSAL_PAGE: + assert event[2]["reason"] == MISCONFIGURED_ABORT and event[2]["probed"] == plan.asked + + def test_the_inverses_resolve_their_alerts_and_are_not_rule_targetable() -> None: assert _AUTO_RESOLVE["ad_reconcile_breaker_cleared"] == "ad_reconcile_aborted" assert _AUTO_RESOLVE["ad_reconcile_hold_released"] == "ad_reconcile_held" diff --git a/tests/test_ldap_decode_boundary.py b/tests/test_ldap_decode_boundary.py index 365933571a..bc15b30dcb 100644 --- a/tests/test_ldap_decode_boundary.py +++ b/tests/test_ldap_decode_boundary.py @@ -21,7 +21,9 @@ from __future__ import annotations import ast +import contextlib import inspect +import time from collections.abc import Callable from types import SimpleNamespace from typing import Any @@ -29,10 +31,14 @@ import ldap3 import pytest from ldap3.core.results import RESULT_CODES, RESULT_SUCCESS +from ldap3.utils import asn1 from ldap3.utils.asn1 import decode_sequence from messagefoundry.auth import ldap as ldap_module -from messagefoundry.auth.ldap import LdapError, LdapUndecodableError +from messagefoundry.auth.ldap import LdapError, LdapReferralError, LdapUndecodableError +from messagefoundry.auth.reconcile import ProbeOutcome +from messagefoundry.auth.service import AuthService +from messagefoundry.store.store import MessageStore from tests._ast_sites import callee_name, parse_source from tests.test_ldap_referrals import _BIND_PASSWORD, _BIND_RESPONSE, _SEARCH_DONE, _tlv from tests.test_ldap_search_result_codes import ( @@ -47,6 +53,7 @@ _scripted_dc, _service_conn, ) +from tests.test_ldap_timeouts import _ad_settings #: Stands for text the directory sends: it must never reach an error, a log line or an audit row. _REFERRED = "ldap://dc9.synthetic-leak-3010.invalid/DC=other" @@ -68,13 +75,12 @@ def _result(tag: int, code: int, *, matched: bytes | None = None, extra: bytes = return _tlv(tag, _tlv(0x0A, _enumerated(code)) + matched_dn + _tlv(0x04, b"") + extra) +#: A bind response carrying a referral, which ldap3's decoder cannot read (``KeyError(3)``). It is +#: refused as a referral, not as an undecodable response (D3b, BACKLOG #3010 and #2538). +_REFERRAL_BIND = _result(_BIND_RESPONSE, 10, extra=_tlv(0xA3, _tlv(0x04, _REFERRED.encode()))) + #: Malformed bind responses: the reply, and what ldap3's decoder raises on it. _BAD_BINDS = [ - pytest.param( - _result(_BIND_RESPONSE, 10, extra=_tlv(0xA3, _tlv(0x04, _REFERRED.encode()))), - KeyError, - id="referral-KeyError-3", - ), pytest.param( _result(_BIND_RESPONSE, RESULT_SUCCESS, extra=_tlv(0x96, b"synthetic")), KeyError, @@ -203,6 +209,151 @@ def bind(raw: bytes) -> _Reply | None: assert texts[0] == texts[1] +# --- a bind response carrying a referral is a referral (D3b) -------------------------------------- + + +def _assert_referred_bind(exc: BaseException, operation: str) -> None: + """The refusal is a referral naming the operation, with no host: the hosts are in the part + ldap3 could not decode, and a value the directory sent never reaches the text.""" + assert isinstance(exc, LdapReferralError), repr(exc) + text = str(exc) + assert f"AD answered the {operation} with a referral" in text + assert "could not decode" not in text and "result code" not in text + assert _REFERRED not in text and "synthetic-leak" not in text, "a referred host leaked" + assert isinstance(exc.__cause__, KeyError) and exc.__cause__.args == (3,) + + +def test_a_service_bind_answered_with_a_referral_is_a_referral() -> None: + """Before D3b it read as an undecodable response, an outage the reconciler pages nobody for. + THE PREMISE first: the bare ldap3 bind raises ``KeyError(3)``. The socket is still closed.""" + with _scripted_dc(lambda _raw: [], bind_reply=lambda _raw: (_REFERRAL_BIND, b"")) as ( + dc, + authenticator, + ): + with pytest.raises(KeyError) as bare: + _bare_bind(dc.port) + assert bare.value.args == (3,) + _reading_connection(bare.value).strategy.close() + assert dc.ended.wait(_CLOSE_WAIT) + dc.ended.clear() + with pytest.raises(LdapReferralError) as refused: + authenticator.probe_principal("alice") + _assert_referred_bind(refused.value, "service-account bind") + assert dc.searches == [] + assert dc.ended.wait(_CLOSE_WAIT), "the referred bind's socket was left open" + + +def test_a_user_bind_answered_with_a_referral_fails_a_present_and_an_absent_account_alike() -> None: + """The decoy bind meets the same referral, so the two branches refuse with one text.""" + + def bind(raw: bytes) -> _Reply | None: + is_user = b"CN=alice" in raw or b"mf-nonexistent-timing-equalizer" in raw + return (_REFERRAL_BIND, b"") if is_user else None + + texts = [] + with _scripted_dc(_alice, bind_reply=bind) as (_dc, authenticator): + for username in ("alice", "nobody"): + with pytest.raises(LdapReferralError) as refused: + authenticator.authenticate(username, "synthetic-user-pw") + _assert_referred_bind(refused.value, "user bind") + texts.append(str(refused.value)) + assert texts[0] == texts[1] + + +async def test_the_reconciler_reads_a_referred_service_bind_as_referred() -> None: + """So the probe reaches the referral's alert path (BACKLOG #2538), not the outage's.""" + with _scripted_dc(lambda _raw: [], bind_reply=lambda _raw: (_REFERRAL_BIND, b"")) as ( + dc, + authenticator, + ): + store = await MessageStore.open(":memory:") + try: + settings = _ad_settings( + ad_server=f"ldap://127.0.0.1:{dc.port}", ad_allow_insecure_ldap=True + ) + service = AuthService(store, settings, ldap=authenticator) + user = SimpleNamespace(id="u-1", username="alice", directory_object_id=None) + probe = await service._probe_principal(user) # type: ignore[arg-type] + finally: + await store.close() + assert probe.outcome is ProbeOutcome.REFERRED + assert probe.detail is not None and _REFERRED not in probe.detail + + +def test_a_referral_in_a_who_am_i_response_is_still_undecodable() -> None: + """THE CONTROL: only a BIND response's referral field is read as a referral. ldap3's decoder + raises the same ``KeyError(3)`` on an extended response carrying one, and that stays an + undecodable response, because the origin check reads which response was being decoded.""" + reply = ( + _result(_EXTENDED_RESPONSE, 10, extra=_tlv(0xA3, _tlv(0x04, _REFERRED.encode()))), + b"", + ) + with _scripted_dc(lambda _raw: [_done(RESULT_SUCCESS)], extended_reply=reply) as ( + _dc, + authenticator, + ): + with _service_conn(authenticator) as conn, pytest.raises(KeyError) as bare: + conn.extend.standard.who_am_i() + assert bare.value.args == (3,) + reading = authenticator.read_bind_account() + assert reading.whoami_error is not None + assert "Who am I with a response ldap3 could not decode (KeyError)" in reading.whoami_error + + +# --- the service connection is released by every lookup (D3b) ------------------------------------ + + +def _groups_answered(raw: bytes) -> list[_Reply]: + """The user search finds alice; the nested group search ends with success and no group.""" + return [_done(RESULT_SUCCESS)] if _IN_CHAIN in raw else _alice(raw) + + +def _group_search_busy(raw: bytes) -> list[_Reply]: + return [_done(51)] if _IN_CHAIN in raw else _alice(raw) + + +def _group_page_undecodable(raw: bytes) -> list[_Reply]: + no_value = _tlv(0x30, _tlv(0x04, _OID)) # ldap3 raises TypeError on it + return [_bad_page(no_value)] if _IN_CHAIN in raw else _alice(raw) + + +def _probe(name: str) -> Callable[[Any], object]: + return lambda authenticator: authenticator.probe_principal(name) + + +def _sign_in(name: str) -> Callable[[Any], object]: + return lambda authenticator: authenticator.authenticate(name, "synthetic-user-pw") + + +@pytest.mark.parametrize( + ("answer", "call", "sessions"), + [ + pytest.param(_groups_answered, _probe("alice"), 1, id="probe-present"), + pytest.param(_groups_answered, _probe("nobody"), 1, id="probe-absent"), + pytest.param(_group_search_busy, _probe("alice"), 1, id="probe-group-search-busy"), + pytest.param(_group_page_undecodable, _probe("alice"), 1, id="probe-undecodable-page"), + pytest.param(_groups_answered, _sign_in("alice"), 2, id="sign-in-present"), + pytest.param(_groups_answered, _sign_in("nobody"), 2, id="sign-in-absent-decoy"), + pytest.param(_group_search_busy, _sign_in("alice"), 2, id="sign-in-group-search-busy"), + ], +) +def test_every_directory_session_a_lookup_opens_is_ended( + answer: Callable[[bytes], list[_Reply]], call: Callable[[Any], object], sessions: int +) -> None: + """D3b, a finding of BACKLOG #3010's review. ``authenticate`` and ``probe_principal`` held the + service connection in ldap3's own ``with``, which never unbinds a connection bound on entry, + so every lookup left one directory session open until the connection was collected. Counted + at the loopback server, by the unbind requests it receives, whatever the lookup answered.""" + with _scripted_dc(answer) as (dc, authenticator): + with contextlib.suppress(LdapError): + call(authenticator) + assert dc.opened == sessions, "the lookup did not open the sessions this case names" + deadline = time.monotonic() + _CLOSE_WAIT + while dc.unbound < dc.opened and time.monotonic() < deadline: + time.sleep(0.01) + assert dc.unbound == dc.opened, f"{dc.opened} session(s) opened, {dc.unbound} ended" + + # --- the nested group search: a malformed paged results control --------------------------------- _OID = ldap_module._PAGED_RESULTS_OID.encode() @@ -397,6 +548,9 @@ def test_the_ldap3_methods_the_origin_check_names_still_exist() -> None: assert readers["ldap3.extend."] == {"decode_response", "populate_result", "set_response"} assert {"decode_response", "set_response"} <= set(vars(ExtendedOperation)) assert "populate_result" in vars(WhoAmI) + # What _is_bind_referral reads (D3b): the decoder's locals, and a bind table with no referral. + assert {"context_decoders", "ber_type"} <= set(asn1.decode_sequence.__code__.co_varnames) + assert ldap_module._REFERRAL_TAG not in asn1.BIND_RESPONSE_CONTEXT def test_an_unbind_error_never_replaces_the_refusal() -> None: diff --git a/tests/test_ldap_search_result_codes.py b/tests/test_ldap_search_result_codes.py index 5908d41d2c..a43d53b99c 100644 --- a/tests/test_ldap_search_result_codes.py +++ b/tests/test_ldap_search_result_codes.py @@ -64,15 +64,17 @@ LdapUnansweredError, ) from messagefoundry.auth.permissions import Role -from messagefoundry.auth.reconcile import ProbeOutcome +from messagefoundry.auth.reconcile import MISCONFIGURED_ABORT, ProbeOutcome from messagefoundry.auth.service import AuthService from messagefoundry.store.store import MessageStore from tests.test_ad_user_account_control import _alive, _audited, _pass, _signed_in_estate +from tests.test_approval_requester_recheck import _Sink from tests.test_ldap_referrals import ( _BIND_REQUEST, _BIND_RESPONSE, _SEARCH_DONE, _SEARCH_REQUEST, + _UNBIND_REQUEST, _read_message, _tlv, ) @@ -308,27 +310,28 @@ def test_a_group_search_the_directory_did_not_finish_is_an_outage_not_fewer_grou @pytest.mark.parametrize( - ("code", "partial"), + ("code", "partial", "outcome"), [ - (RESULT_BUSY, False), - (RESULT_ADMIN_LIMIT_EXCEEDED, False), - (RESULT_NO_SUCH_OBJECT, False), - (RESULT_SIZE_LIMIT_EXCEEDED, True), + (RESULT_BUSY, False, ProbeOutcome.UNAVAILABLE), + (RESULT_ADMIN_LIMIT_EXCEEDED, False, ProbeOutcome.UNAVAILABLE), + (RESULT_NO_SUCH_OBJECT, False, ProbeOutcome.MISCONFIGURED), + (RESULT_SIZE_LIMIT_EXCEEDED, True, ProbeOutcome.UNAVAILABLE), ], ids=["busy", "adminLimitExceeded", "noSuchObject", "sizeLimit-partial"], ) async def test_a_directory_that_did_not_answer_never_revokes_a_present_account( - monkeypatch: pytest.MonkeyPatch, code: int, partial: bool + monkeypatch: pytest.MonkeyPatch, code: int, partial: bool, outcome: ProbeOutcome ) -> None: """The row's defect, end to end. One signed-in account, under the breaker's floor, whose user search the directory answers with ``code`` on every pass. Before the fix the user search read - ABSENT and the account would be revoked at the strike threshold. Now each probe is unavailable - and the session survives. The group search has its own test below, which also reads roles.""" + ABSENT and the account would be revoked at the strike threshold. Now each probe is unavailable, + or misconfigured for a code that points at the configuration, and the session survives. The + group search has its own test below, which also reads roles.""" async with _signed_in_estate(monkeypatch, ["jdoe"]) as (_directory, service, store, tokens): _answer_with(monkeypatch, code, when=_user_search, keep_entries=partial) user = await store.get_user_by_username("jdoe") assert user is not None - assert (await service._probe_principal(user)).outcome is ProbeOutcome.UNAVAILABLE + assert (await service._probe_principal(user)).outcome is outcome for _ in range(5): # far past ad_session_recheck_strikes assert (await _pass(service)).revocations == () assert await _alive(service, tokens) == {"jdoe"} @@ -350,6 +353,64 @@ async def test_an_account_that_left_the_directory_is_still_revoked( assert await _alive(service, tokens) == set() +@pytest.mark.parametrize( + ("code", "paged"), + [ + (RESULT_NO_SUCH_OBJECT, True), + (RESULT_INVALID_DN_SYNTAX, True), + (RESULT_INSUFFICIENT_ACCESS_RIGHTS, True), + # THE CONTROLS: a transient code is a plain outage, which pages nobody. + (RESULT_BUSY, False), + (RESULT_UNAVAILABLE, False), + ], + ids=[ + "32-noSuchObject", + "34-invalidDNSyntax", + "50-insufficientAccess", + "51-busy", + "52-unavailable", + ], +) +async def test_a_refused_user_search_base_above_the_breaker_floor_pages_and_never_revokes( + monkeypatch: pytest.MonkeyPatch, code: int, paged: bool +) -> None: + """D3b, #3010 limb 5, through the real authenticator. Eight signed-in accounts, above the + breaker's floor, whose user search base the directory refuses on every pass. Before #3010 a + deleted base read "not found", struck all eight and tripped the breaker; with #3010 alone it + read as an outage that paged nobody. A configuration code now aborts every pass as + ``directory_misconfigured`` and pages under its own source, counted over the probes asked. It + never revokes, and it is never the breaker's trip. + + RED before D3b for the three configuration codes: each pass is a plain outage.""" + names = [f"user{i:02d}" for i in range(8)] + async with _signed_in_estate(monkeypatch, names) as (_directory, service, store, tokens): + _answer_with(monkeypatch, code, when=_user_search) + sink = _Sink() + for _ in range(4): # far past ad_session_recheck_strikes + plan = await _pass(service, sink) + assert plan.revocations == () + if paged: + assert plan.aborted == MISCONFIGURED_ABORT + else: + assert plan.directory_outage + assert await _alive(service, tokens) == set(names) + assert await _audited(store, "auth.ad_session_revoked") == [] + assert await _audited(store, "auth.ad_reconcile_aborted") == [] + refused = await _audited(store, "auth.ad_reconcile_misconfigured") + assert len(refused) == (4 if paged else 0) + assert all(_LEAK not in str(dict(r)["detail"]) for r in refused) + pages = [ + (e[1], e[2]["reason"], e[2]["probed"]) + for e in sink.events + if e[0] == "ad_reconcile_aborted" + ] + expected = ("directory-reconciler-misconfigured", MISCONFIGURED_ABORT, len(names)) + assert pages == ([expected] * 4 if paged else []) + detail = service.directory_reconcile_misconfigured + assert (detail is not None) is paged + assert detail is None or _LEAK not in detail + + async def test_a_busy_account_beside_a_departed_one_is_left_alone_while_the_other_is_revoked( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -739,7 +800,8 @@ class _ScriptedDc: answered by ``bind_reply(raw request)``, or with success where that is absent or returns ``None``; an extended operation such as Who am I is answered with ``extended_reply``, or ends with ``extended_code``; each search is answered by ``answer(raw request)``. ``searches`` holds every search request's bytes, and - ``ended`` is set when a client's socket closes.""" + ``ended`` is set when a client's socket closes. ``opened`` counts the sessions accepted, and + ``unbound`` the sessions a client ended with an unbind request.""" def __init__( self, @@ -752,6 +814,9 @@ def __init__( self._extended_reply = extended_reply self.searches: list[bytes] = [] self.ended = threading.Event() + self.opened = 0 + self.unbound = 0 + self._count_lock = threading.Lock() self._listener = socket.create_server(("127.0.0.1", 0)) self.port: int = self._listener.getsockname()[1] threading.Thread(target=self._accept, daemon=True).start() @@ -765,6 +830,8 @@ def _accept(self) -> None: threading.Thread(target=self._session, args=(sock,), daemon=True).start() def _session(self, sock: socket.socket) -> None: + with self._count_lock: + self.opened += 1 try: with sock, contextlib.suppress(OSError): sock.settimeout(10) @@ -784,6 +851,9 @@ def _session(self, sock: socket.socket) -> None: self.searches.append(raw) replies = self._answer(raw) else: # unbind, or anything this server does not speak + if op == _UNBIND_REQUEST: + with self._count_lock: + self.unbound += 1 return for reply, controls in replies: sock.sendall(_tlv(0x30, message_id + reply + controls)) diff --git a/tests/test_ldap_timeouts.py b/tests/test_ldap_timeouts.py index 3f8547684b..779c4e3e73 100644 --- a/tests/test_ldap_timeouts.py +++ b/tests/test_ldap_timeouts.py @@ -378,9 +378,12 @@ def test_no_login_can_be_told_from_another_by_the_directory_work_it_costs() -> N # never driven. Only the bind/release pair is pinned here: the construction counts are already # owned by test_authenticate_builds_only_finitely_timed_ldap_objects, and a second copy of that # number is a second place to update. + # Two releases: the user (or decoy) connection, and since BACKLOG #3010 the service-account + # connection too, which ldap3's own ``with`` had left open on every branch alike. _, _, binds, unbinds = next(iter(measured.values())) - assert (binds, unbinds) == (1, 1), ( - f"expected every branch to issue exactly one bind and release it once; got {measured}" + assert (binds, unbinds) == (1, 2), ( + "expected every branch to issue exactly one bind and release both connections once; got " + f"{measured}" ) @@ -670,7 +673,9 @@ def test_a_rejected_password_still_unbinds_the_user_connection( assert len(rec.connections) == 2, ( f"expected a service bind then a user bind; recorded {len(rec.connections)}" ) - assert len(rec.unbinds) == 1, ( + # The service connection is released too since BACKLOG #3010, so count the user's by its DN. + assert rec.unbinds.count("CN=alice,OU=Users,DC=example,DC=com") == 1, ( "the user connection was NOT unbound on the rejected-password path — the release procedure " "documented in docs/CONNECTIONS.md Table B does not run where it matters most (ASVS 13.1.3)" ) + assert len(rec.unbinds) == 2, f"expected the service and user connections; got {rec.unbinds}" diff --git a/tests/test_mfa_directory_recheck.py b/tests/test_mfa_directory_recheck.py index 16459b18fc..9b254c7e04 100644 --- a/tests/test_mfa_directory_recheck.py +++ b/tests/test_mfa_directory_recheck.py @@ -34,6 +34,7 @@ DirectoryProbe, LdapError, LdapReferralError, + LdapUnansweredError, ) from messagefoundry.auth.service import DIRECTORY_UNCONFIRMED, AuthService from messagefoundry.auth.tokens import hash_token @@ -201,8 +202,14 @@ async def test_a_directory_the_account_is_not_confirmed_in_refuses_the_renewal( (ValueError("synthetic: malformed objectGUID"), "probe_fault"), # A network-class error that escaped the directory layer is still unreachable. (TimeoutError("synthetic: read timed out"), "unavailable"), + # BACKLOG #3010: a code that points at the configuration, and one that does not. + ( + LdapUnansweredError("synthetic: noSuchObject", code=32, code_name="noSuchObject"), + "misconfigured", + ), + (LdapUnansweredError("synthetic: busy", code=51, code_name="busy"), "unavailable"), ], - ids=["referral", "key-error", "value-error", "timeout"], + ids=["referral", "key-error", "value-error", "timeout", "no-such-object", "busy"], ) async def test_a_lookup_that_raises_refuses_the_renewal_rather_than_failing( store: MessageStore, From e11a6b27c0495a141d16eddf191baf96f1ce919c Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Sat, 17 Oct 2026 01:18:50 -0500 Subject: [PATCH 30/57] Docs and changelogs for the reconcile follow-ups (BACKLOG #3010, #3170) SECURITY.md: the configuration refusal paragraph; the breaker bullet no longer says a deleted base reads "not found"; a pass where every probe fails and none faults is a plain outage, which pages nobody. PHI.md: the alert_instance row names the new source and slug, and every alert but the breaker's counts the lookups sent. changelog.d/3010 and 3170 rewritten to state the result; 3170's orphan line and garbled sentence are gone. --- changelog.d/3010.security.md | 54 +++++++++++++++++++++++------------- changelog.d/3170.security.md | 23 +++++++-------- docs/PHI.md | 2 +- docs/SECURITY.md | 38 +++++++++++++++++++------ 4 files changed, 77 insertions(+), 40 deletions(-) diff --git a/changelog.d/3010.security.md b/changelog.d/3010.security.md index f207f5cdff..5dcc53cec5 100644 --- a/changelog.d/3010.security.md +++ b/changelog.d/3010.security.md @@ -1,17 +1,28 @@ -- **An LDAP search that does not end with success is an outage, never "no such account".** Each - search the engine sends refuses any result code but success, with or without entries. That - includes busy, unavailable, unwillingToPerform and adminLimitExceeded. It includes a time or size - limit, and noSuchObject on a missing search base. Before, such a search read as no entries. The - session reconciler would then have struck a present account toward revocation. A busy nested - group search read as fewer groups. The reconciler now reads these as unavailable, which never - revokes. A sign-in writes `auth.login_error` naming the code, and `check-privileges` names the - code for the bind-account read. These refusals name the operation and the code, never the - directory's diagnostic message. Other directory errors keep the text they had. +- **An LDAP search that does not end with success is never "no such account".** Each search the + engine sends refuses any result code but success, with or without entries. That includes busy, + unavailable, unwillingToPerform and adminLimitExceeded. It includes a time or size limit, and + noSuchObject on a missing search base. Before, such a search read as no entries. The session + reconciler would then have struck a present account toward revocation. A busy nested group + search read as fewer groups. A sign-in now writes `auth.login_error` naming the code, and + `check-privileges` names the code for the bind-account read. These refusals name the operation + and the code, never the directory's diagnostic message. Other directory errors keep the text + they had. + **The session reconciler tells a configuration refusal from an outage.** noSuchObject, + invalidDNSyntax and insufficientAccessRights point at the configuration: a search base that was + deleted or moved, or that the service account may not read. Such a refusal leaves that account + unjudged and never revokes it. Every pass with one logs an ERROR naming the code, audits + `auth.ad_reconcile_misconfigured`, and raises `ad_reconcile_aborted` with reason + `directory_misconfigured` under the source `directory-reconciler-misconfigured`. A pass where + every lookup is refused, faulted or fails, at least one refused and none referred, aborts as + `directory_misconfigured`. Before, a deleted search base read as "not found" for every account, + which tripped the mass-revoke breaker on a large enough estate. Nothing resolves the new alert + automatically; an operator does. Any other code is an outage, which never revokes and pages + nobody. Step-up refuses a refused account with outcome `misconfigured`. **Not covered:** ldap3 fetches a ranged `memberOf` with follow-up searches of its own, and those are not refused. **The cost:** a code that recurs on every pass, such as a deleted search base, stops revocation - and the role re-diff for as long as it lasts. It warns once per process for each operation and - code, and latches no alert. + and the role re-diff for as long as it lasts. A configuration code alerts; any other code warns + once per process for each operation and code, and latches no alert. **The nested group search is paged**, at 500 entries per page. So an account in more groups than a directory's unpaged cap no longer fails sign-in with sizeLimitExceeded. AD's cap, MaxPageSize, is 1000 by default. A page that does not end with success refuses the whole search. So does a @@ -21,11 +32,16 @@ against a loopback server through ldap3, not against a domain controller. **A response ldap3 cannot decode is an outage, not an unhandled error.** ldap3's default decoder raises `KeyError`, `TypeError`, `IndexError`, `AttributeError` or `UnicodeError` on a malformed - response. Cases include at least a bind response carrying a referral, a result code outside its - table, a malformed paged results control, and a Who am I value that is not text. A sign-in that - met one would have failed with an unhandled error and no audit row. The searches, binds and - Who am I read the engine sends now run inside one boundary. It maps such a fault to an outage when - ldap3 raised it reading the response, and lets any other exception propagate. Such a fault now - reaches the session reconciler as an outage, which it logs at debug level, rather than as an - unexpected error, which it logged at WARNING for each account. After one, the engine closes the - socket of a service-account bind that met it. (vault `BACKLOG #3010`) + response. Cases include at least a result code outside its table, a malformed paged results + control, and a Who am I value that is not text. A sign-in that met one would have failed with an + unhandled error and no audit row. The searches, binds and Who am I read the engine sends now run + inside one boundary. It maps such a fault to an outage when ldap3 raised it reading the response, + and lets any other exception propagate. The session reconciler reads it as an outage and logs it + at WARNING once per process for each kind, where it once logged a WARNING for each account. + After one, the engine closes the socket of a service-account bind that met it. + **A bind response carrying a referral is a referral.** ldap3 cannot decode one. The boundary + refuses it as a referral, naming no host, so the session reconciler alerts on it as on any + referral. + **The service-account connection is unbound** after every sign-in lookup and reconciler probe. + ldap3's own `with` block never unbound it, so each lookup left a directory session open until the + connection was collected. (vault `BACKLOG #3010`) diff --git a/changelog.d/3170.security.md b/changelog.d/3170.security.md index 9008375017..634f18fd3d 100644 --- a/changelog.d/3170.security.md +++ b/changelog.d/3170.security.md @@ -1,11 +1,12 @@ -- **The session reconciler counts a referral over the lookups it sent, and strikes an id-less row - on every pass.** A directory row with no `objectGUID` is judged without a lookup, so no referral - can answer it. The referral's `auth.ad_reconcile_referred` audit row now carries the `asked` - field in place of `probed`, and that row, the referral's latched message and its - `ad_reconcile_aborted` alert count only the lookups the pass sent. The breaker's own alert still - counts every account it judged. Such a row is also struck now on a pass that is a directory - outage or aborted on referrals or probe faults, as on any other pass. Such a pass still revokes - nothing. Only a - pass that is not aborted can, once the row's strikes reach the threshold and the row is in that - pass's probe budget, so while every pass aborts its sessions last to their own expiry. Its revocation still counts toward the mass-revoke breaker, as before. - (`vault BACKLOG #3170`) +- **The session reconciler counts a referral or a probe fault over the lookups it sent, and strikes + an id-less row on every pass.** A directory row with no `objectGUID` is judged without a lookup, + so no referral or fault can answer it. The `auth.ad_reconcile_referred` and + `auth.ad_reconcile_faulted` audit rows now carry the `asked` field in place of `probed`. Each + row, its latched message and its `ad_reconcile_aborted` alert count only the lookups the pass + sent. The breaker's alert still counts every account the pass judged, its denominator. + An id-less row is now struck on every pass, as on a pass that judged the directory's answers. + That includes a directory outage, and a pass aborted because every lookup was referred, faulted + or failed. No aborted pass revokes the row. Only a pass that is not aborted can, once the row's + strikes reach the threshold and the row is in that pass's probe budget. So while every pass + aborts, its sessions last to their own expiry. Its revocation still counts toward the + mass-revoke breaker, as before. (`vault BACKLOG #3170`) diff --git a/docs/PHI.md b/docs/PHI.md index c44aa755b5..a36917e2e1 100644 --- a/docs/PHI.md +++ b/docs/PHI.md @@ -1808,7 +1808,7 @@ with materially different PHI profiles, so they get their own rows; stream 4 is | **5. `audit_log` table** (SQLite, Postgres, SQL Server) | who / what / **where-from** / when of auth + PHI *access* and admin actions — plus, while `[security].audit_all_authorization_decisions` is on (**default `true`** since BACKLOG #1277, 2026-09-02, which reversed the ADR 0118 §5 `false`; the internal field it desugars to is `audit_all_authz`, whose old `[diagnostics]` TOML spelling is **refused at load** — ADR 0118), an `authz` row for **every** authorization decision including successes. That is the shipped volume of this stream, not an opt-in addition to it: one row per authenticated request on each `require()`-gated route, and this row's retention cell records that **nothing prunes the table** — `actor`, `action`, `channel_id`, `client`, `detail`, `row_hash` | JSON `detail`; **tamper-evident hash chain** over `prev_hash` + the row (the `client` address is **inside** the chained payload — ADR 0150) | the store database | HIPAA §164.312(b) audit controls; incident response; `verify_audit_chain` integrity checks | `GET /audit` requires **`audit:read`**; `GET /audit/export` requires the separate **`audit:export`** and streams CSV with formula-injection neutralisation, recording its own `audit.export` row *before* streaming; `GET /me/security-events` is a per-user view of the same table | **`[retention].audit_days` is reserved and NOT enforced — keep-forever by design** (the audit-retention requirement, ~6 years — **not** chain-breakage; [§8](#8-retention--purge) states the position and cites its source of record) | `detail` is stored **in the clear** (it is not a cipher-covered column): its protection is that writers store at least filter shapes, counts, ids, keyed digests, bounded engine-authored text and client addresses, the `audit_log.detail` row above says which — never bodies or credentials — plus the store ACL and the volume layer. *Corrected 2026-10-16 (vault BACKLOG #3145):* this said writers "only ever store filter shapes, counts and ids" | | **6. `message_events` table** | the per-message disposition timeline — the **complete** vocabulary is `received`, `routed`, `unrouted`, `filtered`, `transformed`, `delivered`, `failed`, `dead`, `error`, `replayed`, `resent`, `reingressed`, `passthrough`, `passthrough_dropped`, `cancelled`, `edit_resend`, `edit_resubmit`, `viewed`, `not_deployed`, the ADR 0154 synchronous-reply pair `reply_returned` / `reply_timeout` (names, counts and `waited_ms` only — **never** a fragment of the partner's reply body), and `reply_dropped`, a delivery whose 2xx reply body was over the byte bound and dropped (the status and the bound only, vault BACKLOG #3040) (CI asserts this list against the engine's own `MESSAGE_EVENT_KINDS`). `[diagnostics].message_events` can thin the set, but never below the compliance floor `viewed` / `dead` / `error` / `failed` / `not_deployed` / `reply_timeout` / `reply_dropped` | rows: `message_id`, `ts`, `event`, `destination`, `detail` | the store database | operator timeline on the message-detail view; the `viewed` row is the HIPAA PHI-access record | `GET /messages/{id}` under **`messages:view_raw`** + `require_phi_read`; the read itself writes a `viewed` event **and** a `message_view` audit row | no dedicated window — `purge_message_bodies` sets `message_events.detail` to `NULL` in the same transaction that blanks the body, so it inherits `[retention].messages_days` | `detail` is `safe_text()`-scrubbed **then** cipher-encrypted (AAD `("message_events","detail",message_id,ts,event)`). Verbosity gate `[diagnostics].message_events` = `all` (default) / `errors` / `off`, with a **compliance floor that can never be thinned**: `viewed`, `dead`, `error`, `failed`, `not_deployed`, `reply_timeout`, `reply_dropped` are retained at every level (`reply_timeout` is the one row that explains a "we called you and got a 504" complaint, so an instance that thinned its logs would lose exactly the record it is later asked for; `reply_dropped` is the one row that answers "you ignored our reply", for the same reason) | | **7. `connection_event` table — DEFAULT ON** (`[diagnostics].connection_events = true`) | transport/lifecycle events per connection: `established`, `closed` (reason `eof`, `idle_timeout` or `frame_deadline` — a fixed vocabulary chosen by the listener, never free text), `idle_timeout`, `at_capacity`, `peer_not_allowlisted`, `frame_oversize`, `framing_error`, `peer_reset`, the MLLP listener's `handler_error` (BACKLOG #1619 -- the inbound handler faulted on a frame the listener read cleanly, such as a store outage at the ingress commit; the reason is scrubbed at the source like every other), the MLLP listener's `tls_handshake_failed` (vault BACKLOG #2613 -- a TLS handshake on a listener that runs its own handshakes (the standard asyncio loop, not uvloop) failed; the first at once, then one per listener per minute carrying the count since the last, and the reason is a count and an exception class name, never the peer's bytes), the DATABASE poll source's `row_undecodable` (BACKLOG #1662 — a polled row the source cannot turn into a body; the reason is scrubbed at the source and names the exception type or the operator's own configured column, never a row value), the FILE poll source's quarantine kinds `file_oversize` / `file_decompress_failed` / `file_content_mismatch` / `file_scan_rejected` (BACKLOG #1621 — a drop moved to `.error`; the reason is a size, a content type or a scrubbed codec or scanner message, never the file name), the inbound-HTTP intake-auth refusals `intake_auth_failed` / `auth_subject_denied` / `auth_rate_limited` (ADR 0154 D6 — peer address and mode only; **never** the credential, a prefix of it, or its length. Each of these also writes a tamper-evident audit-log row — the copy that survives an operator turning this diagnostics stream off), plus the runner's `connection_lost` / `connection_restored`. That is the whole vocabulary, asserted in CI against the literal emit call sites in `transports/` and the pipeline runner **and** cross-checked against the console's own filter tuple. The MLLP, raw-TCP, X12 and HTTP listeners emit these; the **DATABASE poll source** emits `row_undecodable` and nothing else (it dials out, so there is no lifecycle to report and its rows carry a **NULL peer column**); the **FILE poll source** emits only its four quarantine kinds, also with a NULL peer column; the **DICOM inbound C-STORE SCP** emits none — the runner injects the sink onto **every** source (`wiring_runner.py`, over the base-class `on_connection_event` field), so that connector *has* the wiring and simply never calls it — so this stream covers those four listeners plus those poll-source events and the runner's outbound-lane transitions — not literally every connection. A DICOM association's connects and refusals are therefore **absent** from this stream. The **`ISA`/`IEA`-framed X12 inbound** was equally absent until BACKLOG #1665 and is not any more: it emits the same seven kinds as the raw-TCP listener | rows: `ts`, `connection`, `transport`, `direction`, `kind`, `peer_host`, `message_id` (correlation hint), `reason` | the store database, **all three backends** | Corepoint-style transport diagnostics — "did the sender connect, and why did it drop" | `GET /events` and `GET /connections/{name}/events` under **`monitoring:read`** (**not** a PHI permission) with per-channel RBAC — an out-of-scope `connection=` is 403'd *and* audited — server-clamped to ≤1000 rows. `reason` alone is gated on `messages:view_summary` and masked as `****` until the audited per-event `reveal=` act (BACKLOG #2443; the `connection_event.reason` row of §3's PL-2 block says how) | `[retention].connection_event_retention_hours` (its own **hours** window); 0 inherits `[retention].messages_days`; both 0 = keep forever. Plain age `DELETE` (metadata-only) | **`reason` is free text that can carry sensitive fragments.** Defended twice — `safe_exc()` at the source, `safe_text(reason, limit=200)` at the store (the bound §3's PL-2 block states) — then cipher-encrypted (AAD `("connection_event","reason",connection,ts,kind)`). Every other column is config metadata; the table is documented **metadata-only** — never a frame, body or HL7 field value. Writes are a pure side observer: a bounded in-memory queue drained by a background task outside any handoff transaction, so a flood can never block a listener or pin a message disposition | -| **8. `alert_instance` table — default on wherever an `[alerts]` notifier exists** | resolvable operator alerts: `connection_stopped`, `queue_buildup`, `lane_stuck`, `message_stall`, `saturation`, `connection_error`, `storage_threshold`, `cert_expiry`, `secret_rotation`, `initial_credential_expiring` (an UNCLAIMED admin-issued temporary password nearing the instant the login gate stops accepting it -- ASVS 6.4.5, BACKLOG #1141; keyed on `user:`, and the payload is the ISO deadline plus whole hours remaining, repeated in the reason column, never the password), `integrity_drift`, `update_available`, `backup_failed`, `store_privilege_warning` (the store privilege preflight found the store principal over-granted or could not read it -- BACKLOG #305; the payload is the finding, a count and the preflight's summary line, which names the login, the database and role names, never a password or message content), `leadership_acquired`, `dr_activated`, `log_write_failed` (an application-log sink was rolled after a write failure, or is UNWRITABLE and this process's connections were stopped -- BACKLOG #122, ADR 0162; the payload is the sink LABEL, the stage, a `safe_exc` reason and a count of connections stopped, never the record whose write failed), `log_forward_failed` (the off-box log forwarder is absent, losing records or not sending -- BACKLOG #2612; keyed on `forwarder:`, with `@node:` or `@shard:` appended when several engine processes share the store, and the payload is that key, fixed reason words and a count, never a record, the collector's address or an error text; an unpinned cluster node's `node_id` holds this engine's own host name and process id, so the key carries them), `gcm_invocations` (the per-key AES-GCM invocation bound crossing its 2^31 soft warn — ASVS 11.3.4; its payload carries a one-way `key_id` fingerprint plus counters, never key bytes), `approval_stale_requester` (a dual-control release refused because the requester no longer holds the authority the operation needs -- ASVS 8.3.2, BACKLOG #289; keyed on the approval id, and the payload is that id, the operation key and a closed-set reason slug, never the requester's name or the captured params), `approval_too_early` (a dual-control release refused because it arrived before `[approvals].min_dwell_seconds` -- ASVS 2.4.2, BACKLOG #287; keyed on `approval:`, and the payload is that key, the operation key and a fixed reason string, never the approver's or requester's name or the captured params), `audit_write_failed` (the approval gate could not write one of its audit rows -- vault BACKLOG #2255; keyed on `approval:`, or on `config_reload:inline` when an ungated config reload could not write its `config_reload_attempted` row -- vault BACKLOG #2254; the payload is that key and the lost row's audit action name, never the row's detail, a username or the captured params; the ERROR log line beside it holds the detail on some paths and only the exception on others), `ad_reconcile_aborted` (a directory reconciler pass left signed-in accounts unrevoked and needs an operator: the mass-revoke breaker tripped and nothing was revoked, or one or more probes were answered with an LDAP referral and those accounts were left unjudged while the rest of the pass was applied, BACKLOG #2538, or one or more probes raised a fault the engine could not read and those accounts were left unjudged the same way, BACKLOG #3100; the payload is a fixed source label, `directory-reconciler` for the breaker, `directory-reconciler-referral` for a referral and `directory-reconciler-probe-fault` for a probe fault so each is its own instance, the closed-set reason slug `mass_revoke_breaker`, `directory_referral` or `probe_fault`, a count (the accounts the breaker judged; for a probe fault the same total of candidates the pass took up, asked or not, faulted ones included; or for a referral the lookups the pass sent, which leaves out an id-less row -- vault BACKLOG #3170) and the latched operator explanation, never a username), `ad_reconcile_held` (the reconciler is holding accounts whose `userAccountControl` it cannot read, ADR 0195; the payload is the same fixed source label, a closed-set reason slug, the count of undetermined accounts and the latched operator explanation, never a username), `ad_session_revoked` (the reconciler revoked a directory principal's sessions; keyed on that operator account's username, with at least a `directory_absent`, `directory_disabled`, `directory_undetermined`, `directory_object_id_missing`, `roles_changed` or `scope_changed` reason), `approval_approver_provenance` (a dual-control release, or the resolution of an interrupted one, went ahead with an approver or resolver account created, re-passworded or TOTP-enrolled after the request -- BACKLOG #315, #2125; keyed on `approval:`, or `approval::resolve` for a resolution, and the payload is the operation key and the closed-set change slugs, never a username or the captured params), `administrator_granted` (the built-in Administrator role was granted by account creation, a role change, a directory group newly mapped to it -- BACKLOG #315 --, a directory sign-in whose role sync newly gave the account the role -- vault BACKLOG #2610 -- or a directory reconcile pass whose role re-diff did -- vault BACKLOG #3141; keyed on `user:` or `ad-group:`, with the grant route and the granting administrator's username, or `` for a directory sign-in or reconcile pass), `no_enabled_administrator` (a directory sign-in's role sync or the session reconciler took Administrator from its last enabled holder -- vault BACKLOG #3229; keyed on `user:`, and the payload is `via`, `ad-sync` or `ad-reconcile`, and a fixed reason sentence, never message content), `intake_paused` (the engine paused intake because the staged backlog went over `[inbound].max_staged_depth` or the SQLite volume fell below `[retention].min_free_disk_mb` -- BACKLOG #290; keyed on `intake:staged_depth` or `intake:disk_floor`, with `@node:` or `@shard:` appended when several engine processes share the store (BACKLOG #2272), and the payload is the reason, a count or a MiB figure, its limit and the store backend's name, never message content), `config_changed` (a start loaded a config whose fingerprint differs from the store's baseline -- vault BACKLOG #2597; keyed on `config:`, and the payload is the two digests, the node and engine shard labels, and the action, actor username and time of the row the older digest came from, never the config directory, a git commit or message content), `crl_reload_refused` (the CRL reload refused a replaced CRL file, so running TLS contexts keep the copy they hold -- vault BACKLOG #2904; keyed on `crl_reload:`, with `@node:` or `@shard:` appended when several engine processes share the store, and the payload is that key, the file's path, a short class token for the remedy, a count of contexts, the engine's own fixed remedy sentence and, when a start would refuse the file too, a `start_refuses` flag, never the refusal's reason, which quotes CRL issuer names and dates; that reason is in the application log, and in the `detail` of the CRL expiry alert once the held copy nears its `nextUpdate`), `remote_debug_refused` (the engine's remote-debugging guard refused scripts that another process injected into the engine process -- vault BACKLOG #2742; keyed on `remote_debug:refused`, with `@node:` or `@shard:` appended when several engine processes share the store, and the payload is that key, the count of scripts refused since the alert was last raised, the count since the process started and a fixed sentence built from the two, never a script's file name, which the injecting process chooses; that name is in the application log, in the guard's WARNING line), and the six security signals of the audit-stream rule layer -- vault BACKLOG #2613: `signin_failure_burst` (keyed on `signin:`, never the typed username), `access_denied_burst`, `body_view_burst` and `bulk_export` (each keyed on `account:`), `log_level_debug` (keyed on `logging:debug`) and `posture_loosened` (keyed on `posture:start`); the payload is a count and a fixed sentence holding a level name, switch names or a username, never a message body, a message id or an audit row's detail. The reachable **inverse** signals — `connection_restored`, `leadership_lost`, `dr_released`, `store_privilege_clean`, `intake_resumed`, `crl_reload_applied` (it resolves `crl_reload_refused` for the same file on the same engine process, once the reload's record no longer holds the refusal, and when the reload stops -- vault BACKLOG #2904), `ad_reconcile_breaker_cleared`, `ad_reconcile_hold_released` (the last two resolve `ad_reconcile_aborted`, under the breaker's or the referral's source but never the probe fault's, which has no inverse, and `ad_reconcile_held`; when they do, and where they never do, is stated once in [CONFIGURATION.md](CONFIGURATION.md#when-the-reconcilers-two-alerts-resolve-themselves) -- BACKLOG #2136) — are never rows here: `_record_state` routes an inverse through `_AUTO_RESOLVE` to `resolve_alert_instances_for`, never to `upsert_alert_instance`. (One more mapped key, `connection_started`, is emitted by no code path today.) | rows: `event_type`, `connection`, `severity`, `status`, `first_seen`, `last_seen`, `count`, `reason`, `acked_by`, `acked_at`, `resolved_at`, `suspended_until`, `escalation_tier` | the store database, **all three backends** | the operator alert list — acknowledge / resolve / suspend. Durable state is recorded **before** any suppression or throttle return, so a muted alert still leaves a record | `GET /alerts/active` under **`monitoring:diagnose`** (**not** a PHI permission) with the same per-channel scope, and `reason` alone gated on `messages:view_summary` and masked as `****` until the audited per-alert `reveal=` act (BACKLOG #2443; the `alert_instance.reason` row of §3's PL-2 block says how); ack/resolve/suspend/**resume** are POSTs on the same tier, and the separate read-only `GET /alerts/rules` view sits on its own gate | shares the connection-event window; **only RESOLVED instances are DELETEd**, by `resolved_at` — an open or acknowledged condition is never aged out from under an operator | **`reason` is free text** taken from the event's `detail`/`reason`: `safe_text(reason, limit=200)` (the bound §3's PL-2 block states) then cipher-encrypted (AAD `("alert_instance","reason",event_type,connection)` — the de-dup grain, so one AAD covers both the INSERT and the re-fire UPDATE). | +| **8. `alert_instance` table — default on wherever an `[alerts]` notifier exists** | resolvable operator alerts: `connection_stopped`, `queue_buildup`, `lane_stuck`, `message_stall`, `saturation`, `connection_error`, `storage_threshold`, `cert_expiry`, `secret_rotation`, `initial_credential_expiring` (an UNCLAIMED admin-issued temporary password nearing the instant the login gate stops accepting it -- ASVS 6.4.5, BACKLOG #1141; keyed on `user:`, and the payload is the ISO deadline plus whole hours remaining, repeated in the reason column, never the password), `integrity_drift`, `update_available`, `backup_failed`, `store_privilege_warning` (the store privilege preflight found the store principal over-granted or could not read it -- BACKLOG #305; the payload is the finding, a count and the preflight's summary line, which names the login, the database and role names, never a password or message content), `leadership_acquired`, `dr_activated`, `log_write_failed` (an application-log sink was rolled after a write failure, or is UNWRITABLE and this process's connections were stopped -- BACKLOG #122, ADR 0162; the payload is the sink LABEL, the stage, a `safe_exc` reason and a count of connections stopped, never the record whose write failed), `log_forward_failed` (the off-box log forwarder is absent, losing records or not sending -- BACKLOG #2612; keyed on `forwarder:`, with `@node:` or `@shard:` appended when several engine processes share the store, and the payload is that key, fixed reason words and a count, never a record, the collector's address or an error text; an unpinned cluster node's `node_id` holds this engine's own host name and process id, so the key carries them), `gcm_invocations` (the per-key AES-GCM invocation bound crossing its 2^31 soft warn — ASVS 11.3.4; its payload carries a one-way `key_id` fingerprint plus counters, never key bytes), `approval_stale_requester` (a dual-control release refused because the requester no longer holds the authority the operation needs -- ASVS 8.3.2, BACKLOG #289; keyed on the approval id, and the payload is that id, the operation key and a closed-set reason slug, never the requester's name or the captured params), `approval_too_early` (a dual-control release refused because it arrived before `[approvals].min_dwell_seconds` -- ASVS 2.4.2, BACKLOG #287; keyed on `approval:`, and the payload is that key, the operation key and a fixed reason string, never the approver's or requester's name or the captured params), `audit_write_failed` (the approval gate could not write one of its audit rows -- vault BACKLOG #2255; keyed on `approval:`, or on `config_reload:inline` when an ungated config reload could not write its `config_reload_attempted` row -- vault BACKLOG #2254; the payload is that key and the lost row's audit action name, never the row's detail, a username or the captured params; the ERROR log line beside it holds the detail on some paths and only the exception on others), `ad_reconcile_aborted` (a directory reconciler pass left signed-in accounts unrevoked and needs an operator: the mass-revoke breaker tripped and nothing was revoked, or one or more probes were answered with an LDAP referral and those accounts were left unjudged while the rest of the pass was applied, BACKLOG #2538, or one or more probes raised a fault the engine could not read and those accounts were left unjudged the same way, BACKLOG #3100, or the directory refused one or more probes with a result code that points at the configuration, such as a deleted search base, and those accounts were left unjudged the same way, vault BACKLOG #3010; the payload is a fixed source label, `directory-reconciler` for the breaker, `directory-reconciler-referral` for a referral, `directory-reconciler-probe-fault` for a probe fault and `directory-reconciler-misconfigured` for a configuration refusal so each is its own instance, the closed-set reason slug `mass_revoke_breaker`, `directory_referral`, `probe_fault` or `directory_misconfigured`, a count (the accounts the breaker judged, or for each of the other three the lookups the pass sent, which leaves out an id-less row -- vault BACKLOG #3170) and the latched operator explanation, never a username), `ad_reconcile_held` (the reconciler is holding accounts whose `userAccountControl` it cannot read, ADR 0195; the payload is the same fixed source label, a closed-set reason slug, the count of undetermined accounts and the latched operator explanation, never a username), `ad_session_revoked` (the reconciler revoked a directory principal's sessions; keyed on that operator account's username, with at least a `directory_absent`, `directory_disabled`, `directory_undetermined`, `directory_object_id_missing`, `roles_changed` or `scope_changed` reason), `approval_approver_provenance` (a dual-control release, or the resolution of an interrupted one, went ahead with an approver or resolver account created, re-passworded or TOTP-enrolled after the request -- BACKLOG #315, #2125; keyed on `approval:`, or `approval::resolve` for a resolution, and the payload is the operation key and the closed-set change slugs, never a username or the captured params), `administrator_granted` (the built-in Administrator role was granted by account creation, a role change, a directory group newly mapped to it -- BACKLOG #315 --, a directory sign-in whose role sync newly gave the account the role -- vault BACKLOG #2610 -- or a directory reconcile pass whose role re-diff did -- vault BACKLOG #3141; keyed on `user:` or `ad-group:`, with the grant route and the granting administrator's username, or `` for a directory sign-in or reconcile pass), `no_enabled_administrator` (a directory sign-in's role sync or the session reconciler took Administrator from its last enabled holder -- vault BACKLOG #3229; keyed on `user:`, and the payload is `via`, `ad-sync` or `ad-reconcile`, and a fixed reason sentence, never message content), `intake_paused` (the engine paused intake because the staged backlog went over `[inbound].max_staged_depth` or the SQLite volume fell below `[retention].min_free_disk_mb` -- BACKLOG #290; keyed on `intake:staged_depth` or `intake:disk_floor`, with `@node:` or `@shard:` appended when several engine processes share the store (BACKLOG #2272), and the payload is the reason, a count or a MiB figure, its limit and the store backend's name, never message content), `config_changed` (a start loaded a config whose fingerprint differs from the store's baseline -- vault BACKLOG #2597; keyed on `config:`, and the payload is the two digests, the node and engine shard labels, and the action, actor username and time of the row the older digest came from, never the config directory, a git commit or message content), `crl_reload_refused` (the CRL reload refused a replaced CRL file, so running TLS contexts keep the copy they hold -- vault BACKLOG #2904; keyed on `crl_reload:`, with `@node:` or `@shard:` appended when several engine processes share the store, and the payload is that key, the file's path, a short class token for the remedy, a count of contexts, the engine's own fixed remedy sentence and, when a start would refuse the file too, a `start_refuses` flag, never the refusal's reason, which quotes CRL issuer names and dates; that reason is in the application log, and in the `detail` of the CRL expiry alert once the held copy nears its `nextUpdate`), `remote_debug_refused` (the engine's remote-debugging guard refused scripts that another process injected into the engine process -- vault BACKLOG #2742; keyed on `remote_debug:refused`, with `@node:` or `@shard:` appended when several engine processes share the store, and the payload is that key, the count of scripts refused since the alert was last raised, the count since the process started and a fixed sentence built from the two, never a script's file name, which the injecting process chooses; that name is in the application log, in the guard's WARNING line), and the six security signals of the audit-stream rule layer -- vault BACKLOG #2613: `signin_failure_burst` (keyed on `signin:`, never the typed username), `access_denied_burst`, `body_view_burst` and `bulk_export` (each keyed on `account:`), `log_level_debug` (keyed on `logging:debug`) and `posture_loosened` (keyed on `posture:start`); the payload is a count and a fixed sentence holding a level name, switch names or a username, never a message body, a message id or an audit row's detail. The reachable **inverse** signals — `connection_restored`, `leadership_lost`, `dr_released`, `store_privilege_clean`, `intake_resumed`, `crl_reload_applied` (it resolves `crl_reload_refused` for the same file on the same engine process, once the reload's record no longer holds the refusal, and when the reload stops -- vault BACKLOG #2904), `ad_reconcile_breaker_cleared`, `ad_reconcile_hold_released` (the last two resolve `ad_reconcile_aborted`, under the breaker's or the referral's source but never the probe fault's or the configuration refusal's, which have no inverse, and `ad_reconcile_held`; when they do, and where they never do, is stated once in [CONFIGURATION.md](CONFIGURATION.md#when-the-reconcilers-two-alerts-resolve-themselves) -- BACKLOG #2136) — are never rows here: `_record_state` routes an inverse through `_AUTO_RESOLVE` to `resolve_alert_instances_for`, never to `upsert_alert_instance`. (One more mapped key, `connection_started`, is emitted by no code path today.) | rows: `event_type`, `connection`, `severity`, `status`, `first_seen`, `last_seen`, `count`, `reason`, `acked_by`, `acked_at`, `resolved_at`, `suspended_until`, `escalation_tier` | the store database, **all three backends** | the operator alert list — acknowledge / resolve / suspend. Durable state is recorded **before** any suppression or throttle return, so a muted alert still leaves a record | `GET /alerts/active` under **`monitoring:diagnose`** (**not** a PHI permission) with the same per-channel scope, and `reason` alone gated on `messages:view_summary` and masked as `****` until the audited per-alert `reveal=` act (BACKLOG #2443; the `alert_instance.reason` row of §3's PL-2 block says how); ack/resolve/suspend/**resume** are POSTs on the same tier, and the separate read-only `GET /alerts/rules` view sits on its own gate | shares the connection-event window; **only RESOLVED instances are DELETEd**, by `resolved_at` — an open or acknowledged condition is never aged out from under an operator | **`reason` is free text** taken from the event's `detail`/`reason`: `safe_text(reason, limit=200)` (the bound §3's PL-2 block states) then cipher-encrypted (AAD `("alert_instance","reason",event_type,connection)` — the de-dup grain, so one AAD covers both the INSERT and the re-fire UPDATE). | | **9. `response` rows with `kind='ack_sent'` — DEFAULT ON** (`[diagnostics].response_sent = true`) | the ACK/NAK the engine returned to an inbound sender, under a sentinel destination `\x1fack:` | rows: `ack_code` (`AA`/`AE`/`AR`/`CA`/`CE`/`CR`), `ack_phase` (`decode`/`parse`/`strict`/`ingest`), `outcome`, `body`, `detail` | the store database | "what did we actually reply, and why" — the operator's answer to a sender disputing an ACK | `GET /messages/{id}/responses` under `messages:read` + `require_phi_read`; the `body` only for a caller who also holds `messages:view_raw` and `messages:view_summary`; every read writes a `response.read` audit row | `body`, `detail` and `resp_headers` are set to `NULL` in place by `purge_message_bodies` on the message-body window, on all three backends | **PHI fail-safe:** the ACK **body** is stored **only when the store cipher is active** — on a keyless store it is `NULL` rather than plaintext — and every NAK passes no body at all, so the offending field value is never persisted. The disposition metadata (`ack_code`/`ack_phase`/`outcome`) is non-PHI and always captured; `detail` is `safe_text`-scrubbed, held to the 200-character bound §3's PL-2 block states, and encrypted | | **10. `[alerts]` webhook transport** (off by default — `webhook_url` unset) | one HTTPS POST per alert, carrying every non-underscore event key as JSON | JSON | the operator's webhook endpoint (Slack/Teams/PagerDuty/custom) | operator notification | **`https` only** — a plaintext `http://` webhook URL is refused at construction unless the `MEFOR_ALLOW_INSECURE_TLS` escape is set (and then a warning is logged); since #329 this path routes that escape through the clamped `weakened_tls_escape_permitted(posture)` (the instance posture threaded from the API lifespan), so on an enforcing-PHI instance the escape is inert and a cleartext webhook POST stays refused — the same clamp as the connectors, no longer the raw escape. Redirects are refused; an optional `webhook_allowed_hosts` egress allowlist gates the host | the endpoint's | **carries the alert's `detail`/`reason` free text** (`safe_exc()`-scrubbed at the emit sites, but **not** re-run through `safe_text` on this path). Internal `_`-prefixed keys (per-rule recipients, rule id, cooldown) are stripped before send, so recipient addresses never cross the wire | | **11. `[alerts]` SMTP transport — operator alert list** (off unless `email_smtp_host` + `email_from` + ≥1 `email_to`) | one email per alert; default subject `[MessageFoundry] — `, default body every non-underscore event key as `k: v` | plain text (always kept — never HTML-only); optional HTML alternative | the operators' mailboxes | operator notification | `smtp_allowed_hosts` egress allowlist; the SMTP password comes from `MEFOR_ALERTS_EMAIL_PASSWORD` or a `[secrets]` provider, never the config file; per-send timeout `email_timeout` | the mail system's | carries the same `detail`/`reason` free text as the webhook. #138 operator templates are constrained to a **closed non-PHI variable allowlist** validated fail-closed at config load. **Transport posture:** `send_plain_email` builds an explicit **verifying** context (chain + hostname + strict RFC 5280, TLS 1.2 floor) via `tls_policy.build_smtp_tls_context()` and passes it to `starttls()`, anchored to the OS roots, `[alerts].email_tls_ca_file`, or `[tls].internal_ca_file` — the same factory the EMAIL and DIRECT *message destinations* use, so all three SMTP cells now share one policy ([#323](BACKLOG.md), closed 2026-08-02). Before that this call passed **no** context and Python's stdlib default applied (`ssl._create_stdlib_context` **is** `ssl._create_unverified_context` — `CERT_NONE`, `check_hostname = False`), leaving the hop encrypted but unauthenticated. There is still **no hop gradient or attestation on this path** — unlike the connectors, this cell is constructed outside the `active_hop_posture` scope, so its deviations (`email_use_tls = false`, or `email_tls_verify = false`) are gated by a `[security].allow_unverified_alert_smtp_tls` **acknowledgment switch at the serve gate** rather than by the clamped escape: on an enforcing PHI instance `serve` refuses to start without it, and permits + `AUDIT`-logs the start with it. Both deviations are named by `security_loosenings()` and reported by `messagefoundry check`'s `alert-smtp-tls` advisory | diff --git a/docs/SECURITY.md b/docs/SECURITY.md index cc5008b5ea..782a370922 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -4143,10 +4143,11 @@ writes no roles, and it strikes and revokes as an absent account does, with the `directory_object_id_missing`. It is not a `userAccountControl` answer, so it never moves the ADR 0195 hold. It is never asked, so it is no part of an outage either: a pass whose every lookup fails still audits `auth.ad_reconcile_skipped`, and that row's `asked` count leaves it out. Such a pass, -and one that aborts because its lookups were only referred, faulted or failed, still strikes the -row, because its answer never came from the directory (BACKLOG #3170). No such pass revokes it: only a pass that is not aborted can, once -the row's strikes reach the threshold and the row is in that pass's probe budget. A referral's `auth.ad_reconcile_referred` row and alert count only -the lookups sent too. Its revocation does count toward the mass-revoke breaker below. Such a +and one that aborts because its lookups were only referred, refused for the configuration, faulted +or failed, still strikes the row, because its answer never came from the directory (BACKLOG #3170). +No such pass revokes it: only a pass that is not aborted can, once the row's strikes reach the +threshold and the row is in that pass's probe budget. The audit rows and alerts of a referral, a +configuration refusal and a probe fault count only the lookups sent too. Its revocation does count toward the mass-revoke breaker below. Such a row cannot take a federated binding: the bind refuses it, so every binding the bind has made since BACKLOG #1143 slice C sits on a row probed by its id (ADR 0184 AC-5). A binding already on an id-less row, made before that refusal, is **never probed by name** (BACKLOG #2027). The pass skips the row and @@ -4158,7 +4159,8 @@ disable or demotion no longer ends that row's sessions within one interval, only Removing the binding (`DELETE /users/{user_id}/federated-identity`) returns the row to the pass. The pass has these safety properties, because the lookup still returns one indistinguishable "not -found" for *deleted*, *moved out of the search base* and *the search base was never right*: +found" for *deleted*, *moved out of the search base* and *a search base that exists but was never +the right one*: - **Fail-OPEN.** An unreachable domain controller revokes **nothing**, and an account the pass asked about accrues no strike. An id-less row, never asked, is still struck (BACKLOG #3170). @@ -4184,8 +4186,10 @@ found" for *deleted*, *moved out of the search base* and *the search base was ne and no floor. **The cost:** two genuinely disabled accounts whose attribute the bind account cannot read keep their sessions to the absolute cap. -- **A mass-revoke circuit breaker.** A misconfigured `ad_user_search_base`, an OU reorganisation, or a - service account that lost read rights answers "not found" for *every* user. A pass whose revocation +- **A mass-revoke circuit breaker.** An `ad_user_search_base` that names the wrong OU, or an OU + reorganisation that moves the users out of it, answers "not found" for *every* user. A base that + was deleted or moved, or that the service account may not read, does not: the search ends with a + result code, which is the configuration refusal described below (BACKLOG #3010). A pass whose revocation set exceeds **both** `ad_session_revoke_max` (5) **and** `ad_session_revoke_max_fraction` (0.34) of the probed population **aborts**: nothing is revoked, nothing is written, the engine logs at ERROR and audits `auth.ad_reconcile_aborted`, raises the `ad_reconcile_aborted` alert, and the condition @@ -4231,8 +4235,9 @@ directory outage (BACKLOG #3100). Each such probe is logged at WARNING by name, estate is still reconciled. A pass with any fault logs one ERROR naming no account and audits `auth.ad_reconcile_faulted`. It also raises the `ad_reconcile_aborted` alert with reason `probe_fault`, under its own source `directory-reconciler-probe-fault`. If every probe faults or -fails, the pass judges no account it asked about and revokes nothing, like an outage, but it still -pages, and it still strikes an id-less row (BACKLOG #3170). A +fails, at least one a fault, the pass judges no account it asked about and revokes nothing, like an +outage, but it still pages, and it still strikes an id-less row (BACKLOG #3170). A pass where every +probe fails and none faults is a plain outage, and pages nobody. A connection, timeout or name-lookup error that escapes the directory layer is still read as unreachable. **The cost:** while a fault repeats, that account's sessions are never revoked by the reconciler, so they run to the absolute cap. That cap is the only bound; the alert makes the gap @@ -4240,6 +4245,21 @@ visible and limits nothing. A pass with a fault marks no alert clear. When it re [CONFIGURATION.md](CONFIGURATION.md#when-the-reconcilers-two-alerts-resolve-themselves). Step-up refuses the account with outcome `probe_fault`. +A search the directory refuses with noSuchObject, invalidDNSyntax or insufficientAccessRights is a +configuration refusal (BACKLOG #3010). A search base that was deleted or moved, or that the service +account may not read, answers so for every account on every pass. Before #3010 the engine read that +as "not found", which struck every account and tripped the breaker on a large enough estate. Nor is +it read as an outage, which would page nobody. The refused account is left unjudged: no strike, no +revocation. A pass with any such refusal logs one ERROR naming the code and no account, audits +`auth.ad_reconcile_misconfigured`, and raises the `ad_reconcile_aborted` alert with reason +`directory_misconfigured`, under its own source `directory-reconciler-misconfigured`. If every +probe is refused, faulted or fails, at least one refused and none referred, the pass aborts as +`directory_misconfigured`: it judges no account it asked about and revokes nothing, but it pages. +Like the fault's, that alert has no inverse; an operator resolves it. **The cost:** while the +refusal lasts, the refused accounts' sessions are never revoked by the reconciler, so they run to +the absolute cap. Other result codes that are not an answer, such as busy or unavailable, are an +outage. Step-up refuses the account with outcome `misconfigured`. + ### The engine session is independent of the IdP session (ASVS 7.6.1) The engine mints its own session at sign-in. That session does not follow the identity provider From e109956b05183746e6b1ac9414cb62750e65ee2b Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Sat, 17 Oct 2026 01:39:25 -0500 Subject: [PATCH 31/57] Review round one on D3b: quiet service release, context-class bind referral (BACKLOG #3170) BACKLOG #3010 follow-ups (subject names only the claim held here): - the service connection release swallows an unbind error after an answered lookup, as the ldap3 with block it replaced did; - a bind referral needs a context-class tag 3; a universal tag 3 stays undecodable; - the referral message names [auth].ad_bind_dn; a configuration code no longer says the directory is not answering; - each unjudged-kind message counts the other kinds; - stale comments in alerts.py, alert_sinks.py and the cert outage docstring; SECURITY.md keeps lost read rights on the entries among the "not found" cases. --- docs/SECURITY.md | 15 ++++++---- messagefoundry/auth/ldap.py | 33 +++++++++++++++++----- messagefoundry/auth/reconcile.py | 7 +++-- messagefoundry/auth/service.py | 16 ++++++----- messagefoundry/pipeline/alert_sinks.py | 4 +-- messagefoundry/pipeline/alerts.py | 4 +-- tests/test_ldap_decode_boundary.py | 39 +++++++++++++++++++++++++- 7 files changed, 90 insertions(+), 28 deletions(-) diff --git a/docs/SECURITY.md b/docs/SECURITY.md index 782a370922..4135d92f21 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -4186,10 +4186,12 @@ the right one*: and no floor. **The cost:** two genuinely disabled accounts whose attribute the bind account cannot read keep their sessions to the absolute cap. -- **A mass-revoke circuit breaker.** An `ad_user_search_base` that names the wrong OU, or an OU - reorganisation that moves the users out of it, answers "not found" for *every* user. A base that - was deleted or moved, or that the service account may not read, does not: the search ends with a - result code, which is the configuration refusal described below (BACKLOG #3010). A pass whose revocation +- **A mass-revoke circuit breaker.** An `ad_user_search_base` that names the wrong OU, an OU + reorganisation that moves the users out of it, or a service account that lost read rights on the + user entries while it can still read the base, answers "not found" for *every* user. A base that + was deleted or moved, or that the service account may not read at all, usually does not: the + search ends with a result code, which is the configuration refusal described below (BACKLOG + #3010). Which of the two a directory answers is not measured against a domain controller. A pass whose revocation set exceeds **both** `ad_session_revoke_max` (5) **and** `ad_session_revoke_max_fraction` (0.34) of the probed population **aborts**: nothing is revoked, nothing is written, the engine logs at ERROR and audits `auth.ad_reconcile_aborted`, raises the `ad_reconcile_aborted` alert, and the condition @@ -4214,8 +4216,9 @@ revoked. A pass with any referral audits `auth.ad_reconcile_referred` and raises `directory-reconciler-referral`. So it is its own instance, apart from the breaker's, and acknowledging or suspending one does not silence the other. An `[alerts]` rule keyed on the event type alone matches both, and so does a source glob such as `directory-reconciler*`. To mute only -the referral, a rule names its source exactly. The alert's text names `ad_user_search_base` and -`ad_group_search_base`. When it resolves itself is stated once, in +the referral, a rule names its source exactly. The alert's text names `ad_user_search_base`, +`ad_group_search_base` and `ad_bind_dn`: a bind answered with a referral is one too (BACKLOG +#3010), and its hosts cannot be named. When it resolves itself is stated once, in [CONFIGURATION.md](CONFIGURATION.md#when-the-reconcilers-two-alerts-resolve-themselves). Referred accounts stay in the mass-revoke breaker's count, as unreachable ones do. Left out, a referring group base would trip the breaker on any handful of genuine disables. A base in another domain of the forest diff --git a/messagefoundry/auth/ldap.py b/messagefoundry/auth/ldap.py index 920b5bff45..ee57207352 100644 --- a/messagefoundry/auth/ldap.py +++ b/messagefoundry/auth/ldap.py @@ -872,9 +872,16 @@ def _unanswered(operation: str, code: int | None) -> LdapUnansweredError: else: name = RESULT_CODES.get(code, "an unknown result code") shown, hint = f"result code {code} ({name})", _UNANSWERED_HINTS.get(code, "") + # A configuration code is not read as the directory not answering: the session reconciler + # reads it as a configuration refusal, which alerts (LdapUnansweredError.configuration). + reading = ( + "a fault in the configuration" + if code in _CONFIGURATION_CODES + else "the directory not answering" + ) message = ( f"AD ended the {operation} with {shown}, which is not an answer; the engine reads it as " - f"the directory not answering (BACKLOG #3010).{hint}" + f"{reading} (BACKLOG #3010).{hint}" ) # Keyed on ldap3's name, not the raw code: a directory chooses the code, and ldap3's table is # finite, so the set of warnings stays bounded. @@ -1006,6 +1013,8 @@ def _ldap3_call(operation: str, *, close: bool = False) -> Iterator[None]: #: The context tag of the ``referral`` field of an LDAPResult (RFC 4511 section 4.1.9). _REFERRAL_TAG = 3 +#: ldap3's number for the context-specific tag class (``utils/asn1.py``, ``CLASSES``). +_CONTEXT_CLASS = 2 def _is_bind_referral(exc: BaseException) -> bool: @@ -1017,7 +1026,9 @@ def _is_bind_referral(exc: BaseException) -> bool: ``KeyError(3)`` from anywhere else, such as a stray tag [3] in some other response or engine code, is not this. A test pins the ldap3 names this reads. """ - if not isinstance(exc, KeyError): + # The context-class tag alone: a universal, application or private tag 3 raises KeyError((class, + # 3)) from the class table instead, and is a malformed response, not a referral. + if not isinstance(exc, KeyError) or exc.args != (_REFERRAL_TAG,): return False from ldap3.utils import asn1 # lazy, like every ldap3 import here @@ -1031,6 +1042,7 @@ def _is_bind_referral(exc: BaseException) -> bool: frame.f_globals.get("__name__") == asn1.__name__ and frame.f_code.co_name == "decode_sequence" and frame.f_locals.get("context_decoders") is asn1.BIND_RESPONSE_CONTEXT + and frame.f_locals.get("ber_class") == _CONTEXT_CLASS and frame.f_locals.get("ber_type") == _REFERRAL_TAG ) @@ -1220,14 +1232,16 @@ def _socket_faults_as_ldap_error() -> Iterator[None]: @contextlib.contextmanager -def _released(conn: Any) -> Iterator[None]: +def _released(conn: Any, *, quiet: bool = False) -> Iterator[None]: """Unbind ``conn`` when the block ends, however it ends. An ldap3 error from the unbind never replaces an exception the block raised (BACKLOG #3010). The decoy bind swallows an ``LDAPException`` and the real bind maps it to :class:`LdapError`, so an unbind failing after an undecodable bind would otherwise answer an absent account and a present one differently. After a block that raised nothing, the unbind's error propagates as - before. + before, unless ``quiet``: the service connection passes it, because its lookup has already been + answered, and ldap3's own ``with``, which it replaced, swallowed an unbind error too. Its type + is logged at debug level then. """ import ldap3 @@ -1237,7 +1251,12 @@ def _released(conn: Any) -> Iterator[None]: with contextlib.suppress(ldap3.core.exceptions.LDAPException): conn.unbind() raise - conn.unbind() + try: + conn.unbind() + except ldap3.core.exceptions.LDAPException as failed: + if not quiet: + raise + logger.debug("AD unbind after an answered lookup failed: %s", _type_name(failed)) class LdapAuthenticator: @@ -1697,7 +1716,7 @@ def authenticate( # Released explicitly (BACKLOG #3010): ldap3's own ``with`` never unbinds a connection # that was bound on entry, so the session stayed open until it was collected. svc = self._service_conn() - with _released(svc): + with _released(svc, quiet=True): found = ( self._lookup_by_object_id(svc, object_id, fallback_username=username) if object_id is not None @@ -1813,7 +1832,7 @@ def probe_principal( raise ValueError("pass object_id or tie_break_object_id, not both") try: svc = self._service_conn() - with _released(svc): # not ldap3's own ``with``: see ``authenticate`` + with _released(svc, quiet=True): # not ldap3's own ``with``: see ``authenticate`` found = ( self._lookup_by_object_id(svc, object_id, fallback_username=username) if object_id is not None diff --git a/messagefoundry/auth/reconcile.py b/messagefoundry/auth/reconcile.py index c310e2e824..f121262e08 100644 --- a/messagefoundry/auth/reconcile.py +++ b/messagefoundry/auth/reconcile.py @@ -24,9 +24,10 @@ at once. :func:`hold_engaged` holds those accounts, and only those: none of them is revoked, and the rest of the estate is reconciled as usual. The mass-revoke breaker below cannot do this on a small estate, because its absolute floor lets five or fewer revocations through. -4. **A mass-revoke circuit breaker.** Users moved out of the search base, or a base that exists - but is the wrong one, return "not found" for **every** user, which looks the same as "everyone - was deleted". :func:`breaker_tripped` aborts such a pass wholesale rather than signing out the +4. **A mass-revoke circuit breaker.** Users moved out of the search base, a base that exists but + is the wrong one, or a service account that lost read rights on the user entries but not on + the base, return "not found" for **every** user, which looks the same as "everyone was + deleted". :func:`breaker_tripped` aborts such a pass wholesale rather than signing out the estate. This is why the pass is planned in full before anything is written: an abort must leave the store byte-identical, including the role re-diff. A base that was deleted or moved, or that the service account may not read, is not "not found": the search ends with a result code, which diff --git a/messagefoundry/auth/service.py b/messagefoundry/auth/service.py index a815af8992..9e7c806be5 100644 --- a/messagefoundry/auth/service.py +++ b/messagefoundry/auth/service.py @@ -7395,8 +7395,8 @@ async def _record_reconcile_referral( ) others = self._unjudged_others(plan, others, faults=True, refusals=True) self._reconcile_referral_alert = ( - "LDAP referral: check [auth].ad_user_search_base and, for nested groups, " - "[auth].ad_group_search_base; one names a base in another domain of the forest. " + "LDAP referral: check [auth].ad_user_search_base, [auth].ad_group_search_base for " + "nested groups, and [auth].ad_bind_dn; one names another domain of the forest. " f"{len(plan.referred)} of {plan.asked} signed-in directory account(s) probed were " f"referred and left unjudged. {others} A referred account is never revoked while it " "is referred, because the engine does not follow referrals. Use a base in the bound " @@ -7458,7 +7458,7 @@ async def _record_reconcile_fault(self, plan: reconcile.ReconcilePlan) -> None: if plan.aborted is not None else "Accounts that answered were judged as usual." ) - others = self._unjudged_others(plan, others) + others = self._unjudged_others(plan, others, refusals=True) # The pointer to the account comes first: the store keeps only the first 200 characters # of an alert instance's reason, as `_record_reconcile_referral` says. self._reconcile_fault_alert = ( @@ -7515,7 +7515,7 @@ async def _record_reconcile_misconfigured( if plan.aborted is not None else "Accounts that answered were judged as usual." ) - others = self._unjudged_others(plan, others) + others = self._unjudged_others(plan, others, faults=True) # The settings to change come first: the store keeps only the first 200 characters of an # alert instance's reason, as `_record_reconcile_referral` says. self._reconcile_misconfigured_alert = ( @@ -7552,8 +7552,9 @@ def _unjudged_others( A pass whose every answer went unjudged is recorded by its referral, refusal or fault and not as an outage, so its failures are counted here, or a near-total outage would read as a - configuration fault. ``faults`` and ``refusals`` add those counts too: the referral's - message carries both, since either is recorded under its own alert as well.""" + configuration fault. ``faults`` and ``refusals`` add those counts too, each passed by every + recorder but its own kind's, so each message names the other unjudged kinds, which are + recorded under their own alerts as well.""" if faults and plan.faulted: others = f"{len(plan.faulted)} raised a probe fault. {others}" if refusals and plan.misconfigured: @@ -8739,7 +8740,8 @@ def _note_cert_directory_answer(self, answer: reconcile.Probe | str) -> None: """Log the certificate path's directory refusals without logging one line per request. An outage is a refusal that says the directory could not be asked: unreachable, referring, - or the probe cap full. It logs one WARNING per outage and, if that + refusing the search for its configuration (``misconfigured``, BACKLOG #3010), or the probe + cap full. It logs one WARNING per outage and, if that WARNING was logged, one INFO on the next answer the directory actually gave, whatever it said about the account. WARNINGs are at least :data:`_CERT_OUTAGE_LOG_INTERVAL_SECONDS` apart, so an outage that flaps (a full cap turning over, or one referring account beside a diff --git a/messagefoundry/pipeline/alert_sinks.py b/messagefoundry/pipeline/alert_sinks.py index 00da40a03c..5cf99e2f61 100644 --- a/messagefoundry/pipeline/alert_sinks.py +++ b/messagefoundry/pipeline/alert_sinks.py @@ -1177,8 +1177,8 @@ def administrator_granted(self, name: str, *, via: str, granted_by: str) -> None def ad_reconcile_aborted(self, name: str, *, reason: str, probed: int, detail: str) -> None: # ADR 0079 mechanism 2: the mass-revoke breaker tripped and the pass applied nothing, or one - # or more probes were referred (BACKLOG #2538) or faulted (BACKLOG #3100) and left unjudged; - # `reason` tells which. The + # or more probes were referred (BACKLOG #2538), refused for the configuration (BACKLOG + # #3010) or faulted (BACKLOG #3100) and left unjudged; `reason` tells which. The # fixed source label keys the throttle, so a standing misconfiguration that trips every pass # pages once per cooldown rather than once per pass. `detail` is the latched operator explanation, # and it is what the instance's reason column shows (detail wins over reason there). diff --git a/messagefoundry/pipeline/alerts.py b/messagefoundry/pipeline/alerts.py index 2f13353d45..224f749c08 100644 --- a/messagefoundry/pipeline/alerts.py +++ b/messagefoundry/pipeline/alerts.py @@ -1097,8 +1097,8 @@ def no_enabled_administrator(self, name: str, *, via: str) -> None: def ad_reconcile_aborted(self, name: str, *, reason: str, probed: int, detail: str) -> None: log.warning( - # The count is the candidates the pass took up for the breaker and a probe fault, or a - # referral's probes asked (BACKLOG #3170); see AlertSink.ad_reconcile_aborted. + # The count is the candidates the breaker judged, or for every other source the probes + # the pass asked (BACKLOG #3170); see AlertSink.ad_reconcile_aborted. "ALERT ad_reconcile_aborted: %r left accounts unrevoked, counting %d principal(s) " "(%s): %s", name, diff --git a/tests/test_ldap_decode_boundary.py b/tests/test_ldap_decode_boundary.py index bc15b30dcb..45511436f3 100644 --- a/tests/test_ldap_decode_boundary.py +++ b/tests/test_ldap_decode_boundary.py @@ -86,6 +86,12 @@ def _result(tag: int, code: int, *, matched: bytes | None = None, extra: bytes = KeyError, id="context-tag-22", ), + # A universal tag 3 is not the referral field, so it stays undecodable (D3b review round 1). + pytest.param( + _result(_BIND_RESPONSE, RESULT_SUCCESS, extra=_tlv(0x03, b"\x00")), + KeyError, + id="universal-tag-3", + ), pytest.param(_result(_BIND_RESPONSE, 4242), KeyError, id="unknown-code-4242"), pytest.param( _result(_BIND_RESPONSE, RESULT_SUCCESS, matched=_tlv(0x02, b"\x05")), @@ -549,7 +555,9 @@ def test_the_ldap3_methods_the_origin_check_names_still_exist() -> None: assert {"decode_response", "set_response"} <= set(vars(ExtendedOperation)) assert "populate_result" in vars(WhoAmI) # What _is_bind_referral reads (D3b): the decoder's locals, and a bind table with no referral. - assert {"context_decoders", "ber_type"} <= set(asn1.decode_sequence.__code__.co_varnames) + assert {"context_decoders", "ber_class", "ber_type"} <= set( + asn1.decode_sequence.__code__.co_varnames + ) assert ldap_module._REFERRAL_TAG not in asn1.BIND_RESPONSE_CONTEXT @@ -567,6 +575,35 @@ def test_an_unbind_error_never_replaces_the_refusal() -> None: pass +def test_the_service_connection_release_never_fails_an_answered_lookup() -> None: + """D3b review round 1. ``authenticate`` and ``probe_principal`` release the service connection + with ``quiet``: the lookup was answered, and ldap3's own ``with``, which the release replaced, + swallowed an unbind error too. Without it, a reset after the last search would turn a verified + sign-in into a login error and a read answer into an outage.""" + unbind_fault = ldap3.core.exceptions.LDAPSocketSendError("synthetic") + conn = SimpleNamespace(unbind=_raise(unbind_fault)) + with ldap_module._released(conn, quiet=True): + pass + refusal = LdapUndecodableError("synthetic refusal") + with pytest.raises(LdapUndecodableError) as raised, ldap_module._released(conn, quiet=True): + raise refusal + assert raised.value is refusal + + +def test_a_probe_answered_before_the_unbind_fails_keeps_its_answer() -> None: + """The same, end to end: the unbind's send fails, and the probe still answers.""" + with _scripted_dc(_groups_answered) as (_dc, authenticator): + real = ldap3.Connection.unbind + + def failing_unbind(self: Any) -> Any: + real(self) + raise ldap3.core.exceptions.LDAPSocketSendError("synthetic") + + with pytest.MonkeyPatch.context() as mp: + mp.setattr(ldap3.Connection, "unbind", failing_unbind) + assert authenticator.probe_principal("alice").principal is not None + + def test_the_refusal_warns_once_per_operation_and_fault(caplog: pytest.LogCaptureFixture) -> None: with _scripted_dc(lambda _raw: [_done(4242)]) as (_dc, authenticator): for _ in range(3): From d8d955a5e8b0fecce9ce03a3c52f457fae20e673 Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Sat, 17 Oct 2026 01:54:23 -0500 Subject: [PATCH 32/57] D3b handover: PR title and banner text (BACKLOG #3010 #3170) No code. Carries the proposed pull request title and the ledger banner addenda for the batch 257 D3b follow-ups, so the branch says what it is. Proposed PR title: Directory reconciler: a refused search base pages, a referred bind is a referral, the service connection is unbound (BACKLOG #3010, #3170) Proposed banner addendum, BACKLOG #3010: D3b (branch b257-d3b) closes limb 5's reconciler follow-ups. noSuchObject, invalidDNSyntax and insufficientAccessRights read as a configuration refusal (ProbeOutcome.MISCONFIGURED): unjudged, never revoked; every pass with one latches a message, audits auth.ad_reconcile_misconfigured and pages ad_reconcile_aborted with reason directory_misconfigured under directory-reconciler-misconfigured; a pass whose every answer went unjudged with at least one refusal and no referral aborts as directory_misconfigured. Transient codes stay a plain outage. A bind response carrying a referral (context tag 3) is LdapReferralError, so the reconciler reads REFERRED. authenticate and probe_principal unbind the service connection. Limb 5 item C (WARNING visibility) was already met by ldap.py's own warn-once and needed no reconciler line. Open: a failed service-account bind (49, locked) still reads as an outage; CONFIGURATION.md does not yet name the new alert instance. Proposed banner addendum, BACKLOG #3170: D3b: the probe fault's alert and auth.ad_reconcile_faulted row count the probes asked (field asked), one event one count, as the referral's do; the breaker's alert keeps probed. Fault twin of the asked-count test and a service test that a pass of faults or refusals only strikes but never revokes an id-less row. SECURITY.md no longer says an all-failure pass pages; the 3170 changelog and service.py comments are corrected. From 7554d4182ca8c3cab8fa217c853e299cbd997e80 Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Sat, 17 Oct 2026 02:16:51 -0500 Subject: [PATCH 33/57] Lock notice: a full queue reads mailed false, an actor index for the throttle (BACKLOG #3120) Follow-up 2: SecurityNotifier.notify returns whether it queued the event. The shipped notifier answers False for no address and for a full queue (_enqueue now returns whether it queued), and _notify_security returns that, so the auth.lock_notice row records mailed: false for a notice the full queue dropped and the next lock of the kind mails again. A send that fails later still leaves mailed: true; kept as a residual in SECURITY.md. All 22 test notifier fakes now return bool. (d) and follow-up 5: ix_audit_actor on audit_log(actor, action, id) on SQLite, Postgres and SQL Server; one idempotent DDL line each, as 42ffdb3e did. The server _SCHEMA edits move the ADR 0064 schema hash. Follow-up 1: the drain comment names NSSM's 15 s window and every drain that shares it; their bounds add up to 17 s. approvals.py and outlive.py point there. Follow-up 3: SECURITY.md states the one-read-at-a-time limit. Follow-up 4: the lock-notice tests drain in finally. (a) cross-shard double mail, (b) failure-log timing and (c) the uncapped task set are declined in writing, residuals in SECURITY.md. --- changelog.d/3120.security.md | 11 ++ docs/SECURITY.md | 35 +++- messagefoundry/api/approvals.py | 3 +- messagefoundry/api/outlive.py | 10 +- messagefoundry/auth/notifications.py | 20 ++- messagefoundry/auth/service.py | 73 ++++++--- messagefoundry/pipeline/alert_sinks.py | 5 +- messagefoundry/pipeline/security_notify.py | 14 +- messagefoundry/store/postgres.py | 2 + messagefoundry/store/sqlserver.py | 4 + messagefoundry/store/store.py | 3 + .../tests/test_ui_notify_address.py | 3 +- tests/test_ad_directory_identity.py | 3 +- tests/test_ad_session_reconcile.py | 3 +- tests/test_admin_new_ip.py | 3 +- tests/test_admin_notify_email_update.py | 3 +- tests/test_audit_actor_index.py | 140 ++++++++++++++++ tests/test_auth_oidc_service.py | 3 +- tests/test_auth_service.py | 13 +- tests/test_combined_sign_in_audit_oracle.py | 3 +- tests/test_initial_credential_reminder.py | 3 +- tests/test_lock_notice_off_request_path.py | 153 ++++++++++++++---- tests/test_login_new_ip.py | 9 +- tests/test_mfa.py | 34 ++-- tests/test_notify_email_confinement.py | 3 +- tests/test_provision_first_administrator.py | 5 +- tests/test_reauth_lockout.py | 17 +- .../test_suspicious_login_security_events.py | 13 +- tests/test_webauthn.py | 3 +- 29 files changed, 464 insertions(+), 130 deletions(-) create mode 100644 changelog.d/3120.security.md create mode 100644 tests/test_audit_actor_index.py diff --git a/changelog.d/3120.security.md b/changelog.d/3120.security.md new file mode 100644 index 0000000000..c5b27b8d9e --- /dev/null +++ b/changelog.d/3120.security.md @@ -0,0 +1,11 @@ +- **A lock notice the relay's full queue drops no longer holds the next one back.** The security + notifier now says whether it queued a notice, and the `auth.lock_notice` row records + `mailed: false` for one it dropped, so the next lock of the kind mails again rather than staying + quiet for 24 hours. A send that fails later, on the relay's own task, still leaves + `mailed: true`; `docs/SECURITY.md` states that residual. + **The lock notice's throttle read seeks an index.** A new `ix_audit_actor` index on + `audit_log(actor, action, id)`, on SQLite, Postgres and SQL Server, lets the read find one + account's rows instead of walking every recent notice row of every account. On the server + backends the schema change moves the ADR 0064 schema hash, so the next open runs the DDL. + **Declined, with residuals in `docs/SECURITY.md`:** two mails across engine shards serving their + own API ports, the timing of the content-free failure lines, and a cap on pending notice tasks. diff --git a/docs/SECURITY.md b/docs/SECURITY.md index 4135d92f21..5c30d3569b 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -4976,7 +4976,12 @@ the lock and its cycle count, and writes its own `auth.lock_notice` row, which i reads. With a relay wired, that read and the mail run in a background task (BACKLOG #2216). The refusal writes `auth.account_locked` first and does not wait for the task. So a large audit log would not push a refused sign-in past its padded slot. The task is queued per account and lock kind within -one API process. Two locks of one kind landing together there would send one mail. A sign-in lock +one API process. Two locks of one kind landing together there would send one mail. **One throttle +read runs at a time in each API process, across every account**, so locks on many accounts at once +cannot fill the store's read pool that sign-ins wait on; each mail then waits behind the reads +queued before it. The read seeks the account's own rows on an `(actor, action, id)` index, on every +backend (vault BACKLOG #3120). The residuals are listed below, under *Declined, each with its +residual*. A sign-in lock notice on a TOTP-enrolled local account tells the owner to sign in with the password and the code together; a second-step notice says which factor was right and, **if the attempts were not the owner's**, to get a password reset or `admin-unlock` and replace that factor. @@ -5396,7 +5401,9 @@ a failed TLS key-exchange pin is logged once per process. **The cost of the tee stays broken, or recovers and breaks again, is reported only by that first line. **A notice the relay loses is audited (BACKLOG #2383).** The `auth.lock_notice` row is written -when the notice is handed to the relay, as `mailed: true`. When the relay's queue is full, or the +when the notice is handed to the relay. It reads `mailed: true` only when the relay queued the +notice: one its full queue dropped reads `mailed: false`, so the next lock of the kind mails again +(vault BACKLOG #3120). When the relay's queue is full, or the send fails, the engine writes one more row, with the notice's recipient as the actor. A lock notice writes `auth.lock_notice_undelivered`, hidden as above. Every other notice kind writes `auth.security_notice_undelivered`, which every `audit:read` reader sees. The detail names the @@ -5410,8 +5417,12 @@ source of record is `SecurityEventNotifier._record_undelivered` in - An offline command such as `provision-admin` wires no audit writer, so it records nothing. - Under a long overload only 64 queue-full records are written at a time, and drops past that are not recorded. -- A lock notice lost this way still leaves its `auth.lock_notice` row reading `mailed: true`, so - the throttle holds back the next one. +- A lock notice whose send fails still leaves its `auth.lock_notice` row reading `mailed: true`, + because the send fails on the relay's own task after that row is written. So the throttle holds + back the next lock notice of the kind for the rest of its 24 hours. The failure is on the + `auth.lock_notice_undelivered` row, which the throttle does not read. *Kept as a residual (vault + BACKLOG #3120): closing it would need that row to carry the lock kind and the throttle to read a + second action.* **Residual: the background task adds one way to lose a notice** (BACKLOG #2216). At shutdown the engine waits about two seconds for pending notices. A notice still pending then is cut off, and may be @@ -5420,6 +5431,22 @@ the request, and now it is logged instead. The mail goes before its row, so that most a duplicate mail at the next lock. Each such line names no account and no lock. It still tells a `logs:view` reader when some notice failed. +**Declined, each with its residual (vault BACKLOG #3120).** + +- **Two mails across engine shards.** The per-account queue and the one-read-at-a-time limit are + each per API process. Engine shards serving their own API ports would each keep their own, so two + locks of one kind landing on two of them at once could each read the throttle before the other + wrote its row, and the owner would get two mails. A duplicate mail is the cheap failure here. +- **The timing of the failure lines.** The paragraph above states it: a line that names nothing + still tells a `logs:view` reader, by when it appears, that some lock's notice failed. +- **No cap on pending notice tasks.** Each lock that lands with a relay wired starts one task, and + nothing bounds how many wait. A task follows a lock, not an attempt, and a lock takes + `lockout_threshold` failures on its counter, so the count grows with the accounts being locked at + once rather than with the attempts against them. Under locks on many accounts the + tasks wait behind the one-at-a-time read, so mails arrive later, and any still pending at the + shutdown bound are cut off, each with one line. A cap would have to drop a notice or run it on + the request path, which is what the task exists to avoid. + **Client attribution ([ADR 0150](adr/0150-client-address-on-audit-entries.md)).** Every row also carries a `client` column — the caller's network address, stamped at write time from the request via the same `client_ip()` the new-client-IP risk signal uses (so behind a declared `trusted_proxies` diff --git a/messagefoundry/api/approvals.py b/messagefoundry/api/approvals.py index ec0843c8d0..c903608612 100644 --- a/messagefoundry/api/approvals.py +++ b/messagefoundry/api/approvals.py @@ -106,7 +106,8 @@ #: How long :meth:`ApprovalGate.drain` waits for outcome writes still running at shutdown. It shares #: NSSM's 15 s graceful-stop window (AppStopMethodConsole in scripts/service/install-service.ps1) -#: with the upload runner's 5 s stop, which runs first, and engine.stop() still has to run after it. +#: with every other shutdown drain, and engine.stop() still has to run after them. The drains, their +#: order and their sum are stated once, at ``_BACKGROUND_DRAIN_SECONDS`` in ``auth/service.py``. #: One outcome write is a status update and an audit row, so a few seconds is ample. DRAIN_TIMEOUT_SECONDS = 3.0 diff --git a/messagefoundry/api/outlive.py b/messagefoundry/api/outlive.py index b1b00a58d7..17a88a8cca 100644 --- a/messagefoundry/api/outlive.py +++ b/messagefoundry/api/outlive.py @@ -38,17 +38,17 @@ log = logging.getLogger(__name__) #: How long :meth:`OutlivingOperations.drain` lets a running operation finish at shutdown before it -#: cancels it. It shares NSSM's 15 s graceful-stop window with the upload runner's stop and the -#: approval gate's drain, so it is short; an operation it cancels records ``interrupted``. +#: cancels it. It shares NSSM's 15 s graceful-stop window with the other shutdown drains, so it is +#: short; an operation it cancels records ``interrupted``. DRAIN_TIMEOUT_SECONDS = 0.5 #: How long the drain then waits for the cancelled operations' rollback arms to finish, so their #: interrupted rows reach the store before ``engine.stop()`` closes it. Longer than the drain itself #: because at least one arm waits on a process: a cancelled DR takeover hook is killed and reaped #: for up to ``_HOOK_REAP_SECONDS`` before the activation's arm writes its row, so this is that -#: bound plus a second for the write. Every shutdown bound together (the upload runner's stop, this -#: drain and grace, the approval gate's drain) comes close to NSSM's 15 s window only when all of -#: them are busy at once. +#: bound plus a second for the write. Every shutdown bound together passes NSSM's 15 s window when +#: all of them are busy at once; the drains, their order and their sum are stated once, at +#: ``_BACKGROUND_DRAIN_SECONDS`` in ``auth/service.py``. CANCEL_GRACE_SECONDS = _HOOK_REAP_SECONDS + 1.0 diff --git a/messagefoundry/auth/notifications.py b/messagefoundry/auth/notifications.py index 080818a65a..c06a1f3b0d 100644 --- a/messagefoundry/auth/notifications.py +++ b/messagefoundry/auth/notifications.py @@ -128,10 +128,12 @@ #: #: Only ``ACCOUNT_LOCKED`` carries that bit. Every other kind is sent on an authenticated action, a #: completed sign-in, or a clock, so it keeps its per-event line (BACKLOG #1139). A lock notice with -#: no relay or no address is still recorded where only an administrator reads it, as ``mailed: false`` -#: on the ``auth.lock_notice`` row. A full queue or a failed send is recorded on its own -#: ``auth.lock_notice_undelivered`` row (BACKLOG #2383), since the ``auth.lock_notice`` row is written -#: at the hand-off, as ``mailed: true``. An instance with no relay at all is reported at startup by +#: no relay, no address or no room on the notifier's queue is still recorded where only an +#: administrator reads it, as ``mailed: false`` on the ``auth.lock_notice`` row (vault BACKLOG #3120 +#: for the full queue). A full queue or a failed send is also recorded on its own +#: ``auth.lock_notice_undelivered`` row (BACKLOG #2383). A failed send happens after the +#: ``auth.lock_notice`` row is written at the hand-off, so that row still reads ``mailed: true``; the +#: residual is stated in ``docs/SECURITY.md``. An instance with no relay at all is reported at startup by #: the serve gate, unless enforcement is ``warn`` and the operator waived notices in writing. LOG_SILENT_EVENT_TYPES: frozenset[str] = frozenset({ACCOUNT_LOCKED}) @@ -231,9 +233,15 @@ class SecurityEvent: class SecurityNotifier(Protocol): """Push a security event to the affected user out-of-band. Implemented in ``pipeline/`` (email over the ``[alerts]`` SMTP transport) and injected into :class:`AuthService`. Must be best-effort and - must not raise into the auth path (the caller still guards it).""" + must not raise into the auth path (the caller still guards it). - async def notify(self, event: SecurityEvent) -> None: ... + ``notify`` returns whether it took the event for sending: ``False`` when it dropped the event + on the spot, for example for want of an address or room on its queue (vault BACKLOG #3120). + ``True`` is a hand-off, never a delivery receipt: a send that fails later is not reported back. + The ``auth.lock_notice`` row's ``mailed`` field is recorded from this, so a notice dropped here + does not hold the next lock's notice back for a day.""" + + async def notify(self, event: SecurityEvent) -> bool: ... def deadline_utc(ts: float) -> str | None: diff --git a/messagefoundry/auth/service.py b/messagefoundry/auth/service.py index 9e7c806be5..ccf6c9b2f2 100644 --- a/messagefoundry/auth/service.py +++ b/messagefoundry/auth/service.py @@ -1804,7 +1804,8 @@ class ProvisionedAdministrator: #: The account had no notification address before the repair, so there was nobody to tell. HOLDER_NOTICE_NO_PRIOR_ADDRESS = "no_prior_address" #: The account had an address, but nothing was handed off: no channel was wired, notices are turned -#: off, or the notifier raised (each logged where it applies). +#: off, the notifier raised, or it dropped the notice on a full queue (each logged where it applies; +#: vault BACKLOG #3120 for the last). HOLDER_NOTICE_NO_CHANNEL = "no_channel" @@ -2026,10 +2027,19 @@ def _idp_error_fields(exc: BaseException) -> dict[str, object]: _LOCK_NOTICE_WINDOW_SECONDS: Final = 24 * 3600.0 #: BACKLOG #2216: how long the API lifespan's :meth:`AuthService.drain_background` lets the #: service's background tasks finish at shutdown before it cancels the rest, and how long it then -#: waits for the cancelled ones to unwind. Both are shares of the service's graceful-stop window, -#: whose budget is stated at ``DRAIN_TIMEOUT_SECONDS`` in ``api/approvals.py``; keep the sum inside -#: it. A task still running at the first bound is queued behind other throttle reads, or stuck on -#: the store. A task still unwinding at the second is left to finish or fail on its own. +#: waits for the cancelled ones to unwind. A task still running at the first bound is queued behind +#: other throttle reads, or stuck on the store. A task still unwinding at the second is left to +#: finish or fail on its own. +#: +#: The budget is NSSM's 15 s graceful-stop window (``AppStopMethodConsole`` in +#: ``scripts/service/install-service.ps1``), not any one drain's constant (vault BACKLOG #3120). The +#: lifespan teardown in ``api/app.py`` spends it in order, after uvicorn's own connection drain: +#: the upload runner's stop (``uploads._DEFAULT_STOP_TIMEOUT_SECONDS``, 5 s), the long-operation +#: drain (``api/outlive.py``, 0.5 s plus ``CANCEL_GRACE_SECONDS``, 6 s), the approval gate's drain +#: (``api/approvals.DRAIN_TIMEOUT_SECONDS``, 3 s), then this one (2.0 s plus 0.5 s), and +#: ``engine.stop()`` after them all. Those bounds add up to 17 s, past the window, so the +#: teardown fits it only while they are not all busy at once; uvicorn's drain has no bound at all. +#: A change to any of them is read against that sum. _BACKGROUND_DRAIN_SECONDS: Final = 2.0 _BACKGROUND_CANCEL_GRACE_SECONDS: Final = 0.5 @@ -2584,15 +2594,19 @@ def __init__( # one kind landing together read the throttle one after the other and mail once. Its own table: # the credential queue above is held across a sign-in's pad, and the notice runs off it. Per # API process, like that queue: engine shards serving their own API ports each keep one, so - # two locks landing on two of them at once can still mail twice. + # two locks landing on two of them at once can still mail twice. Declined in vault BACKLOG + # #3120, with the residual in docs/SECURITY.md. self._lock_notice_locks: dict[str, _KeyedLock] = {} # BACKLOG #2216: strong references to the service's background tasks, so a running one is not - # garbage-collected. Each removes itself when done; drain_background() awaits the rest. + # garbage-collected. Each removes itself when done; drain_background() awaits the rest. No + # cap on how many wait, declined in vault BACKLOG #3120: a cap would have to drop a lock + # notice or run it on the request path. docs/SECURITY.md states the residual. self._background_tasks: set[asyncio.Task[None]] = set() # Set by drain_background(close=True) at shutdown. After it, a lock notice runs inline, # since a task started then would outlive the drain and meet a closed store. self._background_closed = False - # One throttle read at a time (BACKLOG #2216); see _lock_notice_held_back. + # One throttle read at a time in this process, across every account (BACKLOG #2216); see + # _lock_notice_held_back. self._lock_notice_read_slots = asyncio.Semaphore(1) # Per-SESSION failed re-proofs (BACKLOG #1138): token_hash -> (failures charged to that # session, monotonic time of the first, the account's user id). At lockout_threshold the session is revoked, so a stolen session gets that many @@ -9916,9 +9930,9 @@ async def _classify_login_address(self, user: UserRecord, client: str | None) -> **THE BASELINE IS THE ACCOUNT'S KNOWN-ADDRESS RECORD** (vault BACKLOG #2145): the host keys that :meth:`_mark_login_address_known` writes, which says when, read through the record's - primary key on the account id. It replaced a baseline read from ``audit_log``, which has no - actor index and could not tell a finished sign-in from one that stopped at the password on - the directory leg. Keyed on the + primary key on the account id. It replaced a baseline read from ``audit_log``, which then had + no actor index and could not tell a finished sign-in from one that stopped at the password + on the directory leg. Keyed on the account id, so a re-created namesake inherits nothing, and the account's deletion removes it. Only rows seen within ``_LOGIN_ADDRESS_LOOKBACK_SECONDS`` count. Addresses compare by @@ -13468,10 +13482,13 @@ async def _lock_notice_held_back(self, user: UserRecord, lock: str) -> bool: """The throttle's READ: whether a notice of this ``lock`` kind already went out in the window. Kept apart from the row write, so the background task can send the mail between this read - and :meth:`_record_lock_notice` (BACKLOG #2216). One such read runs at a time per - service (``_lock_notice_read_slots``). It walks every recent ``auth.lock_notice`` row of - every account, since the audit log has no actor index. Many locks at once would otherwise - fill the store's read pool, which sign-ins wait on too. + and :meth:`_record_lock_notice` (BACKLOG #2216). It seeks this account's rows of the + action on the ``ix_audit_actor`` (actor, action, id) index, on every backend (vault + BACKLOG #3120); before it, the read walked every recent ``auth.lock_notice`` row of every + account. One such read still runs at a time per service (``_lock_notice_read_slots``), so + locks on many accounts at once cannot fill the store's read pool, which sign-ins wait on + too. The cost is that each notice waits behind the reads queued before it, across every + account in this process; ``docs/SECURITY.md`` states it. A failed read fails OPEN (``False``) and is logged by exception class alone. The line names neither the account nor the notice kind (BACKLOG #1131): it runs only when a lock lands, @@ -13505,9 +13522,15 @@ async def _record_lock_notice(self, user: UserRecord, lock: str, *, handed: bool """Write the ``auth.lock_notice`` row the throttle reads, after the mail. ``mailed`` is true only when the account had an address AND the notifier took the event - (``handed``, BACKLOG #2019). A refused hand-off then writes ``mailed: false``, which holds - back no later mailable notice, so the next lock mails again rather than staying quiet for - a day.""" + (``handed``, BACKLOG #2019). A refused hand-off, or one the notifier dropped on a full + queue (vault BACKLOG #3120), then writes ``mailed: false``, which holds back no later + mailable notice, so the next lock mails again rather than staying quiet for a day. + + **A send that fails after the hand-off still leaves ``mailed: true``.** It fails on the + notifier's own task, after this row is written, and is recorded only on the notifier's + ``auth.lock_notice_undelivered`` row (BACKLOG #2383), which the throttle does not read. So + the next lock of the kind is held back for the rest of the window. A residual + ``docs/SECURITY.md`` states.""" await self._audit( _LOCK_NOTICE_ACTION, actor=user.username, @@ -13528,9 +13551,11 @@ async def _notify_security( or an admin action. The caller writes the audit row, not this method: see :meth:`_record_suspicious_login` for the two 6.3.5 events. - Returns ``True`` only when the notifier took the event without raising (BACKLOG #2019), so a - caller that audits the outcome can record what happened. That is a hand-off, not a delivery. - Most callers ignore it. + Returns ``True`` only when the notifier took the event without raising (BACKLOG #2019) and + said it took it: the shipped notifier answers ``False`` for an event it dropped for want of + an address or of room on its queue (vault BACKLOG #3120). So a caller that audits the outcome + can record what happened. That is a hand-off, not a delivery: a send that fails later, on + the notifier's own task, is not reported back. Most callers ignore it. The docstring used to promise both arms were logged while only the failure arm was (BACKLOG #1139), so read the branches rather than this paragraph if they ever diverge again.""" @@ -13577,7 +13602,7 @@ async def _notify_security( ) return False try: - await self._security_notifier.notify( + taken = await self._security_notifier.notify( SecurityEvent( event_type=event_type, username=username, @@ -13601,4 +13626,6 @@ async def _notify_security( type(exc).__name__, ) return False - return True + # bool(): a notifier outside the contract that returns nothing reads as not taken, so the + # cheap failure follows (a duplicate lock notice), never a day held back. + return bool(taken) diff --git a/messagefoundry/pipeline/alert_sinks.py b/messagefoundry/pipeline/alert_sinks.py index 5cf99e2f61..32fbcb4603 100644 --- a/messagefoundry/pipeline/alert_sinks.py +++ b/messagefoundry/pipeline/alert_sinks.py @@ -170,8 +170,9 @@ def __init__(self, *, max_queue: int = _MAX_QUEUE) -> None: self._queue: asyncio.Queue[_T | None] = asyncio.Queue(maxsize=max_queue) self._task: asyncio.Task[None] | None = None - def _enqueue(self, item: _T, *, dropped: str | None) -> None: + def _enqueue(self, item: _T, *, dropped: str | None) -> bool: """Non-blocking enqueue; on a full queue, drop the item with a warning (``dropped`` names it). + Returns whether the item was queued. ``dropped=None`` drops without a line, for an item whose drop must not show in the general log (a lock notice, BACKLOG #1131; see ``auth.notifications.LOG_SILENT_EVENT_TYPES``).""" @@ -180,6 +181,8 @@ def _enqueue(self, item: _T, *, dropped: str | None) -> None: except asyncio.QueueFull: if dropped is not None: log.warning("%s queue full; dropping %s", type(self).__name__, dropped) + return False + return True def start(self) -> None: if self._task is None: diff --git a/messagefoundry/pipeline/security_notify.py b/messagefoundry/pipeline/security_notify.py index 5f48b64105..76ed4419a8 100644 --- a/messagefoundry/pipeline/security_notify.py +++ b/messagefoundry/pipeline/security_notify.py @@ -584,7 +584,10 @@ def __init__( if use_tls and not tls_verify: warn_smtp_verification_off(cell=ALERTS_SMTP_CELL, host=host) - async def notify(self, event: SecurityEvent) -> None: + async def notify(self, event: SecurityEvent) -> bool: + # Returns whether the event was queued for sending, the contract on ``SecurityNotifier``: + # False for no address and for a full queue (vault BACKLOG #3120), never a delivery receipt. + # # No deliverable address means nothing to email. The caller has audited the event; which # events the /me/security-events feed shows is stated once, in auth/notifications.py. # Non-blocking enqueue. @@ -625,21 +628,20 @@ async def notify(self, event: SecurityEvent) -> None: event.event_type, event.username, ) - return - # Read before the enqueue, with no await between, so it is the queue the enqueue meets. - full = self._queue.full() - self._enqueue( + return False + queued = self._enqueue( event, dropped=None if event.event_type in LOG_SILENT_EVENT_TYPES else f"{event.event_type} for {event.username}", ) - if full and self._audit is not None and len(self._recording) < _MAX_RECORDING: + if not queued and self._audit is not None and len(self._recording) < _MAX_RECORDING: # Off the caller's path: a notice must never block a sign-in or an admin action, and # this write would wait on the store under exactly the overload that filled the queue. task = asyncio.create_task(self._record_undelivered(event, {"reason": "queue_full"})) self._recording.add(task) task.add_done_callback(self._recording.discard) + return queued async def aclose(self) -> None: try: diff --git a/messagefoundry/store/postgres.py b/messagefoundry/store/postgres.py index 4c029ba493..7156cd39fc 100644 --- a/messagefoundry/store/postgres.py +++ b/messagefoundry/store/postgres.py @@ -721,6 +721,8 @@ def __init__(self, method: str, outbox_ids: tuple[str, ...]) -> None: "CREATE INDEX IF NOT EXISTS ix_audit_ts ON audit_log(ts)", # recent_audit_of: the start's config baseline read (vault BACKLOG #2597). "CREATE INDEX IF NOT EXISTS ix_audit_action ON audit_log(action, id)", + # The lock notice's throttle read, one account's rows of one action (vault BACKLOG #3120). + "CREATE INDEX IF NOT EXISTS ix_audit_actor ON audit_log(actor, action, id)", # No table beside audit_log says where its keying starts: the process that holds the key decides, # and the chain's own row at sequence number 1 (the genesis row) names the first range's key. # Per-key AES-GCM invocation bound (ASVS 11.3.4) — see the SQLite `_SCHEMA` for the diff --git a/messagefoundry/store/sqlserver.py b/messagefoundry/store/sqlserver.py index 2ba4688662..08c0dbba6f 100644 --- a/messagefoundry/store/sqlserver.py +++ b/messagefoundry/store/sqlserver.py @@ -2098,6 +2098,10 @@ def __init__(self, conn: Any, cur: Any) -> None: # recent_audit_of: the start's config baseline read (vault BACKLOG #2597). """IF INDEXPROPERTY(OBJECT_ID('audit_log'),'ix_audit_action','IndexID') IS NULL CREATE INDEX ix_audit_action ON audit_log(action, id)""", + # The lock notice's throttle read, one account's rows of one action (vault BACKLOG #3120). The + # key is at most 772 bytes (actor 512, action 256, id 4), inside the 1700-byte nonclustered limit. + """IF INDEXPROPERTY(OBJECT_ID('audit_log'),'ix_audit_actor','IndexID') IS NULL + CREATE INDEX ix_audit_actor ON audit_log(actor, action, id)""", # No table beside audit_log says where its keying starts: the process that holds the key decides, # and the chain's own row at sequence number 1 (the genesis row) names the first range's key. # Per-key AES-GCM invocation bound (ASVS 11.3.4) — see the SQLite `_SCHEMA` for the diff --git a/messagefoundry/store/store.py b/messagefoundry/store/store.py index 3468e5035a..404493ffac 100644 --- a/messagefoundry/store/store.py +++ b/messagefoundry/store/store.py @@ -6749,6 +6749,9 @@ def _dead_target_pairs(rows: Iterable[tuple[Any, Any]]) -> list[tuple[str, str]] -- The newest row of a few actions (recent_audit_of): the start's config baseline read, vault BACKLOG -- #2597. Without it that read walks the whole chain newest-first until one of the actions turns up. CREATE INDEX IF NOT EXISTS ix_audit_action ON audit_log(action, id); +-- One account's rows of one action, newest first: the lock notice's 24-hour throttle read, vault +-- BACKLOG #3120. Without it that read walks every recent row of the action, for every account. +CREATE INDEX IF NOT EXISTS ix_audit_actor ON audit_log(actor, action, id); -- There is no table beside audit_log that says where its keying starts. Whether a chain is keyed is -- decided by the process that holds the key, and the first range's key is named by the chain's own diff --git a/packaging/messagefoundry-webconsole/tests/test_ui_notify_address.py b/packaging/messagefoundry-webconsole/tests/test_ui_notify_address.py index b7c3ae662f..efc5051b81 100644 --- a/packaging/messagefoundry-webconsole/tests/test_ui_notify_address.py +++ b/packaging/messagefoundry-webconsole/tests/test_ui_notify_address.py @@ -32,8 +32,9 @@ class _FakeNotifier: def __init__(self) -> None: self.events: list[SecurityEvent] = [] - async def notify(self, event: SecurityEvent) -> None: + async def notify(self, event: SecurityEvent) -> bool: self.events.append(event) + return True async def _service( diff --git a/tests/test_ad_directory_identity.py b/tests/test_ad_directory_identity.py index fd81fe2461..1e5acbe87b 100644 --- a/tests/test_ad_directory_identity.py +++ b/tests/test_ad_directory_identity.py @@ -592,8 +592,9 @@ class _CapturingNotifier: def __init__(self) -> None: self.sent: list[SecurityEvent] = [] - async def notify(self, event: SecurityEvent) -> None: + async def notify(self, event: SecurityEvent) -> bool: self.sent.append(event) + return True async def test_a_login_rename_sends_the_holder_one_username_notice() -> None: diff --git a/tests/test_ad_session_reconcile.py b/tests/test_ad_session_reconcile.py index 4a7c31cd13..fadcb87d86 100644 --- a/tests/test_ad_session_reconcile.py +++ b/tests/test_ad_session_reconcile.py @@ -2654,8 +2654,9 @@ class _CapturingNotifier: def __init__(self) -> None: self.sent: list[Any] = [] - async def notify(self, event: Any) -> None: + async def notify(self, event: Any) -> bool: self.sent.append(event) + return True async def test_a_rename_and_a_role_change_in_one_pass_record_one_consistent_name() -> None: diff --git a/tests/test_admin_new_ip.py b/tests/test_admin_new_ip.py index b097bf6b39..0d7d941515 100644 --- a/tests/test_admin_new_ip.py +++ b/tests/test_admin_new_ip.py @@ -49,8 +49,9 @@ class _FakeNotifier: def __init__(self) -> None: self.events: list[SecurityEvent] = [] - async def notify(self, event: SecurityEvent) -> None: + async def notify(self, event: SecurityEvent) -> bool: self.events.append(event) + return True async def _enabled_admin(service: AuthService, *, client: str) -> tuple[str, Identity]: diff --git a/tests/test_admin_notify_email_update.py b/tests/test_admin_notify_email_update.py index 3cedf93123..73da029e50 100644 --- a/tests/test_admin_notify_email_update.py +++ b/tests/test_admin_notify_email_update.py @@ -48,8 +48,9 @@ class _FakeNotifier: def __init__(self) -> None: self.events: list[SecurityEvent] = [] - async def notify(self, event: SecurityEvent) -> None: + async def notify(self, event: SecurityEvent) -> bool: self.events.append(event) + return True async def _account(store: MessageStore, *, notify: str | None, profile: str | None) -> None: diff --git a/tests/test_audit_actor_index.py b/tests/test_audit_actor_index.py new file mode 100644 index 0000000000..328375ea38 --- /dev/null +++ b/tests/test_audit_actor_index.py @@ -0,0 +1,140 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +# Copyright (C) 2026 MessageFoundry Foundation, LLC and contributors +"""Vault BACKLOG #3120, item (d) and review follow-up 5: an (actor, action, id) audit index. + +The lock notice's 24-hour throttle reads one account's ``auth.lock_notice`` rows through +``list_audit(actor=, action=, since=, limit=50)``. With only ``ix_audit_ts`` and +``ix_audit_action`` that read walked every recent row of the action, for every account. +``ix_audit_actor`` lets it seek the account's own rows, newest first. + +Backend-parametrized like ``test_fifo_index_migration``: the SQLite case runs everywhere; the SQL +Server and Postgres cases run only when their ``MEFOR_TEST_*`` env is set, which the hosted service- +container legs do. +""" + +from __future__ import annotations + +import contextlib +import os +from collections.abc import AsyncIterator, Sequence +from pathlib import Path +from typing import Any + +import pytest + +from messagefoundry.store import MessageStore + +_SQLSERVER_ON = bool(os.getenv("MEFOR_TEST_SQLSERVER")) +_POSTGRES_ON = bool(os.getenv("MEFOR_TEST_POSTGRES")) + +_LOCK_NOTICE = "auth.lock_notice" + + +async def _open(backend: str, path: Path) -> Any: + if backend == "sqlite": + return await MessageStore.open(path) + from messagefoundry.config.settings import load_settings + + settings = load_settings(environ=os.environ).store + if backend == "postgres": + from messagefoundry.store.postgres import PostgresStore + + return await PostgresStore.open(settings) + from messagefoundry.store.sqlserver import SqlServerStore + + return await SqlServerStore.open(settings) + + +@pytest.fixture(params=["sqlite", "sqlserver", "postgres"]) +async def store(request: pytest.FixtureRequest, tmp_path: Path) -> AsyncIterator[Any]: + backend = request.param + if backend == "sqlserver" and not _SQLSERVER_ON: + pytest.skip("set MEFOR_TEST_SQLSERVER=1 (+ MEFOR_STORE_* env) to run the SQL Server case") + if backend == "postgres" and not _POSTGRES_ON: + pytest.skip("set MEFOR_TEST_POSTGRES=1 (+ MEFOR_STORE_* env) to run the Postgres case") + s: Any = await _open(backend, tmp_path / "actor_idx.db") + s._test_backend = backend + try: + yield s + finally: + await s.close() + + +async def _index_columns(store: Any, name: str) -> list[str]: + """The key columns of the ``audit_log`` index ``name``, in key order; empty when it is absent.""" + backend = store._test_backend + if backend == "sqlite": + async with store._read() as db: + cur = await db.execute(f"PRAGMA index_info({name})") + return [r[2] for r in sorted(await cur.fetchall(), key=lambda r: r[0])] + if backend == "postgres": + rows = await store._fetchall( + "SELECT a.attname FROM pg_index i" + " JOIN pg_class c ON c.oid = i.indexrelid" + " JOIN LATERAL unnest(i.indkey) WITH ORDINALITY AS k(attnum, ord) ON true" + " JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = k.attnum" + " WHERE c.relname = $1 ORDER BY k.ord", + name, + ) + return [r["attname"] for r in rows] + rows = await store._fetchall( # sqlserver + "SELECT c.name FROM sys.indexes i" + " JOIN sys.index_columns ic ON ic.object_id=i.object_id AND ic.index_id=i.index_id" + " JOIN sys.columns c ON c.object_id=i.object_id AND c.column_id=ic.column_id" + " WHERE i.object_id=OBJECT_ID('audit_log') AND i.name=? AND ic.key_ordinal > 0" + " ORDER BY ic.key_ordinal", + (name,), + ) + return [r["name"] for r in rows] + + +async def test_the_actor_index_exists_on_every_backend(store: Any) -> None: + assert await _index_columns(store, "ix_audit_actor") == ["actor", "action", "id"] + # The control: the same probe reads the index that was already there, so an empty list above + # would be the index missing, not the probe failing. + assert await _index_columns(store, "ix_audit_action") == ["action", "id"] + + +async def test_the_throttle_read_seeks_the_actor_index_on_sqlite( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The plan of the statement ``list_audit`` runs for the throttle names ``ix_audit_actor``. + + The statement and its parameters are captured off the read connection as ``list_audit`` + really runs them, not rebuilt here, so the test cannot drift from the method. The control is + an action-only read, whose plan names ``ix_audit_action``, so the instrument is known to report + the index it uses.""" + store = await MessageStore.open(tmp_path / "plan.db") + real_read = store._read + + async def plan(**filters: Any) -> str: + seen: list[tuple[str, Sequence[Any]]] = [] + + class _Spy: + def __init__(self, db: Any) -> None: + self._db = db + + async def execute(self, sql: str, params: Sequence[Any] = ()) -> Any: + seen.append((sql, params)) + return await self._db.execute(sql, params) + + @contextlib.asynccontextmanager + async def spy_read() -> AsyncIterator[_Spy]: + async with real_read() as db: + yield _Spy(db) + + with monkeypatch.context() as m: + m.setattr(store, "_read", spy_read) + await store.list_audit(limit=50, **filters) + [(sql, params)] = seen + async with real_read() as db: + cur = await db.execute(f"EXPLAIN QUERY PLAN {sql}", params) + return " | ".join(str(r[-1]) for r in await cur.fetchall()) + + try: + throttle = await plan(actor="owner", action=_LOCK_NOTICE, since=0.0) + control = await plan(action=_LOCK_NOTICE) + finally: + await store.close() + assert "ix_audit_actor" in throttle, throttle + assert "ix_audit_action" in control, control diff --git a/tests/test_auth_oidc_service.py b/tests/test_auth_oidc_service.py index aaea6cc846..01c8bce299 100644 --- a/tests/test_auth_oidc_service.py +++ b/tests/test_auth_oidc_service.py @@ -856,8 +856,9 @@ class _CapturingNotifier: def __init__(self) -> None: self.events: list[Any] = [] - async def notify(self, event: Any) -> None: + async def notify(self, event: Any) -> bool: self.events.append(event) + return True async def test_the_admin_bind_audits_and_notifies_and_a_login_never_does( diff --git a/tests/test_auth_service.py b/tests/test_auth_service.py index 5d94184cc8..8da1b78449 100644 --- a/tests/test_auth_service.py +++ b/tests/test_auth_service.py @@ -50,8 +50,9 @@ class _FakeNotifier: def __init__(self) -> None: self.events: list[SecurityEvent] = [] - async def notify(self, event: SecurityEvent) -> None: + async def notify(self, event: SecurityEvent) -> bool: self.events.append(event) + return True async def _store() -> MessageStore: @@ -450,9 +451,9 @@ async def _local_user(store: MessageStore, *, email: str = "bob@example.org") -> async def test_notifier_fires_once_on_account_lockout() -> None: store = await _store() + notifier = _FakeNotifier() + service = AuthService(store, AuthSettings(lockout_threshold=3), security_notifier=notifier) try: - notifier = _FakeNotifier() - service = AuthService(store, AuthSettings(lockout_threshold=3), security_notifier=notifier) await _local_user(store) for _ in range(3): await service.login("bob", "wrong", client="10.0.0.9") @@ -464,6 +465,8 @@ async def test_notifier_fires_once_on_account_lockout() -> None: assert locked[0].email == "bob@example.org" assert locked[0].client_ip == "10.0.0.9" finally: + # Again here, so a failure above cannot leave the notice running (vault BACKLOG #3120). + await service.drain_background(timeout=20.0) await store.close() @@ -601,7 +604,7 @@ async def test_notifier_absent_does_not_break_auth() -> None: class _BoomNotifier: """A notifier whose notify() always raises — exercises AuthService's best-effort guard.""" - async def notify(self, event: SecurityEvent) -> None: + async def notify(self, event: SecurityEvent) -> bool: raise RuntimeError("notifier down") @@ -631,7 +634,7 @@ async def test_notifier_failure_is_isolated_from_the_auth_op() -> None: class _AddressQuotingNotifier: """A notifier that raises with the event's address and detail in its message.""" - async def notify(self, event: SecurityEvent) -> None: + async def notify(self, event: SecurityEvent) -> bool: raise RuntimeError(f"cannot mail {event.email}: {event.detail}") diff --git a/tests/test_combined_sign_in_audit_oracle.py b/tests/test_combined_sign_in_audit_oracle.py index 721ef2a49b..27ea3f8623 100644 --- a/tests/test_combined_sign_in_audit_oracle.py +++ b/tests/test_combined_sign_in_audit_oracle.py @@ -245,8 +245,9 @@ class _FakeNotifier: def __init__(self) -> None: self.events: list[Any] = [] - async def notify(self, event: Any) -> None: + async def notify(self, event: Any) -> bool: self.events.append(event) + return True # --- The lock-event limb, closed by owner ruling 2026-09-28 -------------------------------------- diff --git a/tests/test_initial_credential_reminder.py b/tests/test_initial_credential_reminder.py index 645abb23cf..272a6838fc 100644 --- a/tests/test_initial_credential_reminder.py +++ b/tests/test_initial_credential_reminder.py @@ -234,8 +234,9 @@ class _RecordingNotifier: def __init__(self) -> None: self.events: list[SecurityEvent] = [] - async def notify(self, event: SecurityEvent) -> None: + async def notify(self, event: SecurityEvent) -> bool: self.events.append(event) + return True def reminders(self, holder: str) -> list[tuple[str, str, str | None, dict[str, Any]]]: """The reminders about ``holder``'s credential: its own, and its issuer's naming it. Other diff --git a/tests/test_lock_notice_off_request_path.py b/tests/test_lock_notice_off_request_path.py index a752657a88..61d6270ec0 100644 --- a/tests/test_lock_notice_off_request_path.py +++ b/tests/test_lock_notice_off_request_path.py @@ -30,11 +30,13 @@ from messagefoundry.auth.service import AuthService from messagefoundry.config.settings import AlertsSettings, AuthSettings, EgressSettings from messagefoundry.keyed_lock import hold_keyed_lock +from messagefoundry.pipeline.security_notify import SecurityEventNotifier from messagefoundry.store import open_store, sqlite_settings from messagefoundry.store.store import LockoutIncrement, MessageStore, UserRecord from tests._admin_account import ADMIN_USERNAME, create_admin _LOCK_NOTICE = "auth.lock_notice" +_LOCK_NOTICE_UNDELIVERED = "auth.lock_notice_undelivered" #: Longer than the deferred writes' room in the sign-in pad. A write this slow on the request path #: moves the answer to a later slot. _SLOW = service_module._FAILURE_BUDGET_SECONDS * service_module._WRITE_ROOM_SHARE * 2.4 @@ -50,8 +52,9 @@ class _FakeNotifier: def __init__(self) -> None: self.events: list[SecurityEvent] = [] - async def notify(self, event: SecurityEvent) -> None: + async def notify(self, event: SecurityEvent) -> bool: self.events.append(event) + return True def locks(self) -> list[SecurityEvent]: return [e for e in self.events if e.event_type == ACCOUNT_LOCKED] @@ -77,8 +80,9 @@ async def _harness( async def _seed_other_accounts_notices(store: MessageStore, count: int) -> None: """``count`` recent ``auth.lock_notice`` rows for OTHER accounts, written straight to the table. - The throttle walks every recent row of the action to find this account's, since the audit log - has no actor index, so these are the rows that make its read slow. Inserted in one statement + Before the actor index (vault BACKLOG #3120) the throttle walked every recent row of the + action to find this account's, so these were the rows that made its read slow; the test still + holds the read at a gate, so it does not depend on the read's speed. Inserted in one statement rather than through ``record_audit``, which chains each row and would take minutes. The chain is not under test here.""" db = store._db @@ -100,6 +104,18 @@ async def _seed_other_accounts_notices(store: MessageStore, count: int) -> None: await db.commit() +async def _close(store: MessageStore, service: AuthService) -> None: + """Finish the service's background tasks, then close the store (vault BACKLOG #3120). + + Called from each test's ``finally``, so a test that fails before its own drain leaves no + notice running on, into a closed store and a later test on the shared loop. Bounded, because a + failed test can leave a read held at a gate; a task still running then is cancelled.""" + try: + await service.drain_background(timeout=_STALL_SECONDS) + finally: + await store.close() + + def _in_background(service: AuthService) -> bool: """Whether the running task is one of the service's own background tasks.""" return asyncio.current_task() in service._background_tasks @@ -172,9 +188,10 @@ async def test_the_lock_setting_refusal_stays_in_its_slot_with_a_large_audit_log writes its rows in a room of an eighth of a second, and a throttle read slower than that moved the answer to the next slot and fired the writes-overrun warning. - The audit log holds many ``auth.lock_notice`` rows for other accounts, which is what makes the - real read slow. The read is also held at a gate until the refusal has answered. So it cannot - finish inside the answer, and its walk of those rows cannot compete with the refusal's own + The audit log holds many ``auth.lock_notice`` rows for other accounts, which made the real read + slow before the actor index (vault BACKLOG #3120). The read is also held at a gate until the + refusal has answered. So it cannot finish inside the answer, and its read cannot compete with + the refusal's own writes. The test also checks which task ran the read, so a read on the request path fails it directly. The notice still goes out once the gate opens, and the seeded rows of other accounts do not hold this account's mail back.""" @@ -207,7 +224,7 @@ async def test_the_lock_setting_refusal_stays_in_its_slot_with_a_large_audit_log rows = await _lock_notice_rows(store, user.username) assert [json.loads(r["detail"]) for r in rows] == [{"lock": "sign_in", "mailed": True}] finally: - await store.close() + await _close(store, service) async def test_the_overrun_instrument_fires_for_a_slow_write_still_on_the_path( @@ -238,7 +255,7 @@ async def slow_lock_row(action: str, **kwargs: Any) -> Any: assert _write_overruns(caplog), "the overrun warning did not fire for an on-path write" assert refused_by > service_module._FAILURE_BUDGET_SECONDS + 0.05, refused_by finally: - await store.close() + await _close(store, service) # --- the race --------------------------------------------------------------------------------- @@ -417,7 +434,7 @@ async def test_two_same_kind_locks_at_once_send_one_mail_and_write_one_row( assert len(locked) == 2, "each lock still writes its own auth.account_locked row" assert service._lock_notice_locks == {}, "the queue table kept an entry nobody holds" finally: - await store.close() + await _close(store, service) async def test_the_race_is_real_without_the_queue(monkeypatch: pytest.MonkeyPatch) -> None: @@ -436,7 +453,7 @@ async def test_the_race_is_real_without_the_queue(monkeypatch: pytest.MonkeyPatc assert len(notifier.locks()) == 2 assert len(await _lock_notice_rows(store, user.username)) == 2 finally: - await store.close() + await _close(store, service) async def _peak_reads_across_accounts( @@ -507,7 +524,7 @@ async def counted(*args: Any, **kwargs: Any) -> Any: progress.stalls, ) finally: - await store.close() + await _close(store, service) async def test_one_throttle_read_runs_at_a_time_across_accounts( @@ -580,7 +597,7 @@ async def counted(*args: Any, **kwargs: Any) -> Any: assert peak == 2, "the two kinds' notices queued behind each other" assert sorted(e.detail["lock"] for e in notifier.locks()) == ["second_step", "sign_in"] finally: - await store.close() + await _close(store, service) # --- failure and shutdown --------------------------------------------------------------------- @@ -629,7 +646,7 @@ async def test_a_notice_cancelled_at_shutdown_logs_a_line_that_names_nothing( locked = await store.list_audit(actor=user.username, action="auth.account_locked") assert len(locked) == 1 finally: - await store.close() + await _close(store, service) async def test_a_notice_that_finishes_inside_the_bound_is_not_cancelled( @@ -651,7 +668,7 @@ async def test_a_notice_that_finishes_inside_the_bound_is_not_cancelled( assert len(notifier.locks()) == 1 assert len(await _lock_notice_rows(store, user.username)) == 1 finally: - await store.close() + await _close(store, service) async def test_a_failed_row_write_still_mails_and_logs_the_class_alone( @@ -680,7 +697,7 @@ async def failing(action: str, **kwargs: Any) -> Any: assert _naming(caplog, user) == [], _naming(caplog, user) assert len(notifier.locks()) == 1, "a failed row write dropped the mail" finally: - await store.close() + await _close(store, service) async def test_a_failed_throttle_read_logs_the_class_alone_and_mails( @@ -708,13 +725,13 @@ async def failing(*args: Any, **kwargs: Any) -> Any: assert _naming(caplog, user) == [], _naming(caplog, user) assert len(notifier.locks()) == 1, "a failed read must fail open and mail" finally: - await store.close() + await _close(store, service) class _RefusingNotifier(_FakeNotifier): """A notifier that refuses every hand-off, as a relay whose queue raised would.""" - async def notify(self, event: SecurityEvent) -> None: + async def notify(self, event: SecurityEvent) -> bool: raise OSError("relay refused the hand-off") @@ -731,15 +748,74 @@ async def test_a_refused_hand_off_does_not_hold_the_next_notice_back() -> None: assert [json.loads(r["detail"]) for r in rows] == [{"lock": "sign_in", "mailed": False}] attempts: list[SecurityEvent] = [] - async def took(event: SecurityEvent) -> None: + async def took(event: SecurityEvent) -> bool: attempts.append(event) + return True notifier.notify = took # type: ignore[method-assign] await service._record_lock(user, "sign_in", _lock(2), client=None) await service.drain_background() assert len(attempts) == 1, "the refused notice held the next one back" finally: - await store.close() + await _close(store, service) + + +def _relay(store: MessageStore | None = None) -> SecurityEventNotifier: + """The shipped notifier, never started, so nothing is sent, with room for ONE queued event.""" + relay = SecurityEventNotifier( + host="smtp.example.test", port=25, sender="mf@example.test", audit=store + ) + relay._queue = asyncio.Queue(maxsize=1) + return relay + + +async def test_the_shipped_notifier_says_whether_it_queued_the_notice() -> None: + """Vault BACKLOG #3120, follow-up 2: ``notify`` answers ``True`` only for an event it queued. + A full queue and a missing address each drop the event, and each answers ``False``.""" + relay = _relay() + queued = SecurityEvent(ACCOUNT_LOCKED, username="lock-a", email="a@example.test") + assert await relay.notify(queued) is True + full = SecurityEvent(ACCOUNT_LOCKED, username="lock-b", email="b@example.test") + assert await relay.notify(full) is False + no_address = SecurityEvent(ACCOUNT_LOCKED, username="lock-c", email=None) + assert await relay.notify(no_address) is False + assert relay._queue.qsize() == 1, "only the first event may have been queued" + + +async def test_a_notice_dropped_on_a_full_queue_does_not_hold_the_next_one_back() -> None: + """Vault BACKLOG #3120, follow-up 2. The shipped notifier's full queue drops a lock notice + without a line, and ``notify`` used to return nothing, so the row read ``mailed: true`` and + the throttle held the next lock's notice back for a day behind a mail that never left. The row + now says ``mailed: false``, and the next lock of the kind is queued once there is room. + + The control is the second lock's own row: with room on the queue it reads ``mailed: true``, so + the instrument can see a notice that was taken.""" + store, service, _fake, user = await _harness() + relay = _relay(store) + relay._queue.put_nowait( + SecurityEvent(ACCOUNT_LOCKED, username="filler", email="f@example.test") + ) + service.attach_security_notifier(relay) + try: + await service._record_lock(user, "sign_in", _lock(1), client=None) + await service.drain_background() + rows = await _lock_notice_rows(store, user.username) + assert [json.loads(r["detail"]) for r in rows] == [{"lock": "sign_in", "mailed": False}] + # BACKLOG #2383: the relay records the drop on its own hidden row too. + await asyncio.gather(*relay._recording) + dropped = await store.list_audit(actor=user.username, action=_LOCK_NOTICE_UNDELIVERED) + assert [json.loads(r["detail"])["reason"] for r in dropped] == ["queue_full"] + + relay._queue.get_nowait() # room again + await service._record_lock(user, "sign_in", _lock(2), client=None) + await service.drain_background() + rows = await _lock_notice_rows(store, user.username) + assert [json.loads(r["detail"])["mailed"] for r in rows] == [True, False], ( + "the dropped notice held the next one back" + ) + assert relay._queue.qsize() == 1, "the second lock's notice was not queued" + finally: + await _close(store, service) async def test_after_a_closing_drain_a_notice_runs_inline(monkeypatch: pytest.MonkeyPatch) -> None: @@ -754,17 +830,17 @@ async def test_after_a_closing_drain_a_notice_runs_inline(monkeypatch: pytest.Mo assert len(notifier.locks()) == 1, "the notice did not run inline after the drain" assert len(await _lock_notice_rows(store, user.username)) == 1 finally: - await store.close() + await _close(store, service) async def test_with_no_notifier_the_row_is_written_inline(monkeypatch: pytest.MonkeyPatch) -> None: """With no notifier wired the throttle reads nothing and writes one ``mailed: false`` row, so that arm stays on the request path and starts no task.""" store = await MessageStore.open(":memory:") + service = AuthService( + store, AuthSettings(lockout_threshold=1, require_mfa=False, notify_security_events=True) + ) try: - service = AuthService( - store, AuthSettings(lockout_threshold=1, require_mfa=False, notify_security_events=True) - ) admin = await create_admin(service) user = await store.get_user(admin.user_id) assert user is not None @@ -773,7 +849,7 @@ async def test_with_no_notifier_the_row_is_written_inline(monkeypatch: pytest.Mo rows = await _lock_notice_rows(store, user.username) assert [json.loads(r["detail"])["reason"] for r in rows] == ["no_notifier"] finally: - await store.close() + await _close(store, service) # --- the lifespan drains before the store closes ---------------------------------------------- @@ -832,6 +908,7 @@ async def test_the_lifespan_drains_a_pending_notice_before_the_store_closes( app = _managed_app(db) gate = asyncio.Event() pending_at_drain: list[bool] = [] + auth: AuthService | None = None try: async with app.router.lifespan_context(app): auth = await _lock_in_lifespan(app, monkeypatch, gate=gate) @@ -839,6 +916,7 @@ async def test_the_lifespan_drains_a_pending_notice_before_the_store_closes( async def open_gate_then_drain() -> None: # The notice is held until the shutdown drain starts, so it is pending there. + assert auth is not None pending_at_drain.append(bool(auth._background_tasks)) gate.set() await real_close() @@ -846,8 +924,9 @@ async def open_gate_then_drain() -> None: monkeypatch.setattr(auth, "close_background", open_gate_then_drain) finally: gate.set() # never leave a held read behind in the shared loop - # Join anything the lifespan left running, so it cannot outlive the test. - await auth.drain_background() + # Join anything the lifespan left running, so it cannot outlive the test. + if auth is not None: + await auth.drain_background(timeout=_STALL_SECONDS) assert pending_at_drain, "the lifespan never called close_background" assert pending_at_drain == [True], "the notice was not pending when the drain started" rows = await _rows_after_shutdown(db, _LOCK_NOTICE) @@ -864,17 +943,21 @@ async def test_the_lifespan_control_loses_the_row_without_the_drain( db = tmp_path / "nodrain.db" app = _managed_app(db) gate = asyncio.Event() - async with app.router.lifespan_context(app): - auth = await _lock_in_lifespan(app, monkeypatch, gate=gate) + auth: AuthService | None = None + try: + async with app.router.lifespan_context(app): + auth = await _lock_in_lifespan(app, monkeypatch, gate=gate) - async def skipped() -> None: - return None + async def skipped() -> None: + return None - monkeypatch.setattr(auth, "close_background", skipped) - # The store is closed now. Release the held read and finish the orphaned task here, so it - # meets the closed store deterministically and does not outlive the test. - gate.set() - await auth.drain_background() + monkeypatch.setattr(auth, "close_background", skipped) + finally: + # The store is closed now. Release the held read and finish the orphaned task here, so it + # meets the closed store deterministically and does not outlive the test. + gate.set() + if auth is not None: + await auth.drain_background(timeout=_STALL_SECONDS) assert await _rows_after_shutdown(db, _LOCK_NOTICE) == [], ( "the row survived with the drain neutralised, so the other test does not measure it" ) diff --git a/tests/test_login_new_ip.py b/tests/test_login_new_ip.py index 54ea3e38c1..c71fd6cbd6 100644 --- a/tests/test_login_new_ip.py +++ b/tests/test_login_new_ip.py @@ -50,14 +50,15 @@ class _FakeNotifier: def __init__(self) -> None: self.events: list[SecurityEvent] = [] - async def notify(self, event: SecurityEvent) -> None: + async def notify(self, event: SecurityEvent) -> bool: self.events.append(event) + return True class _RaisingNotifier: """A notifier that always fails, to prove a broken channel cannot refuse a login.""" - async def notify(self, event: SecurityEvent) -> None: + async def notify(self, event: SecurityEvent) -> bool: raise OSError("synthetic relay failure") @@ -798,8 +799,8 @@ async def test_the_record_is_keyed_on_the_account_not_the_username() -> None: async def test_the_signal_never_reads_the_audit_log(monkeypatch: pytest.MonkeyPatch) -> None: - """Gap 5. The baseline read is one primary-key lookup on the record; ``audit_log``, which has - no actor index, is not read at all. A ``list_audit`` that raises cannot change the verdict.""" + """Gap 5. The baseline read is one primary-key lookup on the record; ``audit_log`` is not read + at all. A ``list_audit`` that raises cannot change the verdict.""" store = await MessageStore.open(":memory:") try: notifier = _FakeNotifier() diff --git a/tests/test_mfa.py b/tests/test_mfa.py index 0c4b56cb0b..367c5d7dbb 100644 --- a/tests/test_mfa.py +++ b/tests/test_mfa.py @@ -59,8 +59,9 @@ class _FakeNotifier: def __init__(self) -> None: self.events: list[SecurityEvent] = [] - async def notify(self, event: SecurityEvent) -> None: + async def notify(self, event: SecurityEvent) -> bool: self.events.append(event) + return True async def _store() -> MessageStore: @@ -474,19 +475,19 @@ async def test_parallel_wrong_credentials_cannot_evade_the_account_lockout( also pins the one-notice-per-lockout contract. """ store = await _store() + threshold, burst = 3, 5 + notifier = _FakeNotifier() + service = AuthService( + store, + AuthSettings( + mfa_verify_min_elapsed_seconds=0, + lockout_threshold=threshold, + lockout_minutes=15, + mfa_recovery_code_count=3, + ), + security_notifier=notifier, + ) try: - threshold, burst = 3, 5 - notifier = _FakeNotifier() - service = AuthService( - store, - AuthSettings( - mfa_verify_min_elapsed_seconds=0, - lockout_threshold=threshold, - lockout_minutes=15, - mfa_recovery_code_count=3, - ), - security_notifier=notifier, - ) identity, token, password = await login_admin(service) counts = _count_verifies(monkeypatch, service) @@ -548,10 +549,11 @@ async def test_parallel_wrong_credentials_cannot_evade_the_account_lockout( # The per-account queue leaves no entry behind once every attempt has left it. assert service._credential_locks == {}, "the per-account lock entry must not leak" - # Arms 2 and 3 set locks too; finish their background notices before the store closes - # (BACKLOG #2216), so none runs on into a later test on the shared loop. - await service.drain_background() finally: + # Arms 2 and 3 set locks too; finish their background notices before the store closes + # (BACKLOG #2216), so none runs on into a later test on the shared loop. In the finally, + # so a failed assertion above cannot skip it (vault BACKLOG #3120). + await service.drain_background(timeout=20.0) await store.close() diff --git a/tests/test_notify_email_confinement.py b/tests/test_notify_email_confinement.py index 8db1e3cd43..e8a24b5bda 100644 --- a/tests/test_notify_email_confinement.py +++ b/tests/test_notify_email_confinement.py @@ -52,8 +52,9 @@ class _FakeNotifier: def __init__(self) -> None: self.events: list[SecurityEvent] = [] - async def notify(self, event: SecurityEvent) -> None: + async def notify(self, event: SecurityEvent) -> bool: self.events.append(event) + return True def _no_mfa(**overrides: Any) -> AuthSettings: diff --git a/tests/test_provision_first_administrator.py b/tests/test_provision_first_administrator.py index 0840e6bff2..6b86a16557 100644 --- a/tests/test_provision_first_administrator.py +++ b/tests/test_provision_first_administrator.py @@ -1417,8 +1417,9 @@ def __init__(self) -> None: # How many notices were queued when the drain ran: a drain before the notice loses it. self.queued_at_close: int | None = None - async def notify(self, event: SecurityEvent) -> None: + async def notify(self, event: SecurityEvent) -> bool: self.events.append(event) + return True def start(self) -> None: self.started = True @@ -1801,7 +1802,7 @@ async def test_a_notifier_that_raises_is_recorded_as_no_hand_off() -> None: """The row is written after the notice, so a notifier failure cannot be recorded as dispatched.""" class _Raising(_Recorder): - async def notify(self, event: SecurityEvent) -> None: + async def notify(self, event: SecurityEvent) -> bool: raise RuntimeError("synthetic notifier failure") store = await MessageStore.open(":memory:") diff --git a/tests/test_reauth_lockout.py b/tests/test_reauth_lockout.py index fecdf0b332..3e84423419 100644 --- a/tests/test_reauth_lockout.py +++ b/tests/test_reauth_lockout.py @@ -71,8 +71,9 @@ class _FakeNotifier: def __init__(self) -> None: self.events: list[SecurityEvent] = [] - async def notify(self, event: SecurityEvent) -> None: + async def notify(self, event: SecurityEvent) -> bool: self.events.append(event) + return True def _actions(feed: list[dict[str, Any]], action: str) -> list[dict[str, Any]]: @@ -540,13 +541,13 @@ async def test_a_wrong_current_password_counts_and_is_audited_once() -> None: async def test_the_reauth_crossing_attempt_audits_account_locked_once_and_notifies() -> None: store = await MessageStore.open(":memory:") + notifier = _FakeNotifier() + service = AuthService( + store, + AuthSettings(lockout_threshold=3, require_mfa=False), + security_notifier=notifier, + ) try: - notifier = _FakeNotifier() - service = AuthService( - store, - AuthSettings(lockout_threshold=3, require_mfa=False), - security_notifier=notifier, - ) await _local_user(store) identity, token = await _signed_in(service) for _ in range(3): @@ -570,6 +571,8 @@ async def test_the_reauth_crossing_attempt_audits_account_locked_once_and_notifi assert notices[0].detail == {"failed_attempts": 3, "lock": "sign_in", "cycle": 1} assert notices[0].client_ip == "10.0.0.9" finally: + # Again here, so a failure above cannot leave the notice running (vault BACKLOG #3120). + await service.drain_background(timeout=20.0) await store.close() diff --git a/tests/test_suspicious_login_security_events.py b/tests/test_suspicious_login_security_events.py index 9b092d1340..2770a75775 100644 --- a/tests/test_suspicious_login_security_events.py +++ b/tests/test_suspicious_login_security_events.py @@ -54,8 +54,9 @@ class _FakeNotifier: def __init__(self) -> None: self.events: list[SecurityEvent] = [] - async def notify(self, event: SecurityEvent) -> None: + async def notify(self, event: SecurityEvent) -> bool: self.events.append(event) + return True async def _store() -> MessageStore: @@ -211,11 +212,11 @@ async def test_a_lockout_crossed_on_the_second_factor_is_in_the_users_feed() -> async def test_the_lockout_row_is_written_when_a_notifier_is_wired_too() -> None: store = await _store() + notifier = _FakeNotifier() + service = AuthService( + store, AuthSettings(lockout_threshold=3, require_mfa=False), security_notifier=notifier + ) try: - notifier = _FakeNotifier() - service = AuthService( - store, AuthSettings(lockout_threshold=3, require_mfa=False), security_notifier=notifier - ) await _local_user(store) for _ in range(3): await service.login("bob", WRONG_PASSWORD) @@ -228,4 +229,6 @@ async def test_the_lockout_row_is_written_when_a_notifier_is_wired_too() -> None # The count stays in the notice, beside the lock kind and its cycle count (ADR 0197). assert notices[0].detail == {"failed_attempts": 3, "lock": "sign_in", "cycle": 1} finally: + # Again here, so a failure above cannot leave the notice running (vault BACKLOG #3120). + await service.drain_background(timeout=20.0) await store.close() diff --git a/tests/test_webauthn.py b/tests/test_webauthn.py index 4ac823fa06..19b4e6f834 100644 --- a/tests/test_webauthn.py +++ b/tests/test_webauthn.py @@ -43,8 +43,9 @@ class _FakeNotifier: def __init__(self) -> None: self.events: list[SecurityEvent] = [] - async def notify(self, event: SecurityEvent) -> None: + async def notify(self, event: SecurityEvent) -> bool: self.events.append(event) + return True async def _service( From 4bb59656d14a805564d48c8afc43ffdae17b948c Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Sat, 17 Oct 2026 02:17:22 -0500 Subject: [PATCH 34/57] Docs: the reconciler's configuration-refusal alert, and what each step-up leg refuses D3b carry-over for BACKLOG #3010 (no row of its own). CONFIGURATION.md's alert-resolution section names the new ad_reconcile_aborted instance, reason directory_misconfigured, source directory-reconciler-misconfigured, and says nothing resolves it automatically. Its ad_session_revoke_max row and the settings.py breaker comment stop saying a deleted or moved search base reads "not found": that base ends the search with a result code, which is a configuration refusal. SECURITY.md and changelog.d/3010 stop claiming every step-up refuses such an account as misconfigured: the second-factor and passkey step-ups do, the password step-up (_reauth_ad) refuses it as directory_unavailable. --- changelog.d/3010.security.md | 4 +++- docs/CONFIGURATION.md | 10 +++++++++- docs/SECURITY.md | 4 +++- messagefoundry/config/settings.py | 10 +++++++--- 4 files changed, 22 insertions(+), 6 deletions(-) diff --git a/changelog.d/3010.security.md b/changelog.d/3010.security.md index 5dcc53cec5..ab44d01ceb 100644 --- a/changelog.d/3010.security.md +++ b/changelog.d/3010.security.md @@ -17,7 +17,9 @@ `directory_misconfigured`. Before, a deleted search base read as "not found" for every account, which tripped the mass-revoke breaker on a large enough estate. Nothing resolves the new alert automatically; an operator does. Any other code is an outage, which never revokes and pages - nobody. Step-up refuses a refused account with outcome `misconfigured`. + nobody. A second-factor step-up or a passkey assertion, which asks the directory before it + grants, refuses a refused account with outcome `misconfigured`. The password step-up, a live + re-bind, refuses it as `directory_unavailable`, as it does any directory error. **Not covered:** ldap3 fetches a ranged `memberOf` with follow-up searches of its own, and those are not refused. **The cost:** a code that recurs on every pass, such as a deleted search base, stops revocation diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index 7d00f34439..2af05f9d01 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -758,7 +758,7 @@ document ([SECURITY-DOCS-POLICY.md](SECURITY-DOCS-POLICY.md)). | `ad_session_recheck_seconds` | int | `300` | **Directory session reconciliation** ([ADR 0079](adr/0079-kerberos-idp-session-coordination.md) mechanism 2). How often to re-resolve directory principals holding **live** sessions and revoke those AD has disabled or deleted — without it, an AD disable does not take effect until the `[security].max_session_hours` cap (12 h). **`300` (five minutes) is the default** (ADR 0148 GIVEN 1 — the hardened path is the shipped path), floored at **60 s** (a pass costs one LDAP bind per signed-in directory user). `0` disables the loop and is a **loosening** once AD is on — `security_loosenings()` names it. The default is **inert without AD** (`should_reconcile()` also needs an LDAP client), so a non-AD deployment is unaffected; an **explicit** non-zero value without `ad_enabled` is still refused rather than left silently dead. | | `ad_session_recheck_strikes` | int | `2` | Consecutive passes a principal must fail to resolve before its sessions are revoked. *The search matched nothing* cannot tell *deleted* from *moved out of the search base*, so a single ambiguous result must never revoke; a set disabled bit and an unreadable `userAccountControl` strike the same way. A wave of unreadable answers is held instead of revoked, with no setting ([ADR 0195](adr/0195-brake-the-ad-session-reconciler-on-an-undetermined-useraccountcontrol-wave.md)). Range 1–10. | | `ad_session_recheck_max_users` | int | `200` | Per-pass bind budget. Beyond this, remaining users are picked up by later passes (least-recently-probed first), so a large estate degrades to a longer effective interval instead of a bind storm. | -| `ad_session_revoke_max` | int | `5` | **Mass-revoke circuit breaker**, absolute half. A bad search base / moved OU / service account that lost read rights on the entries answers "not found" for *every* user — indistinguishable from "everyone was deleted". A lost read right on `userAccountControl` alone is held instead ([ADR 0195](adr/0195-brake-the-ad-session-reconciler-on-an-undetermined-useraccountcontrol-wave.md)). | +| `ad_session_revoke_max` | int | `5` | **Mass-revoke circuit breaker**, absolute half. A search base that names the wrong OU, an OU reorganisation that moves the users out of it, or a service account that lost read rights on the entries while it can still read the base answers "not found" for *every* user — indistinguishable from "everyone was deleted". A base that was deleted or moved, or that the service account may not read at all, usually ends the search with a result code instead: a configuration refusal, which judges no account and raises its own `ad_reconcile_aborted` instance with reason `directory_misconfigured` (BACKLOG #3010; [SECURITY.md](SECURITY.md)). A lost read right on `userAccountControl` alone is held instead ([ADR 0195](adr/0195-brake-the-ad-session-reconciler-on-an-undetermined-useraccountcontrol-wave.md)). | | `ad_session_revoke_max_fraction` | float | `0.34` | Circuit breaker, proportional half. A pass exceeding **both** thresholds aborts, revokes nothing, logs at ERROR and writes an `auth.ad_reconcile_aborted` audit row. Requiring both means it fires only on a change simultaneously *large* and *broad* — the signature of a misconfiguration, not of offboarding (3-of-3 or 50-of-300 still applies). Range >0.0–1.0; `1.0` disables the proportional half. | | `kerberos_enabled` | bool | `false` | Windows SSO (experimental, off by default; needs `ad_enabled`) | | `kerberos_spn` | str | — | service principal, exactly `SERVICE/host`, e.g. `HTTP/host.example.com`. The engine splits it at the `/` into pyspnego's `service` and `hostname`. A malformed value is **refused at load**, including at least one with no `/`, more than one `/`, an empty half, whitespace or control characters, or a realm suffix (`HTTP/host@REALM`); the host's own domain supplies the realm. | @@ -851,6 +851,14 @@ source `directory-reconciler-probe-fault` (BACKLOG #3100). The reconciler raises which a probe fails to read an entry. Nothing resolves it automatically. Resolve it by hand once the WARNING lines that name each faulting account stop. +A configuration refusal opens its own `ad_reconcile_aborted` instance as well, with reason +`directory_misconfigured` and source `directory-reconciler-misconfigured` (BACKLOG #3010). The +reconciler raises it on every pass in which the directory refuses a search with noSuchObject, +invalidDNSyntax or insufficientAccessRights, as a deleted or moved search base does. Before #3010 +such a base read as "not found" for every account. Nothing resolves it automatically either. A +search base changes only on a restart, so resolve it by hand once the base is fixed and the ERROR +line naming the code stops. + A pass judges only accounts that hold a session, and nothing reads an account again once it has left. An account leaves at least when it signs out, reaches the session cap, is revoked by the reconciler, is disabled locally, or is deleted. The give-up rules above stop an alert resolving diff --git a/docs/SECURITY.md b/docs/SECURITY.md index 5c30d3569b..a5c9af7c12 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -4261,7 +4261,9 @@ probe is refused, faulted or fails, at least one refused and none referred, the Like the fault's, that alert has no inverse; an operator resolves it. **The cost:** while the refusal lasts, the refused accounts' sessions are never revoked by the reconciler, so they run to the absolute cap. Other result codes that are not an answer, such as busy or unavailable, are an -outage. Step-up refuses the account with outcome `misconfigured`. +outage. The step-up legs that ask the directory before they grant, a second-factor step-up and a +passkey assertion, refuse the account with outcome `misconfigured`. The password step-up, a live +re-bind, refuses it as `directory_unavailable`, the reason it gives for any directory error. ### The engine session is independent of the IdP session (ASVS 7.6.1) diff --git a/messagefoundry/config/settings.py b/messagefoundry/config/settings.py index 37ab6f318a..a8819e9a44 100644 --- a/messagefoundry/config/settings.py +++ b/messagefoundry/config/settings.py @@ -3184,9 +3184,13 @@ class AuthSettings(_Section): # effective interval instead of a bind storm against the DC. ad_session_recheck_max_users: int = 200 # --- mass-revoke circuit breaker --- - # A misconfigured search base, a moved OU, or a service account that lost read rights on the - # entries returns "not found" for EVERY user — indistinguishable from "everyone was deleted". - # Without a brake the reconciler would sign out the entire estate during exactly the incident when + # A search base that names the wrong OU, an OU reorganisation that moves the users out of it, or + # a service account that lost read rights on the entries while it can still read the base returns + # "not found" for EVERY user — indistinguishable from "everyone was deleted". A base that was + # deleted or moved, or that the service account may not read at all, usually ends the search with + # a result code instead (noSuchObject, invalidDNSyntax, insufficientAccessRights): a configuration + # refusal, which judges no account and pages as `ad_reconcile_aborted` with reason + # `directory_misconfigured` rather than reaching this breaker (BACKLOG #3010). Without a brake the reconciler would sign out the entire estate during exactly the incident when # operators need the console. A pass that would revoke more than BOTH of these thresholds aborts, # revokes nothing, and raises a loud operator-visible alert (log ERROR + an # `auth.ad_reconcile_aborted` audit row). From 00cb6f591402052d19d6bb325719efada37adb48 Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Sat, 17 Oct 2026 02:36:14 -0500 Subject: [PATCH 35/57] Review round one on D4: Postgres keys a prefix of the actor, honest drain and step-up docs (BACKLOG #3120) Code review at xhigh. Postgres's actor is unbounded TEXT, and a btree entry past about 2.7 KB fails its INSERT, so a whole-value index would turn an overlong typed username into a failed audit write. ix_audit_actor there keys left(actor, 200), and _audit_where filters on that prefix and then on the whole value; a new test writes an 8000-character actor and reads it back on SQLite and Postgres. The SQLite plan test captures the statement list_audit really runs. The drain comment lists at least the bounded drains and names the unbounded ones, notifier aclose included. The offline MFA notice reports what notify returned. The step-up sentence in SECURITY.md and changelog.d/3010 names the federated step-up too (carries #3010 work). Stale "no actor index" claims, the external-schema-management upgrade path and the full-queue mailed: false bullet are corrected. A settings.py comment line is rewrapped. --- changelog.d/3010.security.md | 6 ++-- changelog.d/3120.security.md | 9 +++-- docs/SECURITY.md | 15 ++++---- messagefoundry/__main__.py | 5 +-- messagefoundry/auth/service.py | 17 +++++---- messagefoundry/config/settings.py | 5 +-- messagefoundry/store/postgres.py | 14 ++++++-- messagefoundry/store/store.py | 2 +- tests/test_audit_actor_index.py | 40 +++++++++++++++++----- tests/test_lock_notice_off_request_path.py | 5 ++- 10 files changed, 80 insertions(+), 38 deletions(-) diff --git a/changelog.d/3010.security.md b/changelog.d/3010.security.md index ab44d01ceb..f128dc30d0 100644 --- a/changelog.d/3010.security.md +++ b/changelog.d/3010.security.md @@ -17,9 +17,9 @@ `directory_misconfigured`. Before, a deleted search base read as "not found" for every account, which tripped the mass-revoke breaker on a large enough estate. Nothing resolves the new alert automatically; an operator does. Any other code is an outage, which never revokes and pages - nobody. A second-factor step-up or a passkey assertion, which asks the directory before it - grants, refuses a refused account with outcome `misconfigured`. The password step-up, a live - re-bind, refuses it as `directory_unavailable`, as it does any directory error. + nobody. A second-factor step-up or a passkey assertion, which probes the directory before it + grants, refuses a refused account with outcome `misconfigured`. The password step-up and the + federated step-up refuse it as `directory_unavailable`, as they do any directory error. **Not covered:** ldap3 fetches a ranged `memberOf` with follow-up searches of its own, and those are not refused. **The cost:** a code that recurs on every pass, such as a deleted search base, stops revocation diff --git a/changelog.d/3120.security.md b/changelog.d/3120.security.md index c5b27b8d9e..444bad2a62 100644 --- a/changelog.d/3120.security.md +++ b/changelog.d/3120.security.md @@ -3,9 +3,12 @@ `mailed: false` for one it dropped, so the next lock of the kind mails again rather than staying quiet for 24 hours. A send that fails later, on the relay's own task, still leaves `mailed: true`; `docs/SECURITY.md` states that residual. - **The lock notice's throttle read seeks an index.** A new `ix_audit_actor` index on + **The lock notice's throttle read has an index.** A new `ix_audit_actor` index on `audit_log(actor, action, id)`, on SQLite, Postgres and SQL Server, lets the read find one - account's rows instead of walking every recent notice row of every account. On the server - backends the schema change moves the ADR 0064 schema hash, so the next open runs the DDL. + account's rows instead of walking every recent notice row of every account. On Postgres it keys + the first 200 characters of the actor, since a whole overlong value would fail the audit write. + On the server backends the schema change moves the ADR 0064 schema hash: under their default + `[store].schema_management`, `external`, the engine refuses to open until `provision-schema` has + run; under `auto` the next open runs the DDL. **Declined, with residuals in `docs/SECURITY.md`:** two mails across engine shards serving their own API ports, the timing of the content-free failure lines, and a cap on pending notice tasks. diff --git a/docs/SECURITY.md b/docs/SECURITY.md index a5c9af7c12..744248ab1c 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -4261,9 +4261,10 @@ probe is refused, faulted or fails, at least one refused and none referred, the Like the fault's, that alert has no inverse; an operator resolves it. **The cost:** while the refusal lasts, the refused accounts' sessions are never revoked by the reconciler, so they run to the absolute cap. Other result codes that are not an answer, such as busy or unavailable, are an -outage. The step-up legs that ask the directory before they grant, a second-factor step-up and a -passkey assertion, refuse the account with outcome `misconfigured`. The password step-up, a live -re-bind, refuses it as `directory_unavailable`, the reason it gives for any directory error. +outage. A second-factor step-up and a passkey assertion, which probe the directory before they +grant, refuse the account with outcome `misconfigured`. At least two other step-up legs refuse it +as `directory_unavailable`, the reason each gives for any directory error: the password step-up, a +live re-bind, and the federated step-up's directory lookup. ### The engine session is independent of the IdP session (ASVS 7.6.1) @@ -4981,8 +4982,9 @@ not push a refused sign-in past its padded slot. The task is queued per account one API process. Two locks of one kind landing together there would send one mail. **One throttle read runs at a time in each API process, across every account**, so locks on many accounts at once cannot fill the store's read pool that sign-ins wait on; each mail then waits behind the reads -queued before it. The read seeks the account's own rows on an `(actor, action, id)` index, on every -backend (vault BACKLOG #3120). The residuals are listed below, under *Declined, each with its +queued before it. Every backend carries an `(actor, action, id)` index for that read, keyed on a +prefix of the actor on Postgres (vault BACKLOG #3120). That the read seeks it is measured on SQLite +only; on Postgres and SQL Server the planner's choice is not tested. The residuals are listed below, under *Declined, each with its residual*. A sign-in lock notice on a TOTP-enrolled local account tells the owner to sign in with the password and the code together; a second-step notice says which factor was right and, **if the @@ -5353,7 +5355,8 @@ and header are that route's. An administrator can still read each case in the audit trail, with the gaps listed below. No relay (`mailed: false`, `reason: no_notifier`) and no address (`mailed: false`) are on the `auth.lock_notice` row. A full queue and a failed send are on an `auth.lock_notice_undelivered` - row (BACKLOG #2383). A relay that is down still shows, on every other notice kind. An instance with no relay at all is reported at + row (BACKLOG #2383), and a full queue also reads `mailed: false` on the `auth.lock_notice` row + (vault BACKLOG #3120). A relay that is down still shows, on every other notice kind. An instance with no relay at all is reported at startup by the serve gate, except under `[security].enforcement = "warn"` with `[alerts].security_notifications_required = false`. Every other notice kind keeps its per-event line (BACKLOG #1139); none of them fires on a refused sign-in. The list is diff --git a/messagefoundry/__main__.py b/messagefoundry/__main__.py index c2af797ee0..41aeb54d75 100644 --- a/messagefoundry/__main__.py +++ b/messagefoundry/__main__.py @@ -8586,12 +8586,13 @@ async def notice() -> str: return "no_channel" notifier.start() try: - await notifier.notify( + queued = await notifier.notify( SecurityEvent(event_type=MFA_ENABLED, username=user.username, email=address) ) finally: await notifier.aclose() - return "dispatched" + # What the notifier said it took (vault BACKLOG #3120), never a forecast. + return "dispatched" if queued else "no_channel" try: report["holder_notice"] = run_guarded(notice()) diff --git a/messagefoundry/auth/service.py b/messagefoundry/auth/service.py index ccf6c9b2f2..33588e6caa 100644 --- a/messagefoundry/auth/service.py +++ b/messagefoundry/auth/service.py @@ -2033,13 +2033,16 @@ def _idp_error_fields(exc: BaseException) -> dict[str, object]: #: #: The budget is NSSM's 15 s graceful-stop window (``AppStopMethodConsole`` in #: ``scripts/service/install-service.ps1``), not any one drain's constant (vault BACKLOG #3120). The -#: lifespan teardown in ``api/app.py`` spends it in order, after uvicorn's own connection drain: -#: the upload runner's stop (``uploads._DEFAULT_STOP_TIMEOUT_SECONDS``, 5 s), the long-operation -#: drain (``api/outlive.py``, 0.5 s plus ``CANCEL_GRACE_SECONDS``, 6 s), the approval gate's drain -#: (``api/approvals.DRAIN_TIMEOUT_SECONDS``, 3 s), then this one (2.0 s plus 0.5 s), and -#: ``engine.stop()`` after them all. Those bounds add up to 17 s, past the window, so the -#: teardown fits it only while they are not all busy at once; uvicorn's drain has no bound at all. -#: A change to any of them is read against that sum. +#: lifespan teardown in ``api/app.py`` spends it, after uvicorn's own connection drain, on at least +#: these drains with a named bound, in order: the upload runner's stop +#: (``uploads._DEFAULT_STOP_TIMEOUT_SECONDS``, 5 s), the long-operation drain (``api/outlive.py``, +#: 0.5 s plus ``CANCEL_GRACE_SECONDS``, 6 s), the approval gate's drain +#: (``api/approvals.DRAIN_TIMEOUT_SECONDS``, 3 s), then this one (2.0 s plus 0.5 s). Those alone add +#: up to 17 s, past the window when all are busy at once. The rest of the teardown has no shared +#: bound: uvicorn's drain, the summary-access flush, ``engine.stop()``, and the security notifier's +#: and alert notifier's ``aclose()``, each of which drains its queue at the SMTP or transport +#: timeout per item. So nothing here makes the teardown fit the window; a change to any bound is +#: read against this list. _BACKGROUND_DRAIN_SECONDS: Final = 2.0 _BACKGROUND_CANCEL_GRACE_SECONDS: Final = 0.5 diff --git a/messagefoundry/config/settings.py b/messagefoundry/config/settings.py index a8819e9a44..06ec085554 100644 --- a/messagefoundry/config/settings.py +++ b/messagefoundry/config/settings.py @@ -3190,8 +3190,9 @@ class AuthSettings(_Section): # deleted or moved, or that the service account may not read at all, usually ends the search with # a result code instead (noSuchObject, invalidDNSyntax, insufficientAccessRights): a configuration # refusal, which judges no account and pages as `ad_reconcile_aborted` with reason - # `directory_misconfigured` rather than reaching this breaker (BACKLOG #3010). Without a brake the reconciler would sign out the entire estate during exactly the incident when - # operators need the console. A pass that would revoke more than BOTH of these thresholds aborts, + # `directory_misconfigured` rather than reaching this breaker (BACKLOG #3010). Without a brake + # the reconciler would sign out the entire estate during exactly the incident when operators need + # the console. A pass that would revoke more than BOTH of these thresholds aborts, # revokes nothing, and raises a loud operator-visible alert (log ERROR + an # `auth.ad_reconcile_aborted` audit row). # diff --git a/messagefoundry/store/postgres.py b/messagefoundry/store/postgres.py index 7156cd39fc..a6cdaf3b74 100644 --- a/messagefoundry/store/postgres.py +++ b/messagefoundry/store/postgres.py @@ -721,8 +721,13 @@ def __init__(self, method: str, outbox_ids: tuple[str, ...]) -> None: "CREATE INDEX IF NOT EXISTS ix_audit_ts ON audit_log(ts)", # recent_audit_of: the start's config baseline read (vault BACKLOG #2597). "CREATE INDEX IF NOT EXISTS ix_audit_action ON audit_log(action, id)", - # The lock notice's throttle read, one account's rows of one action (vault BACKLOG #3120). - "CREATE INDEX IF NOT EXISTS ix_audit_actor ON audit_log(actor, action, id)", + # The lock notice's throttle read, one account's rows of one action (vault BACKLOG #3120). Keyed + # on a PREFIX of the actor, because `actor` is unbounded TEXT and a btree entry over about 2.7 KB + # fails the INSERT: a sign-in can audit a typed username of any length, and an index on the whole + # value would turn an overlong one into a failed audit write. `_audit_where` filters on the same + # prefix and then on the whole value, so the planner can seek it. 200 characters is at most + # 800 bytes. + "CREATE INDEX IF NOT EXISTS ix_audit_actor ON audit_log(left(actor, 200), action, id)", # No table beside audit_log says where its keying starts: the process that holds the key decides, # and the chain's own row at sequence number 1 (the genesis row) names the first range's key. # Per-key AES-GCM invocation bound (ASVS 11.3.4) — see the SQLite `_SCHEMA` for the @@ -8010,7 +8015,10 @@ def _audit_where( params: list[Any] = [] if actor is not None: params.append(actor) - clauses.append(f"actor = ${len(params)}") + # The prefix term lets the planner seek ix_audit_actor, which is keyed on that prefix + # (vault BACKLOG #3120); the whole-value term is what decides the match. + n = len(params) + clauses.append(f"left(actor, 200) = left(${n}::text, 200) AND actor = ${n}") if action is not None: params.append(action) clauses.append(f"action = ${len(params)}") diff --git a/messagefoundry/store/store.py b/messagefoundry/store/store.py index 404493ffac..ebdda5ce49 100644 --- a/messagefoundry/store/store.py +++ b/messagefoundry/store/store.py @@ -6984,7 +6984,7 @@ def _dead_target_pairs(rows: Iterable[tuple[Any, Any]]) -> list[tuple[str, str]] -- and client host where a sign-in finished owing nothing more, or a step-up or enrolment passed -- (AuthService._mark_login_address_known says when). Keyed on the account id, never the -- username, so a re-created namesake inherits nothing. The primary key serves the signal's one read --- (WHERE user_id), so audit_log, which has no actor index, is never scanned for it. Plaintext like +-- (WHERE user_id), so audit_log is never read for it. Plaintext like -- sessions.client: a network address, not PHI (docs/PHI.md section 2). ON DELETE CASCADE because -- the write is best-effort and may race delete_user: a row inserted just before the account's -- delete goes with it, and one inserted after fails the key and is dropped by the caller. diff --git a/tests/test_audit_actor_index.py b/tests/test_audit_actor_index.py index 328375ea38..8dae075e3a 100644 --- a/tests/test_audit_actor_index.py +++ b/tests/test_audit_actor_index.py @@ -5,7 +5,9 @@ The lock notice's 24-hour throttle reads one account's ``auth.lock_notice`` rows through ``list_audit(actor=, action=, since=, limit=50)``. With only ``ix_audit_ts`` and ``ix_audit_action`` that read walked every recent row of the action, for every account. -``ix_audit_actor`` lets it seek the account's own rows, newest first. +``ix_audit_actor`` lets it seek the account's own rows, newest first. On Postgres it keys a prefix +of the actor, which is unbounded TEXT there. Only the SQLite plan is measured here; on the server +backends this file checks that the index exists, not that the planner picks it. Backend-parametrized like ``test_fifo_index_migration``: the SQLite case runs everywhere; the SQL Server and Postgres cases run only when their ``MEFOR_TEST_*`` env is set, which the hosted service- @@ -15,6 +17,7 @@ from __future__ import annotations import contextlib +import hashlib import os from collections.abc import AsyncIterator, Sequence from pathlib import Path @@ -68,15 +71,15 @@ async def _index_columns(store: Any, name: str) -> list[str]: cur = await db.execute(f"PRAGMA index_info({name})") return [r[2] for r in sorted(await cur.fetchall(), key=lambda r: r[0])] if backend == "postgres": + # pg_get_indexdef per key position, so an expression key reads as its expression. rows = await store._fetchall( - "SELECT a.attname FROM pg_index i" - " JOIN pg_class c ON c.oid = i.indexrelid" - " JOIN LATERAL unnest(i.indkey) WITH ORDINALITY AS k(attnum, ord) ON true" - " JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = k.attnum" + "SELECT pg_get_indexdef(c.oid, k.ord, true) AS col FROM pg_class c" + " JOIN pg_index i ON i.indexrelid = c.oid" + " CROSS JOIN LATERAL generate_series(1, i.indnkeyatts::int) AS k(ord)" " WHERE c.relname = $1 ORDER BY k.ord", name, ) - return [r["attname"] for r in rows] + return [str(r["col"]).replace('"', "").replace(" ", "") for r in rows] rows = await store._fetchall( # sqlserver "SELECT c.name FROM sys.indexes i" " JOIN sys.index_columns ic ON ic.object_id=i.object_id AND ic.index_id=i.index_id" @@ -89,19 +92,40 @@ async def _index_columns(store: Any, name: str) -> list[str]: async def test_the_actor_index_exists_on_every_backend(store: Any) -> None: - assert await _index_columns(store, "ix_audit_actor") == ["actor", "action", "id"] + # Postgres keys a prefix of the unbounded TEXT actor; see the next test. + actor = "left(actor,200)" if store._test_backend == "postgres" else "actor" + assert await _index_columns(store, "ix_audit_actor") == [actor, "action", "id"] # The control: the same probe reads the index that was already there, so an empty list above # would be the index missing, not the probe failing. assert await _index_columns(store, "ix_audit_action") == ["action", "id"] +async def test_an_overlong_actor_is_still_audited_and_found(store: Any) -> None: + """A btree entry over about 2.7 KB fails its INSERT on Postgres, and ``actor`` is unbounded + there. An index on the whole value would turn an overlong typed username into a failed audit + write. The Postgres index keys a prefix, and ``list_audit`` still matches the whole value. + + SQL Server is left out: its ``actor`` column is NVARCHAR(256), which refuses such a row at the + column whatever the index, so the case is not this change's to pin there.""" + if store._test_backend == "sqlserver": + pytest.skip("SQL Server's actor column is NVARCHAR(256)") + # Random hex, which pglz cannot shrink, so compressing the index entry cannot rescue it. + long_actor = "".join(hashlib.sha256(str(i).encode()).hexdigest() for i in range(125)) # 8000 + near = long_actor[:200] + "-another" # same prefix, another actor + await store.record_audit(_LOCK_NOTICE, actor=long_actor, detail='{"lock": "sign_in"}') + await store.record_audit(_LOCK_NOTICE, actor=near, detail='{"lock": "second_step"}') + rows = await store.list_audit(actor=long_actor, action=_LOCK_NOTICE, since=0.0, limit=50) + assert [r["detail"] for r in rows] == ['{"lock": "sign_in"}'] + + async def test_the_throttle_read_seeks_the_actor_index_on_sqlite( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: """The plan of the statement ``list_audit`` runs for the throttle names ``ix_audit_actor``. The statement and its parameters are captured off the read connection as ``list_audit`` - really runs them, not rebuilt here, so the test cannot drift from the method. The control is + really runs them, not rebuilt here, so the SQL cannot drift from the method. The filters are + the ones ``AuthService._lock_notice_held_back`` passes today, written out here. The control is an action-only read, whose plan names ``ix_audit_action``, so the instrument is known to report the index it uses.""" store = await MessageStore.open(tmp_path / "plan.db") diff --git a/tests/test_lock_notice_off_request_path.py b/tests/test_lock_notice_off_request_path.py index 61d6270ec0..015669e265 100644 --- a/tests/test_lock_notice_off_request_path.py +++ b/tests/test_lock_notice_off_request_path.py @@ -530,9 +530,8 @@ async def counted(*args: Any, **kwargs: Any) -> Any: async def test_one_throttle_read_runs_at_a_time_across_accounts( monkeypatch: pytest.MonkeyPatch, ) -> None: - """Each throttle read walks every recent ``auth.lock_notice`` row, so locks on many accounts at - once would fill the store's read pool that sign-ins wait on. The service runs one at a time. - Every account still gets its mail.""" + """Locks on many accounts at once would each take a read from the store's pool that sign-ins + wait on, so the service runs one throttle read at a time. Every account still gets its mail.""" peak, mailed, locked, stalls = await _peak_reads_across_accounts(monkeypatch, slots=None) assert stalls == [], stalls assert peak == 1, f"{peak} throttle reads ran at once" From 8bbae9b21ed47ddec3d522492571365fda72c477 Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Sat, 17 Oct 2026 02:49:25 -0500 Subject: [PATCH 36/57] D4 handover: PR title and banner text (BACKLOG #3120) Proposed PR title: Lock notice follow-ups: a full queue reads mailed false, an actor index for the throttle, drain and residual docs (BACKLOG #3120) Proposed #3120 ledger banner: BUILT on engine branch b257-d4: SecurityNotifier.notify returns whether it queued the event, so a lock notice the full queue dropped writes mailed: false and the next lock mails again (follow-up 2, queue-full path; a later send_failed still reads mailed: true, a residual in SECURITY.md). ix_audit_actor (actor, action, id) on SQLite and SQL Server, left(actor, 200) prefix on Postgres ((d), follow-up 5). Drain comment names NSSM's 15 s window and the drains sharing it (1); the one-read-at-a-time limit is in SECURITY.md (3); test drains in finally (4). (a) cross-shard double mail, (b) failure-log timing and (c) no cap on queued tasks declined in writing, residuals in SECURITY.md. Postgres and SQL Server legs unread at push. Round-two review notes are open in the PR body, among them a SQLite plan regression for actor-only audit reads. Also carries the D3b doc carry-over for #3010 (CONFIGURATION.md alert resolution and revoke-max row, settings.py breaker comment, step-up refusal wording in SECURITY.md and changelog.d/3010). From 239d8faeea355c6743c8075b3d8b3c9eaadf4d27 Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Sat, 17 Oct 2026 02:57:59 -0500 Subject: [PATCH 37/57] Audit actor-only reads get an (actor, id) index back their id-order plan (BACKLOG #3120) D4's ix_audit_actor (actor, action, id) made SQLite plan every audit read filtered by actor without an action equality as a seek on that index plus a temporary B-tree sort of all the actor's rows: GET /audit?actor=, its export pages, /me/security-events. A companion ix_audit_actor_id (actor, id) on all three backends hands those rows over in id order; on Postgres it keys left(actor, 200) like ix_audit_actor, and _security_events_where_pg now filters on that prefix with a whole-value recheck, as _audit_where does. The throttle read keeps its seek on ix_audit_actor (actor=? AND action=?). Measured on SQLite 3.50.4, 600k rows, 300k of one actor, best of 5: actor-only page 0.059 ms (d3b) / 23.1 ms (d4) / 0.053 ms (this); offset 100000 page 8.6 / 233 / 2.9 ms; security-events page 0.058 / 85 / 0.069 ms; throttle 0.057 / 0.086 / 0.057 ms. The uncapped security-events count for that actor is 42.5 / 85 / 53 ms. Also: the _lock_notice_held_back docstring no longer says the seek holds on every backend, and the test's Postgres index probe is restricted to the store's own schema. --- changelog.d/3120.security.md | 3 + docs/SECURITY.md | 7 +- messagefoundry/auth/service.py | 13 +-- messagefoundry/store/postgres.py | 12 ++- messagefoundry/store/sqlserver.py | 5 ++ messagefoundry/store/store.py | 4 + tests/test_audit_actor_index.py | 135 +++++++++++++++++++++--------- 7 files changed, 129 insertions(+), 50 deletions(-) diff --git a/changelog.d/3120.security.md b/changelog.d/3120.security.md index 444bad2a62..45c4968743 100644 --- a/changelog.d/3120.security.md +++ b/changelog.d/3120.security.md @@ -7,6 +7,9 @@ `audit_log(actor, action, id)`, on SQLite, Postgres and SQL Server, lets the read find one account's rows instead of walking every recent notice row of every account. On Postgres it keys the first 200 characters of the actor, since a whole overlong value would fail the audit write. + A companion `ix_audit_actor_id` index on `audit_log(actor, id)`, keyed the same way, keeps the + reads filtered by actor alone (`GET /audit?actor=`, its export pages, `/me/security-events`) + from sorting all of an actor's rows to return the newest few. On the server backends the schema change moves the ADR 0064 schema hash: under their default `[store].schema_management`, `external`, the engine refuses to open until `provision-schema` has run; under `auto` the next open runs the DDL. diff --git a/docs/SECURITY.md b/docs/SECURITY.md index 744248ab1c..c5ac4f2475 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -4983,8 +4983,11 @@ one API process. Two locks of one kind landing together there would send one mai read runs at a time in each API process, across every account**, so locks on many accounts at once cannot fill the store's read pool that sign-ins wait on; each mail then waits behind the reads queued before it. Every backend carries an `(actor, action, id)` index for that read, keyed on a -prefix of the actor on Postgres (vault BACKLOG #3120). That the read seeks it is measured on SQLite -only; on Postgres and SQL Server the planner's choice is not tested. The residuals are listed below, under *Declined, each with its +prefix of the actor on Postgres (vault BACKLOG #3120). A companion `(actor, id)` index, keyed the +same way, serves the audit reads filtered by actor alone (`GET /audit?actor=`, its export pages, +`/me/security-events`); without it SQLite sought the actor on the first index and then sorted all +of its rows. That each read seeks its index is measured on SQLite only; on Postgres and SQL Server +the planner's choice is not tested. The residuals are listed below, under *Declined, each with its residual*. A sign-in lock notice on a TOTP-enrolled local account tells the owner to sign in with the password and the code together; a second-step notice says which factor was right and, **if the diff --git a/messagefoundry/auth/service.py b/messagefoundry/auth/service.py index 33588e6caa..7618e6c85e 100644 --- a/messagefoundry/auth/service.py +++ b/messagefoundry/auth/service.py @@ -13485,12 +13485,13 @@ async def _lock_notice_held_back(self, user: UserRecord, lock: str) -> bool: """The throttle's READ: whether a notice of this ``lock`` kind already went out in the window. Kept apart from the row write, so the background task can send the mail between this read - and :meth:`_record_lock_notice` (BACKLOG #2216). It seeks this account's rows of the - action on the ``ix_audit_actor`` (actor, action, id) index, on every backend (vault - BACKLOG #3120); before it, the read walked every recent ``auth.lock_notice`` row of every - account. One such read still runs at a time per service (``_lock_notice_read_slots``), so - locks on many accounts at once cannot fill the store's read pool, which sign-ins wait on - too. The cost is that each notice waits behind the reads queued before it, across every + and :meth:`_record_lock_notice` (BACKLOG #2216). Every backend carries an + ``ix_audit_actor`` (actor, action, id) index for it, keyed on a prefix of the actor on + Postgres (vault BACKLOG #3120). That the read seeks it is measured on SQLite only; on + Postgres and SQL Server the planner's choice is not tested. Without it the read walked + every recent ``auth.lock_notice`` row of every account. One such read still runs at a time + per service (``_lock_notice_read_slots``), so locks on many accounts at once cannot fill the + store's read pool, which sign-ins wait on too. The cost is that each notice waits behind the reads queued before it, across every account in this process; ``docs/SECURITY.md`` states it. A failed read fails OPEN (``False``) and is logged by exception class alone. The line names diff --git a/messagefoundry/store/postgres.py b/messagefoundry/store/postgres.py index a6cdaf3b74..5c01635a95 100644 --- a/messagefoundry/store/postgres.py +++ b/messagefoundry/store/postgres.py @@ -728,6 +728,11 @@ def __init__(self, method: str, outbox_ids: tuple[str, ...]) -> None: # prefix and then on the whole value, so the planner can seek it. 200 characters is at most # 800 bytes. "CREATE INDEX IF NOT EXISTS ix_audit_actor ON audit_log(left(actor, 200), action, id)", + # One account's rows of any action, newest first: GET /audit?actor=, its export pages and + # /me/security-events (vault BACKLOG #3120). The SQLite `_SCHEMA` says why ix_audit_actor alone + # is not enough there. The same actor prefix, for the same reason; `_audit_where` and + # `_security_events_where_pg` both filter on it. + "CREATE INDEX IF NOT EXISTS ix_audit_actor_id ON audit_log(left(actor, 200), id)", # No table beside audit_log says where its keying starts: the process that holds the key decides, # and the chain's own row at sequence number 1 (the genesis row) names the first range's key. # Per-key AES-GCM invocation bound (ASVS 11.3.4) — see the SQLite `_SCHEMA` for the @@ -10431,10 +10436,13 @@ def _pg_cutoff_case( def _security_events_where_pg(username: str, until: float | None) -> tuple[str, list[Any]]: """The ``$N`` ``WHERE`` text and values for one user's security-event page and its total (BACKLOG #2438), shared so the two count one set. The SQLite builder states the rule, including - the lower bound at the account's ``created_at`` (BACKLOG #1152).""" + the lower bound at the account's ``created_at`` (BACKLOG #1152). + + The actor prefix term lets the planner seek ``ix_audit_actor_id``, which is keyed on that + prefix, as ``_audit_where`` does (vault BACKLOG #3120); the whole-value term decides the match.""" params: list[Any] = [username] clause = ( - " WHERE actor = $1 AND action LIKE 'auth.%'" + " WHERE left(actor, 200) = left($1::text, 200) AND actor = $1 AND action LIKE 'auth.%'" " AND ts >= (SELECT created_at FROM users WHERE username = $1)" ) if until is not None: diff --git a/messagefoundry/store/sqlserver.py b/messagefoundry/store/sqlserver.py index 08c0dbba6f..fb8cd6ec8a 100644 --- a/messagefoundry/store/sqlserver.py +++ b/messagefoundry/store/sqlserver.py @@ -2102,6 +2102,11 @@ def __init__(self, conn: Any, cur: Any) -> None: # key is at most 772 bytes (actor 512, action 256, id 4), inside the 1700-byte nonclustered limit. """IF INDEXPROPERTY(OBJECT_ID('audit_log'),'ix_audit_actor','IndexID') IS NULL CREATE INDEX ix_audit_actor ON audit_log(actor, action, id)""", + # One account's rows of any action, newest first: GET /audit?actor=, its export pages and + # /me/security-events (vault BACKLOG #3120). The SQLite `_SCHEMA` says why ix_audit_actor alone + # is not enough there. The key is at most 516 bytes (actor 512, id 4). + """IF INDEXPROPERTY(OBJECT_ID('audit_log'),'ix_audit_actor_id','IndexID') IS NULL + CREATE INDEX ix_audit_actor_id ON audit_log(actor, id)""", # No table beside audit_log says where its keying starts: the process that holds the key decides, # and the chain's own row at sequence number 1 (the genesis row) names the first range's key. # Per-key AES-GCM invocation bound (ASVS 11.3.4) — see the SQLite `_SCHEMA` for the diff --git a/messagefoundry/store/store.py b/messagefoundry/store/store.py index ebdda5ce49..e79110fc4a 100644 --- a/messagefoundry/store/store.py +++ b/messagefoundry/store/store.py @@ -6752,6 +6752,10 @@ def _dead_target_pairs(rows: Iterable[tuple[Any, Any]]) -> list[tuple[str, str]] -- One account's rows of one action, newest first: the lock notice's 24-hour throttle read, vault -- BACKLOG #3120. Without it that read walks every recent row of the action, for every account. CREATE INDEX IF NOT EXISTS ix_audit_actor ON audit_log(actor, action, id); +-- One account's rows of any action, newest first: GET /audit?actor=, its export pages and +-- /me/security-events, vault BACKLOG #3120. ix_audit_actor alone made the planner seek the actor +-- there and then sort every one of its rows to read the newest few. +CREATE INDEX IF NOT EXISTS ix_audit_actor_id ON audit_log(actor, id); -- There is no table beside audit_log that says where its keying starts. Whether a chain is keyed is -- decided by the process that holds the key, and the first range's key is named by the chain's own diff --git a/tests/test_audit_actor_index.py b/tests/test_audit_actor_index.py index 8dae075e3a..056d3ded38 100644 --- a/tests/test_audit_actor_index.py +++ b/tests/test_audit_actor_index.py @@ -1,13 +1,18 @@ # SPDX-License-Identifier: AGPL-3.0-or-later # Copyright (C) 2026 MessageFoundry Foundation, LLC and contributors -"""Vault BACKLOG #3120, item (d) and review follow-up 5: an (actor, action, id) audit index. +"""Vault BACKLOG #3120, item (d) and review follow-up 5: the audit actor indexes. The lock notice's 24-hour throttle reads one account's ``auth.lock_notice`` rows through ``list_audit(actor=, action=, since=, limit=50)``. With only ``ix_audit_ts`` and ``ix_audit_action`` that read walked every recent row of the action, for every account. -``ix_audit_actor`` lets it seek the account's own rows, newest first. On Postgres it keys a prefix -of the actor, which is unbounded TEXT there. Only the SQLite plan is measured here; on the server -backends this file checks that the index exists, not that the planner picks it. +``ix_audit_actor`` (actor, action, id) lets it seek the account's own rows, newest first. + +That index alone regressed the reads filtered by actor WITHOUT an action equality (``GET +/audit?actor=``, its export pages, ``/me/security-events``): SQLite sought the actor on it and then +sorted every one of the actor's rows to return the newest few. ``ix_audit_actor_id`` (actor, id) +serves those reads in id order. On Postgres both key a prefix of the actor, which is unbounded TEXT +there. Only the SQLite plans are measured here; on the server backends this file checks that the +indexes exist, not that the planner picks them. Backend-parametrized like ``test_fifo_index_migration``: the SQLite case runs everywhere; the SQL Server and Postgres cases run only when their ``MEFOR_TEST_*`` env is set, which the hosted service- @@ -19,7 +24,7 @@ import contextlib import hashlib import os -from collections.abc import AsyncIterator, Sequence +from collections.abc import AsyncIterator, Awaitable, Callable, Sequence from pathlib import Path from typing import Any @@ -71,12 +76,14 @@ async def _index_columns(store: Any, name: str) -> list[str]: cur = await db.execute(f"PRAGMA index_info({name})") return [r[2] for r in sorted(await cur.fetchall(), key=lambda r: r[0])] if backend == "postgres": - # pg_get_indexdef per key position, so an expression key reads as its expression. + # pg_get_indexdef per key position, so an expression key reads as its expression. The + # store's own schema only, so a same-named index in another schema cannot answer for it. rows = await store._fetchall( "SELECT pg_get_indexdef(c.oid, k.ord, true) AS col FROM pg_class c" " JOIN pg_index i ON i.indexrelid = c.oid" " CROSS JOIN LATERAL generate_series(1, i.indnkeyatts::int) AS k(ord)" - " WHERE c.relname = $1 ORDER BY k.ord", + " WHERE c.relname = $1 AND c.relnamespace = current_schema()::regnamespace" + " ORDER BY k.ord", name, ) return [str(r["col"]).replace('"', "").replace(" ", "") for r in rows] @@ -91,10 +98,11 @@ async def _index_columns(store: Any, name: str) -> list[str]: return [r["name"] for r in rows] -async def test_the_actor_index_exists_on_every_backend(store: Any) -> None: +async def test_the_actor_indexes_exist_on_every_backend(store: Any) -> None: # Postgres keys a prefix of the unbounded TEXT actor; see the next test. actor = "left(actor,200)" if store._test_backend == "postgres" else "actor" assert await _index_columns(store, "ix_audit_actor") == [actor, "action", "id"] + assert await _index_columns(store, "ix_audit_actor_id") == [actor, "id"] # The control: the same probe reads the index that was already there, so an empty list above # would be the index missing, not the probe failing. assert await _index_columns(store, "ix_audit_action") == ["action", "id"] @@ -103,7 +111,8 @@ async def test_the_actor_index_exists_on_every_backend(store: Any) -> None: async def test_an_overlong_actor_is_still_audited_and_found(store: Any) -> None: """A btree entry over about 2.7 KB fails its INSERT on Postgres, and ``actor`` is unbounded there. An index on the whole value would turn an overlong typed username into a failed audit - write. The Postgres index keys a prefix, and ``list_audit`` still matches the whole value. + write. The Postgres indexes key a prefix, and ``list_audit`` still matches the whole value, + with or without an action filter. SQL Server is left out: its ``actor`` column is NVARCHAR(256), which refuses such a row at the column whatever the index, so the case is not this change's to pin there.""" @@ -116,49 +125,95 @@ async def test_an_overlong_actor_is_still_audited_and_found(store: Any) -> None: await store.record_audit(_LOCK_NOTICE, actor=near, detail='{"lock": "second_step"}') rows = await store.list_audit(actor=long_actor, action=_LOCK_NOTICE, since=0.0, limit=50) assert [r["detail"] for r in rows] == ['{"lock": "sign_in"}'] + rows = await store.list_audit(actor=long_actor, since=0.0, limit=50) + assert [r["detail"] for r in rows] == ['{"lock": "sign_in"}'] + + +async def _plan( + store: MessageStore, + monkeypatch: pytest.MonkeyPatch, + call: Callable[[MessageStore], Awaitable[object]], +) -> str: + """The SQLite plan of the one statement ``call`` runs on the store's read connection. + + The statement and its parameters are captured off the read connection as the store method + really runs them, not rebuilt here, so the SQL cannot drift from the method.""" + real_read = store._read + seen: list[tuple[str, Sequence[Any]]] = [] + + class _Spy: + def __init__(self, db: Any) -> None: + self._db = db + + async def execute(self, sql: str, params: Sequence[Any] = ()) -> Any: + seen.append((sql, params)) + return await self._db.execute(sql, params) + + @contextlib.asynccontextmanager + async def spy_read() -> AsyncIterator[_Spy]: + async with real_read() as db: + yield _Spy(db) + + with monkeypatch.context() as m: + m.setattr(store, "_read", spy_read) + await call(store) + [(sql, params)] = seen + async with real_read() as db: + cur = await db.execute(f"EXPLAIN QUERY PLAN {sql}", params) + return " | ".join(str(r[-1]) for r in await cur.fetchall()) async def test_the_throttle_read_seeks_the_actor_index_on_sqlite( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - """The plan of the statement ``list_audit`` runs for the throttle names ``ix_audit_actor``. + """The plan of the statement ``list_audit`` runs for the throttle seeks ``ix_audit_actor`` on + both the actor and the action, not ``ix_audit_actor_id`` on the actor alone. - The statement and its parameters are captured off the read connection as ``list_audit`` - really runs them, not rebuilt here, so the SQL cannot drift from the method. The filters are - the ones ``AuthService._lock_notice_held_back`` passes today, written out here. The control is - an action-only read, whose plan names ``ix_audit_action``, so the instrument is known to report - the index it uses.""" + The filters are the ones ``AuthService._lock_notice_held_back`` passes today, written out here. + The control is an action-only read, whose plan names ``ix_audit_action``, so the instrument is + known to report the index it uses.""" store = await MessageStore.open(tmp_path / "plan.db") - real_read = store._read - - async def plan(**filters: Any) -> str: - seen: list[tuple[str, Sequence[Any]]] = [] + try: + throttle = await _plan( + store, + monkeypatch, + lambda s: s.list_audit(actor="owner", action=_LOCK_NOTICE, since=0.0, limit=50), + ) + control = await _plan(store, monkeypatch, lambda s: s.list_audit(action=_LOCK_NOTICE)) + finally: + await store.close() + assert "ix_audit_actor (actor=? AND action=?)" in throttle, throttle + assert "TEMP B-TREE" not in throttle, throttle + assert "ix_audit_action" in control, control - class _Spy: - def __init__(self, db: Any) -> None: - self._db = db - async def execute(self, sql: str, params: Sequence[Any] = ()) -> Any: - seen.append((sql, params)) - return await self._db.execute(sql, params) +# Every read filtered by actor without an action equality: the audit page and its total, the export +# pages (a keyset cursor, or an offset), a time-bounded read, and the self-service security events. +_ACTOR_ONLY_READS: dict[str, Callable[[MessageStore], Awaitable[object]]] = { + "list_audit": lambda s: s.list_audit(actor="owner", limit=50), + "list_audit_offset": lambda s: s.list_audit(actor="owner", limit=50, offset=500), + "list_audit_before_id": lambda s: s.list_audit(actor="owner", limit=50, before_id=1000), + "list_audit_since": lambda s: s.list_audit(actor="owner", since=0.0, until=9e9, limit=50), + "count_audit_capped": lambda s: s.count_audit(actor="owner", limit=1000), + "security_events": lambda s: s.security_events_for_user("owner", limit=100, until=9e9), +} - @contextlib.asynccontextmanager - async def spy_read() -> AsyncIterator[_Spy]: - async with real_read() as db: - yield _Spy(db) - with monkeypatch.context() as m: - m.setattr(store, "_read", spy_read) - await store.list_audit(limit=50, **filters) - [(sql, params)] = seen - async with real_read() as db: - cur = await db.execute(f"EXPLAIN QUERY PLAN {sql}", params) - return " | ".join(str(r[-1]) for r in await cur.fetchall()) +@pytest.mark.parametrize("read", sorted(_ACTOR_ONLY_READS)) +async def test_an_actor_only_read_walks_the_actor_in_id_order_on_sqlite( + read: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """With ``ix_audit_actor`` alone, SQLite sought the actor on it and then sorted all of the + actor's rows into a temporary B-tree to read the newest few: about 45 ms for a 300k-row actor + against 0.05 ms before it, and an export paging by offset re-sorted every page. ``(actor, id)`` + hands the rows over already in ``id`` order. + An empty database plans like a full one here, because SQLite has no statistics until ANALYZE + and the store never runs it.""" + store = await MessageStore.open(tmp_path / "plan.db") try: - throttle = await plan(actor="owner", action=_LOCK_NOTICE, since=0.0) - control = await plan(action=_LOCK_NOTICE) + plan = await _plan(store, monkeypatch, _ACTOR_ONLY_READS[read]) finally: await store.close() - assert "ix_audit_actor" in throttle, throttle - assert "ix_audit_action" in control, control + assert "ix_audit_actor_id (actor=?" in plan, plan + assert "TEMP B-TREE" not in plan, plan From 8d37f34f092a0310d52e847d0cdb1e5133a6fcfd Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Sat, 17 Oct 2026 03:13:15 -0500 Subject: [PATCH 38/57] Review round one on D4b: schema-exact Postgres probe, near-prefix security events test (BACKLOG #3120) The Postgres index probe compares pg_namespace.nspname to current_schema() rather than casting to regnamespace, which case-folds a mixed-case schema. A new test gives two accounts names sharing the 200-character indexed prefix and checks /me/security-events and its count keep them apart on every backend. The actor-only plan pins add an exclusion-bearing read and are worded as a floor; the test docstring quotes the measured numbers. The lock-notice docstring links to docs/SECURITY.md instead of restating the index, and the docs say "without an action equality" rather than "by actor alone". --- changelog.d/3120.security.md | 3 ++- docs/SECURITY.md | 6 ++--- messagefoundry/auth/service.py | 14 +++++------ messagefoundry/store/postgres.py | 4 +-- tests/test_audit_actor_index.py | 43 +++++++++++++++++++++++++++----- 5 files changed, 50 insertions(+), 20 deletions(-) diff --git a/changelog.d/3120.security.md b/changelog.d/3120.security.md index 45c4968743..caa79d5e4a 100644 --- a/changelog.d/3120.security.md +++ b/changelog.d/3120.security.md @@ -8,7 +8,8 @@ account's rows instead of walking every recent notice row of every account. On Postgres it keys the first 200 characters of the actor, since a whole overlong value would fail the audit write. A companion `ix_audit_actor_id` index on `audit_log(actor, id)`, keyed the same way, keeps the - reads filtered by actor alone (`GET /audit?actor=`, its export pages, `/me/security-events`) + reads filtered by actor without an action equality (`GET /audit?actor=`, its export pages, + `/me/security-events`) from sorting all of an actor's rows to return the newest few. On the server backends the schema change moves the ADR 0064 schema hash: under their default `[store].schema_management`, `external`, the engine refuses to open until `provision-schema` has diff --git a/docs/SECURITY.md b/docs/SECURITY.md index c5ac4f2475..336321537e 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -4984,9 +4984,9 @@ read runs at a time in each API process, across every account**, so locks on man cannot fill the store's read pool that sign-ins wait on; each mail then waits behind the reads queued before it. Every backend carries an `(actor, action, id)` index for that read, keyed on a prefix of the actor on Postgres (vault BACKLOG #3120). A companion `(actor, id)` index, keyed the -same way, serves the audit reads filtered by actor alone (`GET /audit?actor=`, its export pages, -`/me/security-events`); without it SQLite sought the actor on the first index and then sorted all -of its rows. That each read seeks its index is measured on SQLite only; on Postgres and SQL Server +same way, serves the audit reads filtered by actor without an action equality (`GET +/audit?actor=`, its export pages, `/me/security-events`); without it SQLite sought the actor on the +first index and then sorted all of its rows. That each read seeks its index is measured on SQLite only; on Postgres and SQL Server the planner's choice is not tested. The residuals are listed below, under *Declined, each with its residual*. A sign-in lock notice on a TOTP-enrolled local account tells the owner to sign in with the password and the code diff --git a/messagefoundry/auth/service.py b/messagefoundry/auth/service.py index 7618e6c85e..35e3a6e1af 100644 --- a/messagefoundry/auth/service.py +++ b/messagefoundry/auth/service.py @@ -13485,14 +13485,12 @@ async def _lock_notice_held_back(self, user: UserRecord, lock: str) -> bool: """The throttle's READ: whether a notice of this ``lock`` kind already went out in the window. Kept apart from the row write, so the background task can send the mail between this read - and :meth:`_record_lock_notice` (BACKLOG #2216). Every backend carries an - ``ix_audit_actor`` (actor, action, id) index for it, keyed on a prefix of the actor on - Postgres (vault BACKLOG #3120). That the read seeks it is measured on SQLite only; on - Postgres and SQL Server the planner's choice is not tested. Without it the read walked - every recent ``auth.lock_notice`` row of every account. One such read still runs at a time - per service (``_lock_notice_read_slots``), so locks on many accounts at once cannot fill the - store's read pool, which sign-ins wait on too. The cost is that each notice waits behind the reads queued before it, across every - account in this process; ``docs/SECURITY.md`` states it. + and :meth:`_record_lock_notice` (BACKLOG #2216). An audit index serves it (vault BACKLOG + #3120); ``docs/SECURITY.md`` names it and says on which backends its use is measured. One + such read still runs at a time per service (``_lock_notice_read_slots``), so locks on many + accounts at once cannot fill the store's read pool, which sign-ins wait on too. The cost is + that each notice waits behind the reads queued before it, across every account in this + process; ``docs/SECURITY.md`` states it. A failed read fails OPEN (``False``) and is logged by exception class alone. The line names neither the account nor the notice kind (BACKLOG #1131): it runs only when a lock lands, diff --git a/messagefoundry/store/postgres.py b/messagefoundry/store/postgres.py index 5c01635a95..364768e87d 100644 --- a/messagefoundry/store/postgres.py +++ b/messagefoundry/store/postgres.py @@ -8020,8 +8020,8 @@ def _audit_where( params: list[Any] = [] if actor is not None: params.append(actor) - # The prefix term lets the planner seek ix_audit_actor, which is keyed on that prefix - # (vault BACKLOG #3120); the whole-value term is what decides the match. + # The prefix term lets the planner seek ix_audit_actor and ix_audit_actor_id, which are + # keyed on that prefix (vault BACKLOG #3120); the whole-value term decides the match. n = len(params) clauses.append(f"left(actor, 200) = left(${n}::text, 200) AND actor = ${n}") if action is not None: diff --git a/tests/test_audit_actor_index.py b/tests/test_audit_actor_index.py index 056d3ded38..623bec3e56 100644 --- a/tests/test_audit_actor_index.py +++ b/tests/test_audit_actor_index.py @@ -31,6 +31,7 @@ import pytest from messagefoundry.store import MessageStore +from messagefoundry.store.audit_exclusion import AuditExclusion _SQLSERVER_ON = bool(os.getenv("MEFOR_TEST_SQLSERVER")) _POSTGRES_ON = bool(os.getenv("MEFOR_TEST_POSTGRES")) @@ -78,11 +79,13 @@ async def _index_columns(store: Any, name: str) -> list[str]: if backend == "postgres": # pg_get_indexdef per key position, so an expression key reads as its expression. The # store's own schema only, so a same-named index in another schema cannot answer for it. + # Compared as a name, not cast to regnamespace, which would case-fold a mixed-case schema. rows = await store._fetchall( "SELECT pg_get_indexdef(c.oid, k.ord, true) AS col FROM pg_class c" " JOIN pg_index i ON i.indexrelid = c.oid" + " JOIN pg_namespace n ON n.oid = c.relnamespace" " CROSS JOIN LATERAL generate_series(1, i.indnkeyatts::int) AS k(ord)" - " WHERE c.relname = $1 AND c.relnamespace = current_schema()::regnamespace" + " WHERE c.relname = $1 AND n.nspname = current_schema()" " ORDER BY k.ord", name, ) @@ -129,6 +132,28 @@ async def test_an_overlong_actor_is_still_audited_and_found(store: Any) -> None: assert [r["detail"] for r in rows] == ['{"lock": "sign_in"}'] +async def test_security_events_tell_apart_two_names_sharing_the_indexed_prefix(store: Any) -> None: + """``/me/security-events`` filters on the Postgres actor prefix too, and only the whole-value + term keeps one account from reading another's events when both names share those 200 + characters. 250 characters fits every backend's username and actor columns. Random names, so a + shared server database run twice does not meet the first run's rows.""" + stem = "".join(hashlib.sha256(os.urandom(16)).hexdigest() for _ in range(4))[:200] + mine, theirs = stem + "a" * 50, stem + "b" * 50 + for n, name in enumerate((mine, theirs)): + await store.create_user( + user_id=f"u-{stem[:16]}-{n}", + username=name, + auth_provider="local", + password_generated=False, + now=1.0, + ) + await store.record_audit("auth.login_success", actor=mine, detail="mine") + await store.record_audit("auth.login_success", actor=theirs, detail="theirs") + rows = await store.security_events_for_user(mine) + assert [r["detail"] for r in rows] == ["mine"] + assert await store.count_security_events_for_user(mine) == 1 + + async def _plan( store: MessageStore, monkeypatch: pytest.MonkeyPatch, @@ -187,12 +212,17 @@ async def test_the_throttle_read_seeks_the_actor_index_on_sqlite( assert "ix_audit_action" in control, control -# Every read filtered by actor without an action equality: the audit page and its total, the export -# pages (a keyset cursor, or an offset), a time-bounded read, and the self-service security events. +# At least these reads filter by actor without an action equality: the audit page by offset and its +# capped total, the export's keyset cursor, a time-bounded read, the page a reader without +# users:manage gets (an exclusion), and the self-service security events. +_HIDDEN = AuditExclusion( + actions=frozenset({"auth.hidden"}), rows=frozenset({("auth.login_failed", "hidden")}) +) _ACTOR_ONLY_READS: dict[str, Callable[[MessageStore], Awaitable[object]]] = { "list_audit": lambda s: s.list_audit(actor="owner", limit=50), "list_audit_offset": lambda s: s.list_audit(actor="owner", limit=50, offset=500), "list_audit_before_id": lambda s: s.list_audit(actor="owner", limit=50, before_id=1000), + "list_audit_excluded": lambda s: s.list_audit(actor="owner", limit=50, exclude=_HIDDEN), "list_audit_since": lambda s: s.list_audit(actor="owner", since=0.0, until=9e9, limit=50), "count_audit_capped": lambda s: s.count_audit(actor="owner", limit=1000), "security_events": lambda s: s.security_events_for_user("owner", limit=100, until=9e9), @@ -204,9 +234,10 @@ async def test_an_actor_only_read_walks_the_actor_in_id_order_on_sqlite( read: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: """With ``ix_audit_actor`` alone, SQLite sought the actor on it and then sorted all of the - actor's rows into a temporary B-tree to read the newest few: about 45 ms for a 300k-row actor - against 0.05 ms before it, and an export paging by offset re-sorted every page. ``(actor, id)`` - hands the rows over already in ``id`` order. + actor's rows into a temporary B-tree to read the newest few. Measured with 300k of 600k rows + under one actor: a first page took 23 ms against 0.06 ms before that index, and a page at + offset 100000 took 233 ms against 8.6 ms. ``(actor, id)`` hands the rows over already in ``id`` + order: 0.05 ms and 2.9 ms. An empty database plans like a full one here, because SQLite has no statistics until ANALYZE and the store never runs it.""" From a67d10587932615f54fc2b5b5f1f5b1dffbf2233 Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Sat, 17 Oct 2026 03:25:59 -0500 Subject: [PATCH 39/57] D4b handover: PR title and banner text (BACKLOG #3120) Proposed PR title: Lock notice follow-ups: a full queue reads mailed false, actor indexes for the throttle and actor-filtered audit reads, drain and residual docs (BACKLOG #3120) Proposed #3120 ledger banner: BUILT on engine branches b257-d4 and b257-d4b: SecurityNotifier.notify returns whether it queued the event, so a lock notice the full queue dropped writes mailed: false and the next lock mails again (follow-up 2, queue-full path; a later send_failed still reads mailed: true, a residual in SECURITY.md). ix_audit_actor (actor, action, id) serves the throttle read, and a companion ix_audit_actor_id (actor, id) serves audit reads filtered by actor without an action equality (GET /audit?actor=, export pages, /me/security-events), on SQLite and SQL Server; both key left(actor, 200) on Postgres, where _audit_where and _security_events_where_pg filter on the prefix with a whole-value recheck ((d), follow-up 5). Drain comment names NSSM's 15 s window and the drains sharing it (1); the one-read-at-a-time limit is in SECURITY.md (3); test drains in finally (4). (a) cross-shard double mail, (b) failure-log timing and (c) no cap on queued tasks declined in writing, residuals in SECURITY.md. Plans measured on SQLite only; Postgres and SQL Server legs unread, and ci.yml does not name tests/test_audit_actor_index.py in either server leg. Open: on SQLite a narrow time window over a heavy actor reads slower than on main (about 50 ms against 0.01 ms at 300k rows); review notes in the PR body. Also carries the D3b doc carry-over for #3010 (CONFIGURATION.md alert resolution and revoke-max row, settings.py breaker comment, step-up refusal wording in SECURITY.md and changelog.d/3010). From 2fd439f4d45273fef8eafabe57a424a7921b4bc3 Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Sat, 17 Oct 2026 03:56:47 -0500 Subject: [PATCH 40/57] Login supersession follows a time-bounded retired-hash lineage (BACKLOG #3028) An elevation that finished before a sign-in's supersession left the presented hash naming no session, so the session lived on under its new token. rotate_session now writes a session_lineage link (retired hash, successor hash, retired_at) in its own transaction on SQLite, Postgres and SQL Server. supersede_session(lineage_since=...) follows the links, over any number of hops up to 32, to the live end and revokes it in the same transaction, through one shared walk in store/session_lineage.py. The service passes oidc_flow_ttl_seconds plus 60 seconds; the session reaper purges older links. No session read consults the table, and a per-backend AST pin keeps it so. The links are unsealed: a forged link can only direct a revoke, which needs no key. ADR 0210 records the design; docs/SECURITY.md states the bound and the second-holder case. --- changelog.d/3028.security.md | 12 + docs/PHI.md | 4 +- docs/SECURITY.md | 37 ++- ...ows-a-time-bounded-retired-hash-lineage.md | 108 +++++++ docs/adr/README.md | 1 + messagefoundry/api/app.py | 15 +- messagefoundry/auth/service.py | 51 +++- messagefoundry/store/base.py | 62 ++-- messagefoundry/store/postgres.py | 87 ++++-- messagefoundry/store/session_lineage.py | 69 +++++ messagefoundry/store/sqlserver.py | 94 ++++-- messagefoundry/store/store.py | 90 ++++-- tests/_session_rotation_contract.py | 76 ++++- tests/test_auth_hardening.py | 10 +- tests/test_postgres_store.py | 14 +- tests/test_session_sql_placeholders.py | 17 +- tests/test_session_supersession.py | 9 +- tests/test_session_supersession_lineage.py | 289 ++++++++++++++++++ tests/test_sqlserver_store.py | 14 +- tests/test_store_pool_acquire_timeout.py | 3 +- tests/test_task_resilience.py | 6 +- tests/test_webauthn_store.py | 12 + 22 files changed, 975 insertions(+), 105 deletions(-) create mode 100644 changelog.d/3028.security.md create mode 100644 docs/adr/0210-session-supersession-follows-a-time-bounded-retired-hash-lineage.md create mode 100644 messagefoundry/store/session_lineage.py create mode 100644 tests/test_session_supersession_lineage.py diff --git a/changelog.d/3028.security.md b/changelog.d/3028.security.md new file mode 100644 index 0000000000..60a086a9e6 --- /dev/null +++ b/changelog.d/3028.security.md @@ -0,0 +1,12 @@ +- **A sign-in now ends the session its presented token became, when an elevation finished first.** + A console sign-in ends the session its browser presented. An elevation in another tab that + finished during the sign-in used to leave that token naming no session, so the session lived on + under its new token. Each rotation now records which token hash replaced the old one, in its own + store transaction, in a new `session_lineage` table on SQLite, Postgres and SQL Server. The + supersession follows those links to the live session and ends it, in the same transaction as the + revoke, for `[auth].oidc_flow_ttl_seconds` plus 60 seconds after each rotation; the session reaper + deletes older links. A retired token still authenticates nothing: no session read consults the + links. `docs/SECURITY.md` states the bound that remains, and ADR 0210 records the design. + On the server backends the schema change moves the ADR 0064 schema hash: under their default + `[store].schema_management`, `external`, the engine refuses to open until `provision-schema` has + run; under `auto` the next open runs the DDL. diff --git a/docs/PHI.md b/docs/PHI.md index a36917e2e1..ff6b4c58cb 100644 --- a/docs/PHI.md +++ b/docs/PHI.md @@ -991,7 +991,9 @@ on the store-file ACL plus the volume/whole-DB layer for the rest. - `audit_log`, `delivered_keys`, `resend_log` — **keep-forever by design**; `[retention].audit_days` is **reserved and not enforced**. - `sessions` — expired rows are deleted by `purge_expired_sessions`, driven from the auth layer (idle - 30 min / absolute 12 h), not by the RetentionRunner. + 30 min / absolute 12 h), not by the RetentionRunner. The same pass deletes `session_lineage` links + (two token hashes and a time, vault BACKLOG #3028) once they are past the window the sign-in + supersession follows them for. - `processed_files` — an age/count prune driven from the wiring runner, also outside the RetentionRunner. - `state.namespace` / `state.key` — removed with their value on `[retention].state_max_age_days`. - `reference.*` — **no purge path**; replaced only by the next sync's build-new-then-flip. diff --git a/docs/SECURITY.md b/docs/SECURITY.md index 336321537e..579565815b 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -4450,14 +4450,35 @@ engine client (`EngineClient.login`) both name the token they replace in `supers client names only a token the engine issued to it, never one adopted with `set_token`. Every elevation gives a session a new token: completing MFA, a step-up, a passkey ceremony or a -password re-check. The supersession finds and ends the presented session in one atomic store -operation, so an elevation cannot land between the two (BACKLOG #2146). If the supersession runs -first, a later elevation on that session fails and asks the user to sign in again. **One case stays -open.** An elevation in another tab can finish while the sign-in is still in progress, before the -supersession runs. That window covers the credential check, and on federated sign-in the round trip -to the identity provider. The presented token then names no session, so nothing is ended. The -session lives on under its new token until it expires, because the engine does not record which -token replaced the old one. +password re-check. The supersession finds and ends the presented session in one store transaction, +so an elevation cannot land between the two (BACKLOG #2146). If the supersession runs first, a later +elevation on that session fails and asks the user to sign in again. + +**An elevation that finished first is followed (vault BACKLOG #3028, +[ADR 0210](adr/0210-session-supersession-follows-a-time-bounded-retired-hash-lineage.md)).** An +elevation in another tab can finish while the sign-in is still in progress, before the supersession +runs. On federated sign-in that window includes the round trip to the identity provider, because +the presented token is read on the start leg. The presented token then names no session. So each +rotation records, in its own store transaction, which token hash replaced the old one, and the +supersession follows those links, over as many elevations as there were, and ends the session at +the live end, in the same transaction as the revoke. The audit row names that session and, in a +`presented` field, the hash the browser presented. + +The link is followed for a bounded time: `[auth].oidc_flow_ttl_seconds` plus 60 seconds, which is +360 seconds at the default, and the session reaper deletes older links. **What stays open is outside +that bound.** A second holder of the same token who completed an elevation longer ago than that, or +a sign-in that runs past it, leaves the presented token naming no session that is followed, so +nothing is ended. That session lives on under the token the elevation handed out, until it expires, +as before this change. + +**A retired token authenticates nothing.** No session read, the validator included, consults the +links; the supersession is their one reader. For the length of the bound, a retired token has one +power: presented at a sign-in that succeeds, it ends the session it became. That power is new, and +it can only end a session, never open or extend one. Whoever held that token before the elevation +could already end the session with `POST /auth/logout`; the link lets them do it for the length of +the bound after the elevation too, and only through a sign-in with a credential that passes. The +links are not under the session integrity seal (BACKLOG #2592): a forged link can only direct a +revoke, and a revoke needs no key on any backend. A session's `id` is its token hash, and that hash changes whenever the session completes MFA or a step-up. So an id shown on a sessions page can go stale. The console's revoke says "Nothing was diff --git a/docs/adr/0210-session-supersession-follows-a-time-bounded-retired-hash-lineage.md b/docs/adr/0210-session-supersession-follows-a-time-bounded-retired-hash-lineage.md new file mode 100644 index 0000000000..04ed8c43a3 --- /dev/null +++ b/docs/adr/0210-session-supersession-follows-a-time-bounded-retired-hash-lineage.md @@ -0,0 +1,108 @@ +# 0210 -- Session supersession follows a time-bounded retired-hash lineage + +- **Status:** Accepted (2026-10-17, on build; the design was set by the batch 257 Manager's brief; + the owner may overrule at review) +- **Date:** 2026-10-17 +- **Related:** vault BACKLOG #3028 (the case this closes), #2146 (engine PR 2028, the atomic + supersession it extends), #3029 (the same race on `logout` and `revoke_own_session`, not built + here), #2592 (the session integrity seal), #1146 (ASVS 7.2.4), ADR 0142 (federated sign-in), + CLAUDE.md section 0 (severity is conditional: there are no deployments) + +Built with the change, in the same pull request. + +## Context + +A console sign-in ends the session its browser presented (ASVS 7.2.4): the new `Set-Cookie` +replaces the old one, so the server is what strands it. Every elevation (MFA, a step-up, a passkey +ceremony, a password re-check) re-keys the session row to a new token hash in place, and the old +hash stops naming any row. + +Engine PR 2028 (BACKLOG #2146) made the supersession's read and revoke one store operation, so an +elevation that starts after it fails closed. It left an elevation that FINISHED before it open. The +presented hash then named nothing, the supersession ended nothing, and the session lived on under +its new token. The window is the whole sign-in, and on the federated leg it includes the identity +provider's round trip, because the start leg reads the cookie (the callback is cross-site and the +Strict cookie never reaches it). Nothing recorded which hash replaced which. + +The constraint the fix must hold is the row's closing item 2: a retired hash must authenticate +nothing, and be usable for nothing but the lookup that ends its successor. + +## Decision + +**A rotation records a link from the hash it retired to its successor, and the login supersession, +and only it, follows those links for a bounded time to the live end and revokes that.** + +1. **A separate table, `session_lineage`** (`retired_hash` primary key, `successor_hash`, + `retired_at`, an index on `retired_at`), on SQLite, Postgres and SQL Server. `rotate_session` + writes one row in the SAME transaction as the re-key, and only when the re-key landed. +2. **The walk is one shared function**, `store/session_lineage.py`'s `revoke_through_lineage`. Each + step tries to revoke the hash it holds; only when that ends nothing does it read the hash's + successor and step. Every backend runs it inside one transaction on one connection. SQLite's + first `UPDATE` takes the database write lock even when it matches nothing, so nothing else can + re-key a row until the commit. On Postgres (READ COMMITTED) and SQL Server each statement reads + what has committed when it runs, so a rotation that commits mid-walk costs one more step, never + the revoke: its link was written in its own transaction. The walk follows at most 32 links. +3. **The bound is the caller's, as `lineage_since`**: a link retired before it is not followed. The + store reads no settings. `AuthService.supersession_lineage_seconds` is + `oidc_flow_ttl_seconds` plus 60 seconds, the longest sign-in a rotation can race plus the + callback's own work after it. With no `lineage_since`, `supersede_session` is the plain revoke + it was, which `_end_lapsed_session` keeps. +4. **The session reaper purges links past the window**, in the same pass as expired sessions, + through a new `lineage_seconds` argument to `purge_expired_sessions`. +5. **No session read consults the table.** `get_session`, `list_sessions` and the service's session + loader read `sessions` only. A test pins, per backend, that only `rotate_session`, + `supersede_session` and `purge_expired_sessions` name the table or a statement on it, and that + only `_supersede_session_hash` passes a window. +6. **The links are not sealed** (BACKLOG #2592). A seal stops a database writer forging authority, + and a link carries none: the most a forged link can do is direct a revoke at a live row, and a + revoke needs no key on any backend, so that writer could revoke the row directly. +7. **The audit row names the session that ended**, by the hash it ended under, and adds the + `presented` hash when a link was followed. + +## Acceptance Criteria + +- **AC-1** -- WHEN an elevation has completed on a session before a sign-in that presents the + session's old token supersedes it, THE SYSTEM SHALL end the session under its new token. + -> `tests/test_session_supersession_lineage.py::test_an_elevation_that_finished_before_the_sign_in_does_not_keep_the_session_live` +- **AC-2** -- WHEN several elevations have completed, THE SYSTEM SHALL follow each link to the live + end. + -> `tests/test_session_supersession_lineage.py::test_the_link_is_followed_over_several_elevations` +- **AC-3** -- IF a link was retired before the window, THEN THE SYSTEM SHALL NOT follow it. + -> `tests/test_session_supersession_lineage.py::test_a_link_older_than_the_window_is_not_followed` +- **AC-4** -- THE SYSTEM SHALL authenticate nothing by a retired token, or by a token whose hash a + link names. + -> `tests/test_session_supersession_lineage.py::test_a_retired_token_authenticates_nothing_while_its_link_is_live`, + `tests/test_session_supersession_lineage.py::test_a_token_whose_hash_links_to_a_live_session_authenticates_nothing` +- **AC-5** -- THE SYSTEM SHALL read the lineage only from the supersession, on every backend. + -> `tests/test_session_supersession_lineage.py::test_only_the_rotation_the_supersession_and_the_purge_touch_the_lineage` +- **AC-6** -- WHEN the reaper runs, THE SYSTEM SHALL delete links past the window and keep the rest. + -> `tests/_session_rotation_contract.py` (`assert_session_lineage_contract`, run on all three + backends) + +## Options considered + +1. **A separate retired-hash table, followed only by the supersession, time-bounded.** CHOSEN. +2. **A `previous_hash` column on `sessions`.** Rejected: one column holds one hop, so a second + elevation during one sign-in would lose the first link, and a sealed column would put the + retired hash under the seal and into every session read's row. +3. **Keep the old hash resolving, revoked, as a tombstone row.** Rejected: a retired hash would + then name a row that every session reader sees, which is the wrong side of closing item 2. +4. **Bound the walk by the sign-in's own start time** instead of a fixed window. Rejected for now: + the password and Kerberos legs would need their start time carried to the mint, and the + Manager's brief set a fixed window of at least the flow TTL. It would narrow the second-holder + case below, and stays open as a refinement. + +## Consequences + +**Positive** -- The case engine PR 2028 left open is closed for any elevation that finished inside +the window, including several in a row. The walk is one function with a `lineage_since` argument, +so `logout` and `revoke_own_session` (#3029) can follow the same links by passing a window. + +**Negative** -- For the length of the window, a retired token gains one power: presented at a +sign-in that succeeds, it ends the session it became. It cannot open or extend one. +`docs/SECURITY.md` states this. Each rotation writes one more row, which the reaper deletes. + +**Stated limits** -- An elevation that finished longer ago than the window, such as one a second +holder of the token completed earlier, is not followed, and neither is a sign-in that runs past the +window; that session lives on under its new token until it expires. More than 32 links in one +window ends nothing. The Postgres and SQL Server legs run only on hosted runners. diff --git a/docs/adr/README.md b/docs/adr/README.md index 3a60cccef7..c4146b27d9 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -231,6 +231,7 @@ what is withheld and what you can request. | [0206](0206-an-hl7-write-or-re-encode-never-lets-data-become-structure.md) | **An HL7 write or re-encode never lets data become structure** (vault BACKLOG #2558, #2559, #2560) -- `Message.set_data` writes a value read from a message as data at every level, and `copy_field` and `split_field` use it for a component or subcomponent source; `set` keeps its meaning for authored text; the Steps view's Copy Field emits `set_data` for a literal leaf source; the advisory handler-security lint gains `leaf-to-whole-field` for the hand-written `msg.set(field, msg.field(leaf))` shape. The MLLP delimiter override escapes a target delimiter found in a leaf with the target escape character and refuses a value it cannot represent, every override failure being `NegativeAckError(permanent=True)`. A source that does not split refuses an HL7 v2 body holding more than one `MSH`, with an `AR` where it has an ACK channel; `RemoteFileSource` now splits a batch like the File source, which is unchanged. Rejected: escaping every whole-field `set`, refusing a mixed-level copy, a marked `str` subclass, splitting on a listener | **Accepted (2026-10-02, on build)** -- decision delegated by the owner under the driver rule and taken by the batch 186 Manager; the owner may overrule at review. Built with the change. Severity conditional per CLAUDE.md section 0. **Amended 2026-10-04** (vault BACKLOG #2861, #2862): the Steps view's Set Field keeps `set` only for the HL7 separators `^ ~ & \`, so a template holding `:`, `>` or `*` writes `set_data`; the `leaf-to-whole-field` finding's advice names the X12 case | | [0207](0207-a-single-use-enrolment-code-for-a-directory-account-s-first-engine-factor.md) | **A single-use enrolment code for a directory account's first engine factor** (vault BACKLOG #2609, closing item 2; the decision ADR 0142 Amendment C left open) -- a directory account's first engine factor is bound only with a single-use enrolment code an Administrator issues and hands over out of band. Required at the step that binds and stamps (`confirm_mfa_enrollment`, `finish_webauthn_registration`) when the account is `ad`, holds no engine factor, and the session is not stamped MFA-verified: so every Kerberos session and a federated one with the claim gate off, never a claim-gated federated one. That step also re-checks that the account still holds no factor, because the web console's passkey finish route does not, and the TOTP enable turns on only the secret that was verified, since a later begin replaces the staged one today. The passkey finish route gains the per-actor attempt budget the TOTP confirm routes already draw. Disabling the account clears an outstanding code, and every refused code gets one client-facing reply. The handover is the control, so it must leave the directory: an Administrator checks who is asking and never replies through the account's own mail or chat. The rule reads no session mechanism and no `kerberos_enabled`, and holds with `require_mfa` on or off. At least 128 bits, stored as its SHA-256 digest, one per account, 24 hours, spent by one conditional write before the session is stamped. Issued by an Administrator behind step-up, never for the caller's own account, also by an MFA reset of a directory account, and, for a site where no Administrator can issue one, by a host command on `admin-unlock`'s gate, which is not BACKLOG #2711's outage recovery. Rejected: password only, a TOTP-on-directory-password shortcut, an emailed link, a short Argon2 code, gating on the mechanism | **Proposed (2026-10-05)** -- doc-only, no code; drafted on the owner's direction. Merging records it as Proposed and accepts nothing; Accepted needs the owner. Open: the code with `require_mfa` off, a fixed lifetime, a code from an MFA reset, and the host command's reach. Severity conditional per CLAUDE.md section 0 | | [0208](0208-a-theia-desktop-app-gives-interface-analysts-a-simpler-steps-only-editor.md) | **A Theia desktop app gives interface analysts a simpler Steps-only editor** (owner rulings and Manager decisions of 2026-10-07; rewrites the uncommitted 2026-10-02 hosted-Theia drafts after two four-lens reviews) -- two Electron builds sharing one native Theia Steps extension: an analyst build capped at the Steps level, and a developer build that is a full IDE with a Steps / Split / Code switch and no per-user limits. The purpose is a simpler interface, not enforcement. The analyst build signs in to the engine, clears every session gate, proves the session on a non-exempt route at start-up and before each save, and turns on editing only for `code:steps` (new, advisory, read by the editor only) or `code:edit`. That role check is the primary control and a guardrail: the developer build, VS Code and any other tool are outside it. An optional repository check classifies a change as Steps-only by comparing `code` rows, every control header and each typed row's full statement against base, and passes anything else only with a `code:edit` reviewer's approval of the head commit; it holds against a deliberate change only when run from a definition the change cannot edit. Analyst Test takes a generator spec and never passes `--show-phi`. Review R1 is a precondition answered by the R1 fix and a typed-only `lens rewrite` mode the analyst build always sets; both landed in PR 2155 (2026-10-08). Analyst is a built-in role. The analyst build drops three ADR 0076 guardrails (Amendment G). Hosting is deferred to its own ADR. Rejected: a filtered `@theia/plugin-ext` analyst build, per-user limits in the developer build, VS Code for analysts, a Steps editor in the web console. Spec: [`docs/design/theia-analyst-editor.md`](../design/theia-analyst-editor.md) | **Proposed (2026-10-07)** -- doc-only, no editor code. Merging records it as Proposed and accepts nothing; Accepted needs the owner. Licence AGPL-3.0-or-later (owner ruling). Open: spikes S-1 to S-4 (S-1 is met except its Test and language-server legs, measured on a browser target; S-2 is met on the browser build and on Electron, spike S-2b; S-4's classifier half is met by a prototype and its base-ref workflow half is untested), the start-up probe route (an implementation detail), and the differences ADR 0076 Amendment G, G.7, records between PR 2155's lens code and its rules | +| [0210](0210-session-supersession-follows-a-time-bounded-retired-hash-lineage.md) | **Session supersession follows a time-bounded retired-hash lineage** (vault BACKLOG #3028; extends #2146, engine PR 2028) -- each session rotation writes a `session_lineage` link (retired hash, successor hash, `retired_at`) in its own transaction, on SQLite, Postgres and SQL Server. The login supersession, and nothing else, follows those links over any number of hops (at most 32) to the live end and revokes it in the same transaction, for `oidc_flow_ttl_seconds` plus 60 seconds after each rotation; the session reaper purges older links. No session read consults the table, so a retired hash authenticates nothing; a test pins the readers per backend. The links are unsealed, because a forged one can only direct a revoke, which needs no key. One shared walk, so #3029 can reuse it. Rejected: a `previous_hash` column, a revoked tombstone row, a per-sign-in start-time bound. Open: an elevation older than the window, such as a second holder's, is not followed | **Accepted (2026-10-17, on build)** -- design set by the batch 257 Manager's brief; the owner may overrule at review. Severity conditional per CLAUDE.md section 0 | ## Retired allocations — numbers taken but never authored diff --git a/messagefoundry/api/app.py b/messagefoundry/api/app.py index 473de2ac18..f955399938 100644 --- a/messagefoundry/api/app.py +++ b/messagefoundry/api/app.py @@ -9336,13 +9336,16 @@ async def _session_reaper( store: Store, *, idle_seconds: float | None = None, + lineage_seconds: float | None = None, wall: Callable[[], float] | None = None, mono: Callable[[], float] | None = None, ) -> None: """Drop expired session rows, first ``_SESSION_REAP_FIRST_DELAY`` after start and then every ``_SESSION_REAP_INTERVAL``, until the task is cancelled. With ``idle_seconds`` given, idle-expired rows go too (BACKLOG #2096); the lifespan passes the idle timeout the validator - uses. + uses. With ``lineage_seconds`` given, session lineage links older than that go too (vault + BACKLOG #3028); the lifespan passes the window the supersession follows them for. Purging as of + the earlier instant below only keeps a link longer, never shortens that window. **The reaper skips a pass when the wall clock has jumped forward (BACKLOG #2283).** The purge judges expiry and idleness by the wall clock, and its deletes cannot be undone. Run just after @@ -9401,7 +9404,9 @@ def _clocks() -> tuple[float, float]: continue try: await store.purge_expired_sessions( - now=now - _SESSION_REAP_STEP_TOLERANCE, idle_seconds=idle_seconds + now=now - _SESSION_REAP_STEP_TOLERANCE, + idle_seconds=idle_seconds, + lineage_seconds=lineage_seconds, ) except asyncio.CancelledError: raise @@ -10605,7 +10610,11 @@ async def _audit_upload_prune(meta: UploadedFileMeta) -> None: auth_settings.oidc_redirect_path, ) reaper = asyncio.create_task( - _session_reaper(store, idle_seconds=auth.session_idle_seconds) + _session_reaper( + store, + idle_seconds=auth.session_idle_seconds, + lineage_seconds=auth.supersession_lineage_seconds, + ) ) if _initial_credential_warn_lead(auth) is not None: # BACKLOG #1141 (ASVS 6.4.5): a nudge for every unclaimed temporary password an diff --git a/messagefoundry/auth/service.py b/messagefoundry/auth/service.py index 35e3a6e1af..326f22cdf2 100644 --- a/messagefoundry/auth/service.py +++ b/messagefoundry/auth/service.py @@ -183,6 +183,14 @@ {"mfa_enroll_confirm", "mfa_verify", "webauthn_enroll", "webauthn_assert"} ) +#: Vault BACKLOG #3028: what :attr:`AuthService.supersession_lineage_seconds` adds to the flow TTL +#: for the sign-in's own work after its last wait. At the federated callback that is the code +#: exchange and a JWKS refresh, each a blocking call under a 10-second socket timeout; on the +#: password and directory legs it is the whole request, whose LDAP calls each sit under +#: ``ad_connect_timeout`` and ``ad_receive_timeout`` (10 seconds by default). A sign-in that runs +#: past the window is a stated limit in docs/SECURITY.md. +_LINEAGE_SIGN_IN_SLACK_SECONDS: Final = 60.0 + @cache def _audit_write_errors() -> tuple[type[Exception], ...]: @@ -4311,6 +4319,21 @@ def oidc_flow_ttl_seconds(self) -> int: """The staged-flow lifetime, so the browser cookie's max-age matches the server-side TTL.""" return self._settings.oidc_flow_ttl_seconds + @property + def supersession_lineage_seconds(self) -> float: + """How long a retired session hash still leads a sign-in's supersession to the session it + became (vault BACKLOG #3028, ADR 0210), in seconds. The session reaper purges links older + than this. + + It must outlast the longest sign-in a rotation can race, because the presented hash is read + at the START of the sign-in and the supersession runs at its end. The longest is the + federated leg: its start leg stages the hash, and the callback can come back as late as + ``oidc_flow_ttl_seconds`` after. :data:`_LINEAGE_SIGN_IN_SLACK_SECONDS` covers the + callback's own work after that, and every other leg is one request inside it. A longer + window only lengthens the time a stale copy of a token can END its successor; it never + lets one authenticate.""" + return float(self._settings.oidc_flow_ttl_seconds) + _LINEAGE_SIGN_IN_SLACK_SECONDS + def _oidc_redirect_uri(self, public_origin: str) -> str: return public_origin.rstrip("/") + self._settings.oidc_redirect_path @@ -8875,18 +8898,27 @@ async def _supersede_session_hash(self, prior_hash: str, *, client: str | None) session that was still LIVE gets an audit row, so the trail does not record the ending of something that had already ended. - **The read and the revoke are one atomic store operation (BACKLOG #2146).** Why that closes - the race with a concurrent rotation is :meth:`AuthStore.supersede_session`'s to say. It holds + **The read and the revoke are one store transaction (BACKLOG #2146).** Why that closes the + race with a concurrent rotation is :meth:`AuthStore.supersede_session`'s to say. It holds across engine processes too, because engine shards serve the API from one shared store. - **A rotation that committed BEFORE that operation is a stated limit, not a race this - closes.** The presented hash then names no session, so this ends nothing, and the session - lives on under its new token. docs/SECURITY.md (ASVS 7.2.4, the supersession paragraphs) - states how wide that window is. + **A rotation that committed BEFORE that transaction is followed (vault BACKLOG #3028, ADR + 0210).** An elevation in another tab can finish while this sign-in is in flight, and on the + federated leg the presented hash was read at the start leg, before the identity provider's + round trip. The presented hash then names no session. The store follows the lineage links + each rotation wrote to the session's live end and revokes that, in the same transaction. + Only links retired within :attr:`supersession_lineage_seconds` of now are followed, so this + is the one place a retired hash leads anywhere, and it leads only to an end. An elevation + that finished longer ago than that, such as one a second holder of the token completed + earlier, is not followed; docs/SECURITY.md (ASVS 7.2.4, the supersession paragraphs) states + that bound. The audit row names the session that ended and, when a link was followed, the + hash that was presented. Returns True when it audited. """ before = time.time() - prior = await self._store.supersede_session(prior_hash, now=before) + prior = await self._store.supersede_session( + prior_hash, now=before, lineage_since=before - self.supersession_lineage_seconds + ) if prior is None: return False after = time.time() @@ -8900,10 +8932,13 @@ async def _supersede_session_hash(self, prior_hash: str, *, client: str | None) if not prior.is_live(now=moment, idle_seconds=self.session_idle_seconds): return False owner = await self._load_user(prior.user_id) + detail = {"scope": "superseded", "session": prior.token_hash[:12]} + if not constant_time_equal(prior.token_hash, prior_hash): # ASVS 11.2.4, as elsewhere + detail["presented"] = prior_hash[:12] await self._audit( "auth.session_revoked", actor=owner.username if owner is not None else None, - detail=_json({"scope": "superseded", "session": prior_hash[:12]}), + detail=_json(detail), client=client, ) return True diff --git a/messagefoundry/store/base.py b/messagefoundry/store/base.py index 71d48bcd76..02fe9f9105 100644 --- a/messagefoundry/store/base.py +++ b/messagefoundry/store/base.py @@ -2771,7 +2771,9 @@ async def list_sessions( async def touch_session(self, token_hash: str, *, now: float | None = None) -> None: ... - async def rotate_session(self, token_hash: str, *, new_token_hash: str) -> bool: + async def rotate_session( + self, token_hash: str, *, new_token_hash: str, now: float | None = None + ) -> bool: """Re-key a live session to ``new_token_hash``, in place (ASVS 7.2.4). A pure re-key: every other column — ``user_id``, ``created_at``, ``expires_at``, ``client``, @@ -2791,6 +2793,12 @@ async def rotate_session(self, token_hash: str, *, new_token_hash: str) -> bool: that report what they changed: every other one is deliberately blind (a write against a dead hash is a silent no-op), but a caller rotating a session is about to hand the new token to a user, so it must be able to fail closed if the session died underneath it. + + **A re-key that lands also records a lineage link, in the SAME transaction** (vault BACKLOG + #3028, ADR 0210): ``token_hash`` retired at ``now``, replaced by ``new_token_hash``. The + session row itself carries nothing of its old hash, so the old hash still resolves to + nothing. The link's one reader is :meth:`supersede_session`; ``store/session_lineage.py`` + says why it is unsealed. A re-key that did not land writes no link. """ ... @@ -2820,21 +2828,33 @@ async def mark_session_mfa_verified( async def revoke_session(self, token_hash: str, *, now: float | None = None) -> None: ... - async def supersede_session(self, token_hash: str, *, now: float) -> SessionRecord | None: - """Revoke the unrevoked session ``token_hash`` names, and return its row as revoked. - - Returns ``None`` when no unrevoked row has that hash, so nothing was written. A row that is - expired or idle is still revoked and returned. Its other columns are as the revoke found - them, so the caller can judge whether the session was live, and ``revoked_at`` is ``now``. - - **ONE atomic operation, so it cannot interleave with :meth:`rotate_session` (BACKLOG - #2146).** It is one statement on Postgres and SQL Server, and one transaction on SQLite. The - login supersession used to read the row and then revoke it by hash, and a rotation that - re-keyed the row between the two left the revoke matching nothing, so the session lived on - under its new hash. Here the store orders the two writes on the row. If this one runs first, - the row is revoked and the rotation's ``revoked_at IS NULL`` guard refuses it, so the - rotation fails closed. If the rotation committed first, the hash names no row and this - returns ``None``. + async def supersede_session( + self, token_hash: str, *, now: float, lineage_since: float | None = None + ) -> SessionRecord | None: + """Revoke the unrevoked session ``token_hash`` names, or the one it became, and return its + row as revoked. + + Returns ``None`` when nothing was revoked. A row that is expired or idle is still revoked + and returned. Its other columns are as the revoke found them, so the caller can judge + whether the session was live, and ``revoked_at`` is ``now``. Its ``token_hash`` is the hash + the session ended under, which differs from ``token_hash`` when a link was followed. + + **ONE store transaction, so it cannot interleave with :meth:`rotate_session` (BACKLOG + #2146).** The login supersession used to read the row and then revoke it by hash, and a + rotation that re-keyed the row between the two left the revoke matching nothing. Here the + store orders the two writes on the row. If this one runs first, the row is revoked and the + rotation's ``revoked_at IS NULL`` guard refuses it, so the rotation fails closed. + + **A rotation that committed FIRST is followed, when ``lineage_since`` is given (vault + BACKLOG #3028, ADR 0210).** The presented hash then names no row, so the revoke steps along + the links :meth:`rotate_session` wrote, over as many rotations as there were, and ends the + live end, all inside this one transaction. A link retired before ``lineage_since`` is not + followed, so a retired hash leads anywhere only for a bounded time; the caller chooses the + bound, because the store reads no settings. A rotation that commits during the walk costs + the walk one more step, never the revoke: ``store/session_lineage.py`` says why. With + ``lineage_since`` left ``None`` no link is followed, and a rotation that committed first + leaves this returning ``None``, as before. The lineage lookup lives here and nowhere else: + no session READ follows a link, so a retired hash authenticates nothing. """ ... @@ -2900,12 +2920,20 @@ async def enforce_session_cap( ... async def purge_expired_sessions( - self, *, now: float | None = None, idle_seconds: float | None = None + self, + *, + now: float | None = None, + idle_seconds: float | None = None, + lineage_seconds: float | None = None, ) -> int: """Delete session rows past their absolute expiry, revoked or not, and return the count. With ``idle_seconds`` given, rows idle for longer are deleted too (BACKLOG #2096): the validator refuses them on presentation, so keeping them only grows the table. + With ``lineage_seconds`` given, session lineage links retired more than that long before + ``now`` are deleted too (vault BACKLOG #3028). The supersession no longer follows them, so + keeping them only grows the table. They are not counted in the return. + The idle variant scans the table, because its ``OR`` reaches past the expiry index, and that stands on purpose (BACKLOG #2283). The scan runs once per reaper pass over a table the purge itself keeps small. The alternative is an index on ``last_used_at``, and every diff --git a/messagefoundry/store/postgres.py b/messagefoundry/store/postgres.py index 364768e87d..2fc1e399ba 100644 --- a/messagefoundry/store/postgres.py +++ b/messagefoundry/store/postgres.py @@ -182,6 +182,7 @@ new_state_cache, sealed_reference_set, ) +from messagefoundry.store.session_lineage import revoke_through_lineage from messagefoundry.store.store import ( _ACTIVE_ALERT_STATUS_SQL, _ALERT_SEVERITY_RANK_SQL, @@ -918,6 +919,15 @@ def __init__(self, method: str, outbox_ids: tuple[str, ...]) -> None: )""", "CREATE INDEX IF NOT EXISTS ix_sessions_user ON sessions(user_id)", "CREATE INDEX IF NOT EXISTS ix_sessions_expires ON sessions(expires_at)", + # Which token hash replaced a retired one (vault BACKLOG #3028, ADR 0210). The SQLite `_SCHEMA` + # says who reads it and why it is unsealed. Adding this DDL moves _schema_hash() automatically + # -- the ADR 0064 bump. + """CREATE TABLE IF NOT EXISTS session_lineage ( + retired_hash TEXT PRIMARY KEY, + successor_hash TEXT NOT NULL, + retired_at DOUBLE PRECISION NOT NULL + )""", + "CREATE INDEX IF NOT EXISTS ix_session_lineage_retired ON session_lineage(retired_at)", # WebAuthn passkeys (WP-14b, ADR 0068 §4). credential_id_hash — sha256 hex of the raw credential # id (the sessions.token_hash precedent) — is the parity-safe PK on all 3 backends; sign_count is # BIGINT (a WebAuthn uint32 overflows signed INT); label is column-capped so ux_webauthn_label @@ -1178,6 +1188,15 @@ def _pg_session_cap_keep_sql(split_mfa_pending: bool) -> str: " client=$6, reauth_at=$7, mfa_verified_at=$8, auth_mechanism=$9, idp_auth_time=$10," " user_epoch=$11, seal=$12 WHERE token_hash=$13 AND seal=$14" ) +# Vault BACKLOG #3028: store.py's three session lineage statements respelled for `$n`. store.py says +# which method runs each, and that no other statement names the table. +_PG_SESSION_LINEAGE_INSERT: Final = ( + "INSERT INTO session_lineage (retired_hash, successor_hash, retired_at) VALUES ($1,$2,$3)" +) +_PG_SESSION_LINEAGE_SUCCESSOR: Final = ( + "SELECT successor_hash FROM session_lineage WHERE retired_hash=$1 AND retired_at >= $2" +) +_PG_SESSION_LINEAGE_PURGE: Final = "DELETE FROM session_lineage WHERE retired_at < $1" #: The whole-account session sweep: revoke and blank the seal (BACKLOG #2592). Each caller raises #: the account's session epoch in the same transaction. _PG_REVOKE_USER_SESSIONS_SQL: Final = ( @@ -9740,16 +9759,22 @@ async def mark_session_mfa_verified(self, token_hash: str, *, now: float | None async with self._timed_acquire(record=False) as conn, conn.transaction(): await self._write_session(conn, token_hash, stamp) - async def rotate_session(self, token_hash: str, *, new_token_hash: str) -> bool: + async def rotate_session( + self, token_hash: str, *, new_token_hash: str, now: float | None = None + ) -> bool: """Re-key a live session in place (ASVS 7.2.4). See :meth:`AuthStore.rotate_session`. The token hash is sealed, so the re-key goes through :meth:`_write_session` (BACKLOG - #2592). The answer is read after the transaction has committed. The borrow is + #2592). The lineage link goes in the same transaction, only when the re-key landed (vault + BACKLOG #3028). The answer is read after the transaction has committed. The borrow is :meth:`_after_commit_txn`'s, for the reason it gives.""" + now = time.time() if now is None else now async with self._after_commit_txn() as conn: rotated = await self._write_session( conn, token_hash, lambda s: replace(s, token_hash=new_token_hash) ) + if rotated is not None: + await conn.execute(_PG_SESSION_LINEAGE_INSERT, token_hash, new_token_hash, now) return rotated is not None async def revoke_session(self, token_hash: str, *, now: float | None = None) -> None: @@ -9760,15 +9785,33 @@ async def revoke_session(self, token_hash: str, *, now: float | None = None) -> token_hash, ) - async def supersede_session(self, token_hash: str, *, now: float) -> SessionRecord | None: - """Revoke and return in one statement (BACKLOG #2146). See :meth:`AuthStore.supersede_session`.""" - d = await self._fetchone( - "UPDATE sessions SET revoked_at=$1, seal='' WHERE token_hash=$2 AND revoked_at IS NULL" - " RETURNING *", - now, - token_hash, - ) - return SessionRecord.from_mapping(dict(d)) if d else None + async def supersede_session( + self, token_hash: str, *, now: float, lineage_since: float | None = None + ) -> SessionRecord | None: + """Revoke and return, one statement per step, in one transaction (BACKLOG #2146, vault + BACKLOG #3028). See :meth:`AuthStore.supersede_session`. + + Each step's ``UPDATE ... RETURNING`` locks the row it ends. Under READ COMMITTED each + statement reads what has committed when it starts, so a rotation that commits while the + walk runs is seen by the next step's lineage read. The borrow stays bounded, as it was.""" + async with self._timed_acquire(record=False) as conn, conn.transaction(): + + async def _revoke(h: str) -> SessionRecord | None: + d = await conn.fetchrow( + "UPDATE sessions SET revoked_at=$1, seal='' WHERE token_hash=$2" + " AND revoked_at IS NULL RETURNING *", + now, + h, + ) + return SessionRecord.from_mapping(dict(d)) if d else None + + async def _successor(h: str, since: float) -> str | None: + found = await conn.fetchval(_PG_SESSION_LINEAGE_SUCCESSOR, h, since) + return None if found is None else str(found) + + return await revoke_through_lineage( + token_hash, lineage_since=lineage_since, revoke=_revoke, successor=_successor + ) async def revoke_user_sessions( self, user_id: str, *, except_token_hash: str | None = None, now: float | None = None @@ -9829,19 +9872,27 @@ async def enforce_session_cap( ) async def purge_expired_sessions( - self, *, now: float | None = None, idle_seconds: float | None = None + self, + *, + now: float | None = None, + idle_seconds: float | None = None, + lineage_seconds: float | None = None, ) -> int: now = time.time() if now is None else now # Through `_execute`, the bounded borrow, rather than `self._pool.execute`, which acquires # with no timeout (BACKLOG #1052). `_execute` borrows with `record=False`, which keeps this # hourly sweep out of the worker acquire-wait curve, and returns the count (BACKLOG #2283). if idle_seconds is None: - return await self._execute("DELETE FROM sessions WHERE expires_at < $1", now) - return await self._execute( - "DELETE FROM sessions WHERE expires_at < $1 OR $1 - last_used_at > $2", - now, - float(idle_seconds), - ) + purged = await self._execute("DELETE FROM sessions WHERE expires_at < $1", now) + else: + purged = await self._execute( + "DELETE FROM sessions WHERE expires_at < $1 OR $1 - last_used_at > $2", + now, + float(idle_seconds), + ) + if lineage_seconds is not None: + await self._execute(_PG_SESSION_LINEAGE_PURGE, now - float(lineage_seconds)) + return purged # --- retention / purge + maintenance (PHI.md §8) ------------------------- diff --git a/messagefoundry/store/session_lineage.py b/messagefoundry/store/session_lineage.py new file mode 100644 index 0000000000..b55d40b364 --- /dev/null +++ b/messagefoundry/store/session_lineage.py @@ -0,0 +1,69 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +# Copyright (C) 2026 MessageFoundry Foundation, LLC and contributors +"""The session lineage: which token hash replaced a retired one (vault BACKLOG #3028, ADR 0210). + +A rotation re-keys a session row to a new token hash, so the old hash stops naming any row. Each +backend's ``rotate_session`` now writes one ``session_lineage`` row in the same transaction: the +retired hash, the hash that replaced it, and when. This module holds the one walk over those rows +that every backend shares, so the three cannot disagree on how far it goes. + +**A retired hash authenticates nothing.** No session read consults this table: ``get_session``, +``list_sessions`` and the service's session loader read ``sessions`` only. The one reader is the +revoke below, so the only thing a retired hash can lead to is the END of the session it became. + +The rows are NOT sealed under BACKLOG #2592, deliberately. A seal stops a database writer forging +authority, and a link grants none: the most a forged link can do is direct a revoke at a live row, +and a revoke needs no key on any backend (it blanks the row's seal). A writer who can forge a link +can already revoke that row directly. +""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable +from typing import TYPE_CHECKING, Final + +if TYPE_CHECKING: # store.py imports this module, so the record type is for the checker only + from messagefoundry.store.store import SessionRecord + +#: How many links one revoke follows before it gives up and ends nothing. A link is written only by +#: a completed elevation, and every link it follows is inside the caller's window, so a chain this +#: long means that many credential ceremonies on one session inside that window. Bounded so a +#: revoke's work is bounded whatever the table holds. +LINEAGE_MAX_HOPS: Final = 32 + +#: Revoke the unrevoked row that hash names and return it as revoked, or ``None`` when none did. +type Revoke = Callable[[str], Awaitable[SessionRecord | None]] +#: The hash that replaced this one, by a link retired at or after ``since``, or ``None``. +type Successor = Callable[[str, float], Awaitable[str | None]] + + +async def revoke_through_lineage( + token_hash: str, + *, + lineage_since: float | None, + revoke: Revoke, + successor: Successor, +) -> SessionRecord | None: + """Revoke the session ``token_hash`` names or, when it names none, the session it BECAME. + + Run it inside ONE store transaction, with ``revoke`` and ``successor`` bound to that + transaction. Each step first tries to revoke the hash it holds; only when that revokes nothing + does it read the hash's successor and step to it. So a rotation that commits between a + ``successor`` read and the next ``revoke`` costs one more step rather than the revoke: the + re-keyed row's old hash then has a link of its own, written in the rotation's transaction. + + ``lineage_since`` is the oldest ``retired_at`` a followed link may carry. ``None`` follows no + link at all, which is the plain revoke of the hash given (BACKLOG #2146's behaviour). + + Returns the row it revoked, whose ``token_hash`` is the hash it ended under, or ``None`` when + the walk reached a hash with no unrevoked row and no live link, or ran out of hops.""" + current = token_hash + for _ in range(LINEAGE_MAX_HOPS + 1): + ended = await revoke(current) + if ended is not None or lineage_since is None: + return ended + following = await successor(current, lineage_since) + if following is None: + return None + current = following + return None diff --git a/messagefoundry/store/sqlserver.py b/messagefoundry/store/sqlserver.py index fb8cd6ec8a..1eb4895b82 100644 --- a/messagefoundry/store/sqlserver.py +++ b/messagefoundry/store/sqlserver.py @@ -161,12 +161,16 @@ new_state_cache, sealed_reference_set, ) +from messagefoundry.store.session_lineage import revoke_through_lineage from messagefoundry.store.store import ( _ACTIVE_ALERT_STATUS_SQL, _ALERT_SEVERITY_RANK_SQL, _SESSION_CAP_ORDER_SQL, _SESSION_CAP_RANK_NOT_AHEAD_SQL, _SESSION_INSERT, + _SESSION_LINEAGE_INSERT, + _SESSION_LINEAGE_PURGE, + _SESSION_LINEAGE_SUCCESSOR, _SESSION_LIVE_SQL, _SESSION_NOT_AHEAD_SQL, _SQL_ADMIN_GUARD_HOLDERS, @@ -2404,6 +2408,14 @@ def __init__(self, conn: Any, cur: Any) -> None: CREATE INDEX ix_sessions_user ON sessions(user_id)""", """IF INDEXPROPERTY(OBJECT_ID('sessions'),'ix_sessions_expires','IndexID') IS NULL CREATE INDEX ix_sessions_expires ON sessions(expires_at)""", + # Which token hash replaced a retired one (vault BACKLOG #3028, ADR 0210). The SQLite `_SCHEMA` + # says who reads it and why it is unsealed. Both hashes are 64-character hex digests, the + # sessions.token_hash width. Adding this DDL moves _schema_hash() -- the ADR 0064 bump. + """IF OBJECT_ID('session_lineage','U') IS NULL CREATE TABLE session_lineage ( + retired_hash NVARCHAR(64) NOT NULL PRIMARY KEY, + successor_hash NVARCHAR(64) NOT NULL, retired_at FLOAT NOT NULL)""", + """IF INDEXPROPERTY(OBJECT_ID('session_lineage'),'ix_session_lineage_retired','IndexID') IS NULL + CREATE INDEX ix_session_lineage_retired ON session_lineage(retired_at)""", # WebAuthn passkeys (WP-14b, ADR 0068 §4). credential_id_hash — sha256 hex of the RAW credential # id (the sessions.token_hash precedent) — is the PK: raw WebAuthn ids may be up to 1023 bytes, # unboundable as an index key here (NVARCHAR(MAX) can't be a PK/index key), so the fixed-width @@ -14000,12 +14012,19 @@ async def _write_session( return updated if written else None async def _write_session_txn( - self, token_hash: str, change: Callable[[SessionRecord], SessionRecord] + self, + token_hash: str, + change: Callable[[SessionRecord], SessionRecord], + *, + landed: Callable[[Any], Awaitable[None]] | None = None, ) -> SessionRecord | None: - """:meth:`_write_session` in a transaction of its own, committed before it returns.""" + """:meth:`_write_session` in a transaction of its own, committed before it returns. + ``landed`` runs on the same cursor, in the same transaction, only when the write landed.""" async with self._acquire() as conn, self._cursor(conn) as cur: try: updated = await self._write_session(cur, token_hash, change) + if updated is not None and landed is not None: + await landed(cur) await self._commit(conn) except Exception: await conn.rollback() @@ -14061,15 +14080,23 @@ async def mark_session_reauthed( async def mark_session_mfa_verified(self, token_hash: str, *, now: float | None = None) -> None: await self._write_session_txn(token_hash, mfa_stamp(time.time() if now is None else now)) - async def rotate_session(self, token_hash: str, *, new_token_hash: str) -> bool: + async def rotate_session( + self, token_hash: str, *, new_token_hash: str, now: float | None = None + ) -> bool: """Re-key a live session in place (ASVS 7.2.4). See :meth:`AuthStore.rotate_session`. The token hash is sealed, so the re-key goes through :meth:`_write_session` (BACKLOG #2592), whose write reads an ``OUTPUT`` rowset and never ``cursor.rowcount`` (BACKLOG #2283). ``token_hash`` is ``NVARCHAR(64)`` and a hex digest is exactly 64 chars, so the - clustered-PK row move is width-safe.""" + clustered-PK row move is width-safe. The lineage link goes in the same transaction, only + when the re-key landed (vault BACKLOG #3028).""" + now = time.time() if now is None else now + + async def _link(cur: Any) -> None: + await cur.execute(_SESSION_LINEAGE_INSERT, (token_hash, new_token_hash, now)) + rotated = await self._write_session_txn( - token_hash, lambda s: replace(s, token_hash=new_token_hash) + token_hash, lambda s: replace(s, token_hash=new_token_hash), landed=_link ) return rotated is not None @@ -14080,17 +14107,41 @@ async def revoke_session(self, token_hash: str, *, now: float | None = None) -> (now, token_hash), ) - async def supersede_session(self, token_hash: str, *, now: float) -> SessionRecord | None: - """Revoke and return in one statement (BACKLOG #2146). See :meth:`AuthStore.supersede_session`. + async def supersede_session( + self, token_hash: str, *, now: float, lineage_since: float | None = None + ) -> SessionRecord | None: + """Revoke and return, one statement per step, in one transaction (BACKLOG #2146, vault + BACKLOG #3028). See :meth:`AuthStore.supersede_session`. - Through ``_execute_output`` rather than ``_fetchone``, which commits as a READ, while this is - a durable write (BACKLOG #2283).""" - rows = await self._execute_output( - "UPDATE sessions SET revoked_at=?, seal='' OUTPUT inserted.*" - " WHERE token_hash=? AND revoked_at IS NULL", - (now, token_hash), - ) - return SessionRecord.from_mapping(rows[0]) if rows else None + Each step's ``UPDATE ... OUTPUT`` takes the row's lock, and its rowset says what it ended, + which a session-wide ``SET NOCOUNT ON`` cannot hide (BACKLOG #2283). Each statement reads + what has committed when it runs, so a rotation that commits while the walk runs is seen by + the next step's lineage read. Committed as a durable write, never as a READ.""" + async with self._acquire() as conn, self._cursor(conn) as cur: + + async def _revoke(h: str) -> SessionRecord | None: + await cur.execute( + "UPDATE sessions SET revoked_at=?, seal='' OUTPUT inserted.*" + " WHERE token_hash=? AND revoked_at IS NULL", + (now, h), + ) + rows = await self._fetch_dicts(cur) + return SessionRecord.from_mapping(rows[0]) if rows else None + + async def _successor(h: str, since: float) -> str | None: + await cur.execute(_SESSION_LINEAGE_SUCCESSOR, (h, since)) + rows = await cur.fetchall() + return str(rows[0][0]) if rows else None + + try: + ended = await revoke_through_lineage( + token_hash, lineage_since=lineage_since, revoke=_revoke, successor=_successor + ) + await self._commit(conn) + except Exception: + await conn.rollback() + raise + return ended async def revoke_user_sessions( self, user_id: str, *, except_token_hash: str | None = None, now: float | None = None @@ -14164,7 +14215,11 @@ async def enforce_session_cap( return len(rows) async def purge_expired_sessions( - self, *, now: float | None = None, idle_seconds: float | None = None + self, + *, + now: float | None = None, + idle_seconds: float | None = None, + lineage_seconds: float | None = None, ) -> int: now = time.time() if now is None else now if idle_seconds is None: @@ -14178,7 +14233,12 @@ async def purge_expired_sessions( # `revoke_user_sessions` this keeps the driver's count, which a session-wide SET NOCOUNT ON # can turn to -1: the reaper ignores it and decides nothing on it, and an OUTPUT rowset # would carry every purged row back each pass for a number nobody acts on. - return await self._execute(sql, params) + purged = await self._execute(sql, params) + if lineage_seconds is not None: + # Its own statement and transaction, as on Postgres: a link is table hygiene, and the + # supersession stops following one at its window whether or not this has run. + await self._execute(_SESSION_LINEAGE_PURGE, (now - float(lineage_seconds),)) + return purged async def stats(self) -> dict[str, int]: rows = await self._fetchall( diff --git a/messagefoundry/store/store.py b/messagefoundry/store/store.py index e79110fc4a..8d60e2b114 100644 --- a/messagefoundry/store/store.py +++ b/messagefoundry/store/store.py @@ -183,6 +183,7 @@ new_state_cache, sealed_reference_set, ) +from messagefoundry.store.session_lineage import revoke_through_lineage from messagefoundry.store.transit_attestation import ( TRANSIT_BOUND_ATTESTED_ACTION, TRANSIT_BOUND_WITHDRAWN_ACTION, @@ -1277,6 +1278,18 @@ class ConnectionEventWrite(TypedDict): " seal=? WHERE token_hash=? AND seal=?" ) +# Vault BACKLOG #3028, ADR 0210: the session lineage's three statements, in `?` form for SQLite and +# SQL Server; postgres.py respells them for `$n`. The INSERT runs only inside rotate_session's own +# transaction, the SELECT only inside supersede_session's, and the DELETE only from the session +# purge. No other statement names the table: a retired hash must authenticate nothing. +_SESSION_LINEAGE_INSERT: Final = ( + "INSERT INTO session_lineage (retired_hash, successor_hash, retired_at) VALUES (?,?,?)" +) +_SESSION_LINEAGE_SUCCESSOR: Final = ( + "SELECT successor_hash FROM session_lineage WHERE retired_hash=? AND retired_at >= ?" +) +_SESSION_LINEAGE_PURGE: Final = "DELETE FROM session_lineage WHERE retired_at < ?" + def session_reseal_params(new: SessionRecord, old: SessionRecord) -> tuple[object, ...]: """The parameters :data:`_SESSION_RESEAL` binds, in order: ``new``'s sealed columns and seal, @@ -6960,6 +6973,17 @@ def _dead_target_pairs(rows: Iterable[tuple[Any, Any]]) -> list[tuple[str, str]] CREATE INDEX IF NOT EXISTS ix_sessions_user ON sessions(user_id); CREATE INDEX IF NOT EXISTS ix_sessions_expires ON sessions(expires_at); +-- Which token hash replaced a retired one, written by rotate_session in its own transaction (vault +-- BACKLOG #3028, ADR 0210). Read by the login supersession ONLY, never by a session read, so a +-- retired hash authenticates nothing. Not sealed: store/session_lineage.py says why. No FK, and the +-- session reaper purges rows past the window the supersession follows them for. +CREATE TABLE IF NOT EXISTS session_lineage ( + retired_hash TEXT PRIMARY KEY, -- the hash the rotation retired + successor_hash TEXT NOT NULL, -- the hash the session carried next + retired_at REAL NOT NULL -- when the rotation committed (wall clock) +); +CREATE INDEX IF NOT EXISTS ix_session_lineage_retired ON session_lineage(retired_at); + CREATE TABLE IF NOT EXISTS webauthn_credentials ( credential_id_hash TEXT PRIMARY KEY, -- sha256 hex of the RAW credential id (sessions.token_hash -- precedent): WebAuthn ids may be up to 1023 raw bytes — @@ -15823,17 +15847,23 @@ async def mark_session_mfa_verified(self, token_hash: str, *, now: float | None await self._write_session(token_hash, stamp) await self._commit() - async def rotate_session(self, token_hash: str, *, new_token_hash: str) -> bool: + async def rotate_session( + self, token_hash: str, *, new_token_hash: str, now: float | None = None + ) -> bool: """Re-key a live session in place (ASVS 7.2.4). See :meth:`AuthStore.rotate_session`. ``token_hash`` is the PK, so this is a key update: the row keeps every other column and the old hash stops resolving in the same transaction. The token hash is sealed, so the re-key re-seals (BACKLOG #2592). Reports whether a row was re-keyed so the caller can fail closed - when the session was revoked out from under it.""" + when the session was revoked out from under it. The lineage link is written in the same + transaction, and only when the re-key landed (vault BACKLOG #3028).""" + now = time.time() if now is None else now async with _writer_txn(self._db, self._lock): rotated = await self._write_session( token_hash, lambda s: replace(s, token_hash=new_token_hash) ) + if rotated is not None: + await self._db.execute(_SESSION_LINEAGE_INSERT, (token_hash, new_token_hash, now)) await self._commit() return rotated is not None @@ -15846,28 +15876,41 @@ async def revoke_session(self, token_hash: str, *, now: float | None = None) -> ) await self._commit() - async def supersede_session(self, token_hash: str, *, now: float) -> SessionRecord | None: - """Revoke, then read the row back in the SAME transaction (BACKLOG #2146). See + async def supersede_session( + self, token_hash: str, *, now: float, lineage_since: float | None = None + ) -> SessionRecord | None: + """Revoke, then read the row back in the SAME transaction (BACKLOG #2146), following the + session lineage when ``lineage_since`` is given (vault BACKLOG #3028). See :meth:`AuthStore.supersede_session`. - Two statements here rather than one ``UPDATE ... RETURNING``, because ``RETURNING`` needs - SQLite 3.35 and a Linux Python can link an older system library. They are still one unit: - the writer lock orders them against every in-process writer, ``rotate_session`` included, - and the write lock the ``UPDATE`` takes holds until the commit, so no other process can - re-key the row between them.""" - async with _writer_guard(self._db, self._lock): + Two statements per step rather than one ``UPDATE ... RETURNING``, because ``RETURNING`` + needs SQLite 3.35 and a Linux Python can link an older system library. They are still one + unit: the writer lock orders them against every in-process writer, ``rotate_session`` + included. The first ``UPDATE`` takes SQLite's write lock even when it matches no row, and + holds it until the commit, so no other process can re-key a row while the walk runs.""" + + async def _revoke(h: str) -> SessionRecord | None: cur = await self._db.execute( "UPDATE sessions SET revoked_at=?, seal='' WHERE token_hash=? AND revoked_at IS NULL", - (now, token_hash), + (now, h), + ) + if not cur.rowcount: + return None + cur = await self._db.execute("SELECT * FROM sessions WHERE token_hash=?", (h,)) + row = await cur.fetchone() + return SessionRecord.from_mapping(dict(row)) if row else None + + async def _successor(h: str, since: float) -> str | None: + cur = await self._db.execute(_SESSION_LINEAGE_SUCCESSOR, (h, since)) + row = await cur.fetchone() + return str(row[0]) if row else None + + async with _writer_guard(self._db, self._lock): + ended = await revoke_through_lineage( + token_hash, lineage_since=lineage_since, revoke=_revoke, successor=_successor ) - row = None - if cur.rowcount: - cur = await self._db.execute( - "SELECT * FROM sessions WHERE token_hash=?", (token_hash,) - ) - row = await cur.fetchone() await self._commit() - return SessionRecord.from_mapping(dict(row)) if row else None + return ended async def revoke_user_sessions( self, user_id: str, *, except_token_hash: str | None = None, now: float | None = None @@ -15947,7 +15990,11 @@ async def enforce_session_cap( return int(cur.rowcount) async def purge_expired_sessions( - self, *, now: float | None = None, idle_seconds: float | None = None + self, + *, + now: float | None = None, + idle_seconds: float | None = None, + lineage_seconds: float | None = None, ) -> int: now = time.time() if now is None else now async with _writer_guard(self._db, self._lock): @@ -15958,8 +16005,11 @@ async def purge_expired_sessions( "DELETE FROM sessions WHERE expires_at < ? OR ? - last_used_at > ?", (now, now, float(idle_seconds)), ) + purged = cur.rowcount if cur.rowcount is not None else 0 + if lineage_seconds is not None: + await self._db.execute(_SESSION_LINEAGE_PURGE, (now - float(lineage_seconds),)) await self._commit() - return cur.rowcount if cur.rowcount is not None else 0 + return purged async def db_status(self) -> DbStatus: """Database health snapshot (size, free space, journal mode, row counts). diff --git a/tests/_session_rotation_contract.py b/tests/_session_rotation_contract.py index c0124cbcaa..57850a2a3a 100644 --- a/tests/_session_rotation_contract.py +++ b/tests/_session_rotation_contract.py @@ -226,7 +226,7 @@ async def assert_session_supersession_contract(store: Any, *, user_id: str = "su # --- an unknown hash ------------------------------------------------------------------------ assert await store.supersede_session(_h(), now=now) is None - # --- the other order: a rotation that committed first leaves the old hash naming nothing ---- + # --- the other order, with no lineage window: a rotation that committed first wins --------- old, new = _h(), _h() await store.create_session(token_hash=old, user_id=user_id, expires_at=expires, now=now) assert await store.rotate_session(old, new_token_hash=new) is True @@ -235,3 +235,77 @@ async def assert_session_supersession_contract(store: Any, *, user_id: str = "su assert survivor is not None and survivor.revoked_at is None, ( "superseding a retired hash must not touch the session that now carries another hash" ) + + +async def assert_session_lineage_contract(store: Any, *, user_id: str = "lin-u1") -> None: + """Drive the session lineage through its contract on any backend (vault BACKLOG #3028). + + ``rotate_session`` links the hash it retired to its successor in its own transaction, and + ``supersede_session`` follows those links, inside a window its caller names, to the live end. + A retired hash still resolves to nothing, and the reaper's purge drops links past the window. + """ + now = time.time() + await store.create_user( + user_id=user_id, + username=f"lin-{user_id}", + auth_provider="local", + display_name=None, + email=None, + password_hash="h", + now=now, + password_generated=False, + ) + expires = now + 9876.5 + + # --- two rotations: a -> b -> c, and neither retired hash resolves ------------------------- + a, b, c = _h(), _h(), _h() + await store.create_session(token_hash=a, user_id=user_id, expires_at=expires, now=now) + assert await store.rotate_session(a, new_token_hash=b, now=now) is True + assert await store.rotate_session(b, new_token_hash=c, now=now + 1) is True + assert await store.get_session(a) is None, "a retired hash must resolve to nothing" + assert await store.get_session(b) is None, "a retired hash must resolve to nothing" + + # --- no window: the plain revoke of the hash given, as before ------------------------------ + assert await store.supersede_session(a, now=now + 2) is None + + # --- a window that starts after the first link: that link is not followed ------------------- + assert await store.supersede_session(a, now=now + 2, lineage_since=now + 0.5) is None + live = await store.get_session(c) + assert live is not None and live.revoked_at is None, "a link past the window was followed" + + # --- inside the window: followed over both hops, and the live end is what ends -------------- + ended = await store.supersede_session(a, now=now + 3, lineage_since=now - 10) + assert ended is not None, "the supersession did not follow the lineage to the live end" + assert ended.token_hash == c and ended.user_id == user_id and ended.revoked_at == now + 3 + stored = await store.get_session(c) + assert stored is not None and stored.revoked_at == now + 3, "the revoke was not persisted" + # The chain now ends at a revoked row, so a second supersession ends nothing. + assert await store.supersede_session(a, now=now + 4, lineage_since=now - 10) is None + # From the middle of the chain, too. + assert await store.supersede_session(b, now=now + 4, lineage_since=now - 10) is None + + # --- a rotation that did not land writes no link ------------------------------------------- + d = _h() + assert await store.rotate_session(c, new_token_hash=d, now=now + 5) is False + await store.create_session(token_hash=d, user_id=user_id, expires_at=expires, now=now) + assert await store.supersede_session(c, now=now + 6, lineage_since=now - 10) is None, ( + "a refused rotation linked its hash to a session it never became" + ) + kept = await store.get_session(d) + assert kept is not None and kept.revoked_at is None + + # --- the purge drops links past the window and keeps the rest ------------------------------ + old, old_next, fresh, fresh_next = _h(), _h(), _h(), _h() + await store.create_session(token_hash=old, user_id=user_id, expires_at=expires, now=now) + await store.create_session(token_hash=fresh, user_id=user_id, expires_at=expires, now=now) + assert await store.rotate_session(old, new_token_hash=old_next, now=now - 1000) is True + assert await store.rotate_session(fresh, new_token_hash=fresh_next, now=now) is True + await store.purge_expired_sessions(now=now) # no window: every link stays + await store.purge_expired_sessions(now=now, lineage_seconds=100) + assert await store.supersede_session(old, now=now, lineage_since=now - 10_000) is None, ( + "a link past the purge window survived the purge" + ) + survivor = await store.get_session(old_next) + assert survivor is not None and survivor.revoked_at is None + swept = await store.supersede_session(fresh, now=now + 7, lineage_since=now - 10) + assert swept is not None and swept.token_hash == fresh_next, "the purge dropped a live link" diff --git a/tests/test_auth_hardening.py b/tests/test_auth_hardening.py index 2a4a894cca..bcfcb1c2ca 100644 --- a/tests/test_auth_hardening.py +++ b/tests/test_auth_hardening.py @@ -893,11 +893,17 @@ async def _run_reaper( class _Store: async def purge_expired_sessions( - self, *, now: float | None = None, idle_seconds: float | None = None + self, + *, + now: float | None = None, + idle_seconds: float | None = None, + lineage_seconds: float | None = None, ) -> None: purged.append(now) if inner is not None: - await inner.purge_expired_sessions(now=now, idle_seconds=idle_seconds) + await inner.purge_expired_sessions( + now=now, idle_seconds=idle_seconds, lineage_seconds=lineage_seconds + ) def _wall() -> float: try: diff --git a/tests/test_postgres_store.py b/tests/test_postgres_store.py index c582b2790d..483a7ee438 100644 --- a/tests/test_postgres_store.py +++ b/tests/test_postgres_store.py @@ -89,6 +89,8 @@ "reference", "reference_version", "sessions", + "session_lineage", # vault BACKLOG #3028 (no FK): a link left behind would lead a later test's + # supersession into a session it never retired "webauthn_credentials", # ADR 0068: FK to users(id) — must clear before users "known_login_addresses", # vault BACKLOG #2145: FK to users(id), cleared before users "user_roles", @@ -5439,13 +5441,21 @@ async def fetch(sql: str, params: tuple[object, ...]) -> list[dict[str, object]] async def test_session_supersession_contract(store) -> None: - """BACKLOG #2146: supersede_session revokes and returns in one statement, so a rotation after it - fails closed. Extra-free shared contract, run here as on SQLite.""" + """BACKLOG #2146: supersede_session revokes and returns in one transaction, so a rotation after + it fails closed. Extra-free shared contract, run here as on SQLite.""" from tests._session_rotation_contract import assert_session_supersession_contract await assert_session_supersession_contract(store) +async def test_session_lineage_contract(store) -> None: + """Vault BACKLOG #3028: a rotation links the hash it retired, the supersession follows the + links inside its window, and the purge drops them past it. Extra-free shared contract.""" + from tests._session_rotation_contract import assert_session_lineage_contract + + await assert_session_lineage_contract(store) + + async def test_session_cap_contract(store) -> None: """BACKLOG #1900: the per-user cap counts only LIVE sessions. What this leg executes that no other does: the liveness clauses respelled for ``$n``, with ``$1`` reused in the UPDATE and in diff --git a/tests/test_session_sql_placeholders.py b/tests/test_session_sql_placeholders.py index 8951d2ead1..9aa1fdc012 100644 --- a/tests/test_session_sql_placeholders.py +++ b/tests/test_session_sql_placeholders.py @@ -57,6 +57,10 @@ "rotate_session": lambda s: s.rotate_session(_HASH, new_token_hash=_OTHER), "revoke_session": lambda s: s.revoke_session(_HASH, now=_NOW), "supersede_session": lambda s: s.supersede_session(_HASH, now=_NOW), + # Vault BACKLOG #3028: nothing is revoked, so the walk reads the lineage once and stops. + "supersede_session_lineage": lambda s: s.supersede_session( + _HASH, now=_NOW, lineage_since=_NOW - 360 + ), "revoke_user_sessions": lambda s: s.revoke_user_sessions("u", now=_NOW), "revoke_user_sessions_except": lambda s: s.revoke_user_sessions( "u", except_token_hash=_HASH, now=_NOW @@ -69,6 +73,9 @@ ), "purge_expired_sessions": lambda s: s.purge_expired_sessions(now=_NOW), "purge_expired_sessions_idle": lambda s: s.purge_expired_sessions(now=_NOW, idle_seconds=1800), + "purge_expired_sessions_lineage": lambda s: s.purge_expired_sessions( + now=_NOW, idle_seconds=1800, lineage_seconds=360 + ), } @@ -221,7 +228,11 @@ async def test_sqlserver_rotate_reads_its_output_rowset_not_the_row_count( store = _nocount_store(seen, output_rows) assert await store.rotate_session(_HASH, new_token_hash=_OTHER) is bool(output_rows) # The read comes first (BACKLOG #2592); the rewrite is the statement that reports. - assert "OUTPUT inserted.token_hash" in seen[-1][0] + assert "OUTPUT inserted.token_hash" in seen[1][0] + # The lineage link follows the rewrite only when the rowset says it landed (vault BACKLOG + # #3028), so a refused rotation links nothing. + links = [sql for sql, _ in seen if sql.startswith("INSERT INTO session_lineage")] + assert len(links) == (1 if output_rows else 0) @pytest.mark.parametrize("revoked", [0, 3]) @@ -332,6 +343,10 @@ async def fetchrow(self, sql: str, *args: Any) -> Any: return {"session_epoch": 0} return None + async def fetchval(self, sql: str, *args: Any) -> Any: + self._seen.append((sql, args)) + return None + def transaction(self) -> Any: @asynccontextmanager async def _txn() -> AsyncIterator[None]: diff --git a/tests/test_session_supersession.py b/tests/test_session_supersession.py index 8e77dc3276..94ef929342 100644 --- a/tests/test_session_supersession.py +++ b/tests/test_session_supersession.py @@ -207,7 +207,8 @@ async def test_a_rotation_racing_the_supersession_cannot_keep_the_session_live( # FIRST store call that names the presented hash, whichever call that is, so it measures the # gap rather than one method name. Once that call has run, the session must end: the rotation # fails closed, or its new token does not validate. A rotation that commits BEFORE that call is - # the stated limit, pinned in tests/_session_rotation_contract.py. If a later change reads the + # followed through the session lineage (vault BACKLOG #3028), which + # tests/test_session_supersession_lineage.py pins. If a later change reads the # presented hash earlier in the sign-in, this fires there, and that read must not decide the # revoke, or the gap is back. service = await _service(store) @@ -258,11 +259,13 @@ def _stepping() -> float: reads[0] += 1 return base + reads[0] * 0.001 - async def _touched_first(token_hash: str, *, now: float) -> object: + async def _touched_first( + token_hash: str, *, now: float, lineage_since: float | None = None + ) -> object: touched = time.time() assert touched > now, "the stepping clock did not separate the touch from the revoke stamp" await store.touch_session(token_hash, now=touched) - return await real_supersede(token_hash, now=now) + return await real_supersede(token_hash, now=now, lineage_since=lineage_since) monkeypatch.setattr(store, "supersede_session", _touched_first) monkeypatch.setattr(time, "time", _stepping) diff --git a/tests/test_session_supersession_lineage.py b/tests/test_session_supersession_lineage.py new file mode 100644 index 0000000000..8900922698 --- /dev/null +++ b/tests/test_session_supersession_lineage.py @@ -0,0 +1,289 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +# Copyright (C) 2026 MessageFoundry Foundation, LLC and contributors +"""A sign-in ends the session its presented token BECAME, for a bounded time (vault BACKLOG #3028). + +Engine PR 2028 (BACKLOG #2146) made the supersession's read and revoke one store operation, so a +rotation that starts after it fails closed. It left one case open: an elevation that FINISHED before +it. The presented hash then named no session, the supersession ended nothing, and the session lived +on under its new token. A rotation now records a link from the retired hash to its successor, and +the supersession follows that link to the live end, inside its own store transaction. + +These pin the service half. The store half, on every backend, is +``tests/_session_rotation_contract.py``'s ``assert_session_lineage_contract``. +""" + +from __future__ import annotations + +import ast +import re +import sqlite3 +import time +from collections.abc import AsyncIterator, Awaitable, Callable +from contextlib import closing +from pathlib import Path + +import pytest + +from messagefoundry.auth.service import AuthService +from messagefoundry.auth.tokens import hash_token +from messagefoundry.store.session_lineage import LINEAGE_MAX_HOPS, revoke_through_lineage +from messagefoundry.store.store import MessageStore, SessionRecord +from tests._ast_sites import code_names, code_strings, parse_source +from tests.test_session_supersession import PW, _live, _service, _superseded, _token + + +@pytest.fixture +async def store(tmp_path: Path) -> AsyncIterator[MessageStore]: + s = await MessageStore.open(tmp_path / "lineage.db") + yield s + await s.close() + + +async def _elevate(service: AuthService, token: str) -> str: + """Complete a real password step-up on ``token`` and return the token it was rotated to.""" + identity = await service.identity_for_token(token, activity=False) + assert identity is not None + elevation = await service.reauth(identity, PW, token=token) + assert elevation.ok and elevation.token is not None, elevation + assert elevation.token != token + return elevation.token + + +async def test_an_elevation_that_finished_before_the_sign_in_does_not_keep_the_session_live( + store: MessageStore, +) -> None: + # Closing item 3 of the row. The elevation runs to completion, so its rotation has committed + # before the sign-in's supersession statement runs. The browser still presents the OLD token, + # which now names no session row. + service = await _service(store) + prior = await _token(service) + rotated = await _elevate(service, prior) + assert await store.get_session(hash_token(prior)) is None, "the rotation did not retire it" + assert await _live(service, rotated) + + new = await _token(service, supersedes=prior) + + assert not await _live(service, rotated), ( + "the sign-in replaced this browser's session, but the session lives on under its new token" + ) + assert await _live(service, new) + assert await _superseded(store) == [ + { + "scope": "superseded", + "session": hash_token(rotated)[:12], + "presented": hash_token(prior)[:12], + "actor": "op", + } + ] + + +async def test_the_link_is_followed_over_several_elevations(store: MessageStore) -> None: + service = await _service(store) + prior = await _token(service) + end = prior + for _ in range(3): + end = await _elevate(service, end) + await _token(service, supersedes=prior) + assert not await _live(service, end), "the chain was not followed to its live end" + + +async def test_a_link_older_than_the_window_is_not_followed( + store: MessageStore, monkeypatch: pytest.MonkeyPatch +) -> None: + service = await _service(store, session_idle_timeout_minutes=240) + window = service.supersession_lineage_seconds + assert window >= service.oidc_flow_ttl_seconds, "the window must outlast the longest sign-in" + prior = await _token(service) + rotated = await _elevate(service, prior) + real = time.time + monkeypatch.setattr(time, "time", lambda: real() + window + 5) + assert await _live(service, rotated) + await _token(service, supersedes=prior) + assert await _live(service, rotated), "a link past its window ended a session" + assert await _superseded(store) == [] + + +def _links(path: Path) -> list[tuple[str, str]]: + with closing(sqlite3.connect(path)) as db: + return [ + (str(r[0]), str(r[1])) + for r in db.execute("SELECT retired_hash, successor_hash FROM session_lineage") + ] + + +async def test_a_retired_token_authenticates_nothing_while_its_link_is_live( + store: MessageStore, tmp_path: Path +) -> None: + # Closing item 2. The link exists and is inside its window, and the retired token still + # resolves to nothing, on every path that reads a session by a presented token. + service = await _service(store) + prior = await _token(service) + rotated = await _elevate(service, prior) + assert _links(tmp_path / "lineage.db") == [(hash_token(prior), hash_token(rotated))] + assert await service.identity_for_token(prior) is None + assert await service.identity_for_token(prior, activity=False) is None + assert await store.get_session(hash_token(prior)) is None + assert await _live(service, rotated), "presenting the retired token ended its successor" + + +async def test_a_token_whose_hash_links_to_a_live_session_authenticates_nothing( + store: MessageStore, tmp_path: Path +) -> None: + # The same, from the other side: a link row a database writer forged from a hash whose token it + # holds to someone else's live session. The validator reads `sessions` only, so the forged + # token is refused and the session it points at is untouched. + service = await _service(store) + victim = await _token(service, "other") + forged = "a-token-nobody-was-ever-issued" + with closing(sqlite3.connect(tmp_path / "lineage.db")) as db: + db.execute( + "INSERT INTO session_lineage (retired_hash, successor_hash, retired_at) VALUES (?,?,?)", + (hash_token(forged), hash_token(victim), time.time()), + ) + db.commit() + assert await service.identity_for_token(forged) is None + assert await _live(service, victim) + + +_BACKENDS = ("store.py", "postgres.py", "sqlserver.py") +_STORE_DIR = Path(__file__).resolve().parents[1] / "messagefoundry" / "store" +_LINEAGE = re.compile(r"\bsession_lineage\b|SESSION_LINEAGE_|\brevoke_through_lineage\b") + + +def _functions_naming_the_lineage(source: str) -> set[str]: + """Every top-level function or method whose CODE names the lineage table, one of its + statements, or the shared walk; a docstring or comment that mentions them does not count. A + nested function counts as the method it sits in.""" + tree = parse_source(source) + named: set[str] = set() + tops: list[ast.stmt] = list(tree.body) + for cls in tree.body: + if isinstance(cls, ast.ClassDef): + tops.extend(cls.body) + for node in tops: + if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef) and any( + _LINEAGE.search(text) for text in (*code_names(node), *code_strings(node)) + ): + named.add(node.name) + return named + + +@pytest.mark.parametrize("backend", _BACKENDS) +def test_only_the_rotation_the_supersession_and_the_purge_touch_the_lineage(backend: str) -> None: + # Closing item 2, structurally: no session READ can consult the lineage, on any backend, + # because no method but these three names the table or a statement on it. + source = (_STORE_DIR / backend).read_text(encoding="utf-8") + assert _functions_naming_the_lineage(source) == { + "rotate_session", + "supersede_session", + "purge_expired_sessions", + } + + +def test_the_service_follows_the_lineage_from_the_supersession_alone() -> None: + # `_load_session` and the validator never pass a lineage window; only the sign-in's + # supersession does. `_end_lapsed_session` shares the store call and passes none. + source = (Path(__file__).resolve().parents[1] / "messagefoundry/auth/service.py").read_text( + encoding="utf-8" + ) + tree = ast.parse(source) + passing: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.AsyncFunctionDef | ast.FunctionDef): + for call in ast.walk(node): + if isinstance(call, ast.Call) and any( + k.arg == "lineage_since" for k in call.keywords + ): + passing.add(node.name) + assert passing == {"_supersede_session_hash"} + assert not _LINEAGE.search(source) + + +async def test_the_lineage_reaches_no_further_than_one_supersession(store: MessageStore) -> None: + # A followed link ends the live end once. Presenting the retired token again finds a revoked row + # and writes nothing. + service = await _service(store) + prior = await _token(service) + await _elevate(service, prior) + await _token(service, supersedes=prior) + await _token(service, supersedes=prior) + assert len(await _superseded(store)) == 1 + + +async def test_the_lapsed_credential_revoke_does_not_follow_the_lineage( + store: MessageStore, +) -> None: + # `_end_lapsed_session` shares `supersede_session`. It revokes the row it was handed, and a row a + # rotation re-keyed first is not reported as ended; the next presentation of the new token is + # refused by the same gate. So it passes no lineage window. + service = await _service(store) + prior = await _token(service) + rotated = await _elevate(service, prior) + await service._end_lapsed_session(hash_token(prior), "op", at="session", client=None) + assert await _live(service, rotated) + + +# --- the shared walk, with no database: a rotation that lands mid-walk, and the hop bound ---------- + + +def _record(token_hash: str) -> SessionRecord: + return SessionRecord( + token_hash=token_hash, + user_id="u", + created_at=0.0, + expires_at=1.0, + last_used_at=0.0, + revoked_at=1.0, + client=None, + ) + + +async def test_a_rotation_landing_mid_walk_costs_a_step_not_the_revoke() -> None: + # On the server backends each step is its own statement, so a rotation can commit between a + # step's lineage read and the next step's revoke. Here `b` is re-keyed to `c` exactly then. The + # walk's revoke of `b` matches nothing, it reads `b`'s new link, and it ends `c`. + live = {"b"} + links = {"a": "b"} + calls: list[str] = [] + + async def _revoke(h: str) -> SessionRecord | None: + calls.append(f"revoke {h}") + if h in live: + live.discard(h) + return _record(h) + return None + + async def _successor(h: str, since: float) -> str | None: + calls.append(f"follow {h}") + found = links.get(h) + if h == "a": # the rotation of b -> c commits right after this read + live.discard("b") + live.add("c") + links["b"] = "c" + return found + + ended = await revoke_through_lineage( + "a", lineage_since=0.0, revoke=_revoke, successor=_successor + ) + assert ended is not None and ended.token_hash == "c" + assert calls == ["revoke a", "follow a", "revoke b", "follow b", "revoke c"] + + +async def test_the_walk_stops_at_its_hop_bound() -> None: + async def _successor(h: str, since: float) -> str | None: + return str(int(h) + 1) + + def _revoke_only(target: int) -> Callable[[str], Awaitable[SessionRecord | None]]: + async def _revoke(h: str) -> SessionRecord | None: + return _record(h) if h == str(target) else None + + return _revoke + + reach = await revoke_through_lineage( + "0", lineage_since=0.0, revoke=_revoke_only(LINEAGE_MAX_HOPS), successor=_successor + ) + assert reach is not None and reach.token_hash == str(LINEAGE_MAX_HOPS) + past = await revoke_through_lineage( + "0", lineage_since=0.0, revoke=_revoke_only(LINEAGE_MAX_HOPS + 1), successor=_successor + ) + assert past is None, "the walk followed more links than its bound" diff --git a/tests/test_sqlserver_store.py b/tests/test_sqlserver_store.py index 19cb1701a5..3d88b8db98 100644 --- a/tests/test_sqlserver_store.py +++ b/tests/test_sqlserver_store.py @@ -113,6 +113,8 @@ async def store() -> AsyncIterator[object]: # silently miscounts an unrelated one "messages", "sessions", + "session_lineage", # vault BACKLOG #3028 (no FK): a stale link would lead a later + # test's supersession into a session it never retired "webauthn_credentials", # ADR 0068: FK to users(id) — must clear before users "known_login_addresses", # vault BACKLOG #2145: per-account, cleared before users "user_roles", @@ -5798,13 +5800,21 @@ async def fetch(sql: str, params: tuple[Any, ...]) -> list[dict[str, Any]]: async def test_session_supersession_contract(store) -> None: - """BACKLOG #2146: supersede_session revokes and returns in one statement, so a rotation after it - fails closed. Extra-free shared contract, run here as on SQLite.""" + """BACKLOG #2146: supersede_session revokes and returns in one transaction, so a rotation after + it fails closed. Extra-free shared contract, run here as on SQLite.""" from tests._session_rotation_contract import assert_session_supersession_contract await assert_session_supersession_contract(store) +async def test_session_lineage_contract(store) -> None: + """Vault BACKLOG #3028: a rotation links the hash it retired, the supersession follows the + links inside its window, and the purge drops them past it. Extra-free shared contract.""" + from tests._session_rotation_contract import assert_session_lineage_contract + + await assert_session_lineage_contract(store) + + async def test_session_cap_contract(store) -> None: """BACKLOG #1900: the per-user cap counts only LIVE sessions. What this leg executes that no other does: ``TOP (?)`` ahead of the subquery's WHERE, which binds ``keep`` before the liveness diff --git a/tests/test_store_pool_acquire_timeout.py b/tests/test_store_pool_acquire_timeout.py index 183af206b5..0279bf770e 100644 --- a/tests/test_store_pool_acquire_timeout.py +++ b/tests/test_store_pool_acquire_timeout.py @@ -376,7 +376,8 @@ async def test_postgres_post_commit_session_writes_wait_out_a_busy_pool( assert not task.done(), f"{name} gave up on a busy pool after its caller's write committed" pool.gate.set() assert await asyncio.wait_for(task, timeout=5) == expected - assert pool.statements and pool.statements[-1].startswith("UPDATE sessions") + # The rotation's lineage link (vault BACKLOG #3028) follows its re-key on the same borrow. + assert any(s.startswith("UPDATE sessions") for s in pool.statements), pool.statements async def test_sqlserver_store_releases_a_healthy_borrow() -> None: diff --git a/tests/test_task_resilience.py b/tests/test_task_resilience.py index 67d3f9c087..158ae95841 100644 --- a/tests/test_task_resilience.py +++ b/tests/test_task_resilience.py @@ -171,7 +171,11 @@ async def test_session_reaper_survives_purge_error(monkeypatch: pytest.MonkeyPat class _Store: async def purge_expired_sessions( - self, *, now: float | None = None, idle_seconds: float | None = None + self, + *, + now: float | None = None, + idle_seconds: float | None = None, + lineage_seconds: float | None = None, ) -> None: calls["n"] += 1 if calls["n"] == 1: diff --git a/tests/test_webauthn_store.py b/tests/test_webauthn_store.py index 799466f990..7ade299a4b 100644 --- a/tests/test_webauthn_store.py +++ b/tests/test_webauthn_store.py @@ -112,3 +112,15 @@ async def test_session_supersession_contract_sqlite(tmp_path: Path) -> None: await assert_session_supersession_contract(store) finally: await store.close() + + +async def test_session_lineage_contract_sqlite(tmp_path: Path) -> None: + """SQLite leg of the vault BACKLOG #3028 session lineage contract, shared with the live Postgres + / SQL Server suites.""" + store = await MessageStore.open(tmp_path / "lin.db") + try: + from tests._session_rotation_contract import assert_session_lineage_contract + + await assert_session_lineage_contract(store) + finally: + await store.close() From 6810a1414ec38d3fae53ac63c9af5e04684e6b8d Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Sat, 17 Oct 2026 04:16:34 -0500 Subject: [PATCH 41/57] Review round one on D5: link time read after the re-key, wider slack, honest bounds (BACKLOG #3028) retired_at is now read once the re-key has landed, inside the rotation's transaction, so a wait for the borrow or the row lock cannot age a link before it exists. The sign-in slack is 120 seconds and names the callback's directory resolve. The walk reads no successor past its hop bound. SECURITY.md lists the cases that stay open (a second holder past the bound, a slow sign-in, more than 32 elevations, a clock step) and that a clock step lengthens the retired token's power; the supersession docstring no longer says it grants no new power; base.py and the walk say the step order, not the transaction, is the control on the server backends. PHI.md lists session_lineage in the at-rest inventory. New tests: the federated leg with an elevation during the IdP round trip, and the SQL Server link INSERT's binding. --- changelog.d/3028.security.md | 2 +- docs/PHI.md | 1 + docs/SECURITY.md | 32 +++++---- ...ows-a-time-bounded-retired-hash-lineage.md | 8 ++- docs/adr/README.md | 2 +- messagefoundry/auth/service.py | 19 ++++-- messagefoundry/store/base.py | 26 ++++--- messagefoundry/store/postgres.py | 9 ++- messagefoundry/store/session_lineage.py | 7 +- messagefoundry/store/sqlserver.py | 7 +- messagefoundry/store/store.py | 11 +-- tests/test_session_sql_placeholders.py | 10 ++- tests/test_session_supersession_lineage.py | 68 ++++++++++++++++++- 13 files changed, 149 insertions(+), 53 deletions(-) diff --git a/changelog.d/3028.security.md b/changelog.d/3028.security.md index 60a086a9e6..4d71abbec8 100644 --- a/changelog.d/3028.security.md +++ b/changelog.d/3028.security.md @@ -4,7 +4,7 @@ under its new token. Each rotation now records which token hash replaced the old one, in its own store transaction, in a new `session_lineage` table on SQLite, Postgres and SQL Server. The supersession follows those links to the live session and ends it, in the same transaction as the - revoke, for `[auth].oidc_flow_ttl_seconds` plus 60 seconds after each rotation; the session reaper + revoke, for `[auth].oidc_flow_ttl_seconds` plus 120 seconds after each rotation; the session reaper deletes older links. A retired token still authenticates nothing: no session read consults the links. `docs/SECURITY.md` states the bound that remains, and ADR 0210 records the design. On the server backends the schema change moves the ADR 0064 schema hash: under their default diff --git a/docs/PHI.md b/docs/PHI.md index ff6b4c58cb..5f2706b985 100644 --- a/docs/PHI.md +++ b/docs/PHI.md @@ -200,6 +200,7 @@ destruction) are documented in [§3](#3-encryption-at-rest) under the matching h | `state.namespace` / `state.key` | all three | **Possibly** — a Handler that keys correlation state on a raw MRN stores that identifier here in the clear | **No** — plaintext by construction: the pair is the composite primary key **and** the AAD input for `state.value`, so it cannot be ciphered without losing the lookup | **PL-4** | Authors must key state on a **surrogate, never a raw identifier**. Covered only by the whole-DB / volume layer. Rides `[retention].state_max_age_days` with its value | ``rides `[retention].state_max_age_days` `` | | `reference.name` / `reference.version` / `reference.key` | all three | **Possibly** — §2's `reference.value` row notes a snapshot row may be patient-keyed; the key column is where that identifier would sit | **No** — plaintext by construction, same reason as `state` | **PL-4** | Same rule: key reference sets on a surrogate. The key columns ride the same delete: `purge_reference_snapshots` removes whole rows of an **undeclared** set, so these go with the `value` they key. A **declared** set is never touched — same orphan-only limit as `reference.value` above | ``orphan-only `[retention].reference_snapshot_days` `` | | `sessions.token_hash` / `client`, `resend_log`, `processed_files`, `pending_approvals.params`, `webauthn_credentials.public_key` | all three | **No** | No (deliberately not ciphered) | **PL-4** | Session tokens are stored as SHA-256 only; `processed_files` holds a hashed derived file key, never a path; approval params carry connection names / channel ids / a config dir by construction; COSE public keys (ADR 0068) are **verification material, not secrets** and are explicitly excluded from the cipher and from rekey | `n/a — not PHI` | +| `session_lineage` (`retired_hash` / `successor_hash` / `retired_at`, vault BACKLOG #3028, ADR 0210) | all three | **No** | No (deliberately not ciphered) | **PL-4** | Two SHA-256 session token hashes and a time, the same class as `sessions.token_hash`: which hash an elevation retired and which replaced it. Read only by the sign-in supersession, never by a session read, so a retired hash authenticates nothing; not sealed, because a forged link can only direct a revoke | the session reaper deletes links past the supersession's window (`oidc_flow_ttl_seconds` plus 120 seconds) | | `known_login_addresses` (`user_id` / `address` / `first_seen` / `last_seen`, vault BACKLOG #2145) | all three | **No** — a network address tied to an account; identifies a *host*, not a patient | No (metadata, deliberately not ciphered) | **PL-4** | The first-seen sign-in address baseline (BACKLOG #288): one row per host an account finished a sign-in or passed a step-up from, keyed on the account id. **Personal data, not PHI** — the same class and the same decision as `sessions.client` and `audit_log.client`. Read only by the sign-in signal; **no API surface**. Rows past the signal's 90-day lookback are deleted at the account's next sign-in or step-up that records an address. A sign-in that records nothing, such as one that still owes a factor, prunes nothing, so a dormant account, or one that never records a new address, keeps its stale rows until it does. `delete_user` removes every row with the account, and the account's foreign key cascades to any row a racing write leaves | `n/a — not PHI` | | `secret_rotation_meta` (`secret_key` / `fingerprint` / `tracked_since` / `last_rotated`) | **All three backends** (#1186 — SQLite `MessageStore`, `SqlServerStore` and `PostgresStore` each create the table at open and implement the `SecretRotationMetaStore` protocol) | **No** — non-secret rotation state: a **keyed MAC** (DEK-derived, one-way — never the secret value) + ISO dates only | No (deliberately not ciphered — it is neither PHI nor a secret; the MAC is one-way and un-guessable without the DEK) | **PL-4** | ASVS 13.3.4 rotation watcher (BACKLOG #282). `fingerprint` is a keyed MAC so obtaining the rows leaks no secret. Written on a keyed store; absent on a keyless one | `n/a — not PHI` | | SQLite DB file + `-wal` / `-shm` / temp files, and every index | SQLite | **Yes** (mirror the above) | **No** — the app cipher cannot reach them | **PL-5** | WAL/shm hold recently-written PHI outside any app-level encryption. Cover: SQLCipher (whole-DB) and/or FDE on the engine host's data volume. A purge zeroes what it frees, and the WAL truncate is tried hourly, both by default; what they miss is in [§8](#8-retention--purge) (vault BACKLOG #2619) | `n/a — not PHI` | diff --git a/docs/SECURITY.md b/docs/SECURITY.md index 579565815b..44a49e4523 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -4450,7 +4450,7 @@ engine client (`EngineClient.login`) both name the token they replace in `supers client names only a token the engine issued to it, never one adopted with `set_token`. Every elevation gives a session a new token: completing MFA, a step-up, a passkey ceremony or a -password re-check. The supersession finds and ends the presented session in one store transaction, +password re-check. The supersession finds and ends the presented session in one write on its row, so an elevation cannot land between the two (BACKLOG #2146). If the supersession runs first, a later elevation on that session fails and asks the user to sign in again. @@ -4460,24 +4460,30 @@ elevation in another tab can finish while the sign-in is still in progress, befo runs. On federated sign-in that window includes the round trip to the identity provider, because the presented token is read on the start leg. The presented token then names no session. So each rotation records, in its own store transaction, which token hash replaced the old one, and the -supersession follows those links, over as many elevations as there were, and ends the session at -the live end, in the same transaction as the revoke. The audit row names that session and, in a -`presented` field, the hash the browser presented. - -The link is followed for a bounded time: `[auth].oidc_flow_ttl_seconds` plus 60 seconds, which is -360 seconds at the default, and the session reaper deletes older links. **What stays open is outside -that bound.** A second holder of the same token who completed an elevation longer ago than that, or -a sign-in that runs past it, leaves the presented token naming no session that is followed, so -nothing is ended. That session lives on under the token the elevation handed out, until it expires, -as before this change. +supersession follows those links, up to 32 elevations in a row, and ends the session at the live +end, in the same transaction as the revoke. The audit row names that session and, in a `presented` +field, the hash the browser presented. + +The link is followed for a bounded time: `[auth].oidc_flow_ttl_seconds` plus 120 seconds, which is +420 seconds at the default, and the session reaper deletes older links. **At least these cases stay +open**, and in each the session lives on under the token the elevation handed out, until it expires, +as before this change: + +- an elevation that a second holder of the same token completed longer ago than the bound; +- a sign-in that runs past the bound, which a slow identity provider or directory can cause, since + their timeouts bound each call and not the whole sign-in; +- more than 32 elevations on one session inside the bound; +- a wall clock that stepped between the elevation and the sign-in, which moves the bound by the + size of the step, in either direction. **A retired token authenticates nothing.** No session read, the validator included, consults the links; the supersession is their one reader. For the length of the bound, a retired token has one power: presented at a sign-in that succeeds, it ends the session it became. That power is new, and it can only end a session, never open or extend one. Whoever held that token before the elevation could already end the session with `POST /auth/logout`; the link lets them do it for the length of -the bound after the elevation too, and only through a sign-in with a credential that passes. The -links are not under the session integrity seal (BACKLOG #2592): a forged link can only direct a +the bound after the elevation too, and only through a sign-in with a credential that passes. A wall +clock that stepped forward at the elevation, or back before the sign-in, lengthens that time by the +step; the reaper's clock-step guard covers its own purge, not this. The links are not under the session integrity seal (BACKLOG #2592): a forged link can only direct a revoke, and a revoke needs no key on any backend. A session's `id` is its token hash, and that hash changes whenever the session completes MFA or a diff --git a/docs/adr/0210-session-supersession-follows-a-time-bounded-retired-hash-lineage.md b/docs/adr/0210-session-supersession-follows-a-time-bounded-retired-hash-lineage.md index 04ed8c43a3..76d1c1ca9f 100644 --- a/docs/adr/0210-session-supersession-follows-a-time-bounded-retired-hash-lineage.md +++ b/docs/adr/0210-session-supersession-follows-a-time-bounded-retired-hash-lineage.md @@ -44,7 +44,7 @@ and only it, follows those links for a bounded time to the live end and revokes the revoke: its link was written in its own transaction. The walk follows at most 32 links. 3. **The bound is the caller's, as `lineage_since`**: a link retired before it is not followed. The store reads no settings. `AuthService.supersession_lineage_seconds` is - `oidc_flow_ttl_seconds` plus 60 seconds, the longest sign-in a rotation can race plus the + `oidc_flow_ttl_seconds` plus 120 seconds, the longest sign-in a rotation can race plus the callback's own work after it. With no `lineage_since`, `supersede_session` is the plain revoke it was, which `_end_lapsed_session` keeps. 4. **The session reaper purges links past the window**, in the same pass as expired sessions, @@ -105,4 +105,8 @@ sign-in that succeeds, it ends the session it became. It cannot open or extend o **Stated limits** -- An elevation that finished longer ago than the window, such as one a second holder of the token completed earlier, is not followed, and neither is a sign-in that runs past the window; that session lives on under its new token until it expires. More than 32 links in one -window ends nothing. The Postgres and SQL Server legs run only on hosted runners. +window ends nothing. `retired_at` and the bound are both wall-clock readings, so a clock step +between the elevation and the sign-in moves the bound by the step, either way. The window adds the +flow TTL even where federated sign-in is off, where every leg is one request; the brief fixed the +window at no less than the flow TTL, and a narrower one there is open as a refinement. The +Postgres and SQL Server legs run only on hosted runners. diff --git a/docs/adr/README.md b/docs/adr/README.md index c4146b27d9..9d8debf8d5 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -231,7 +231,7 @@ what is withheld and what you can request. | [0206](0206-an-hl7-write-or-re-encode-never-lets-data-become-structure.md) | **An HL7 write or re-encode never lets data become structure** (vault BACKLOG #2558, #2559, #2560) -- `Message.set_data` writes a value read from a message as data at every level, and `copy_field` and `split_field` use it for a component or subcomponent source; `set` keeps its meaning for authored text; the Steps view's Copy Field emits `set_data` for a literal leaf source; the advisory handler-security lint gains `leaf-to-whole-field` for the hand-written `msg.set(field, msg.field(leaf))` shape. The MLLP delimiter override escapes a target delimiter found in a leaf with the target escape character and refuses a value it cannot represent, every override failure being `NegativeAckError(permanent=True)`. A source that does not split refuses an HL7 v2 body holding more than one `MSH`, with an `AR` where it has an ACK channel; `RemoteFileSource` now splits a batch like the File source, which is unchanged. Rejected: escaping every whole-field `set`, refusing a mixed-level copy, a marked `str` subclass, splitting on a listener | **Accepted (2026-10-02, on build)** -- decision delegated by the owner under the driver rule and taken by the batch 186 Manager; the owner may overrule at review. Built with the change. Severity conditional per CLAUDE.md section 0. **Amended 2026-10-04** (vault BACKLOG #2861, #2862): the Steps view's Set Field keeps `set` only for the HL7 separators `^ ~ & \`, so a template holding `:`, `>` or `*` writes `set_data`; the `leaf-to-whole-field` finding's advice names the X12 case | | [0207](0207-a-single-use-enrolment-code-for-a-directory-account-s-first-engine-factor.md) | **A single-use enrolment code for a directory account's first engine factor** (vault BACKLOG #2609, closing item 2; the decision ADR 0142 Amendment C left open) -- a directory account's first engine factor is bound only with a single-use enrolment code an Administrator issues and hands over out of band. Required at the step that binds and stamps (`confirm_mfa_enrollment`, `finish_webauthn_registration`) when the account is `ad`, holds no engine factor, and the session is not stamped MFA-verified: so every Kerberos session and a federated one with the claim gate off, never a claim-gated federated one. That step also re-checks that the account still holds no factor, because the web console's passkey finish route does not, and the TOTP enable turns on only the secret that was verified, since a later begin replaces the staged one today. The passkey finish route gains the per-actor attempt budget the TOTP confirm routes already draw. Disabling the account clears an outstanding code, and every refused code gets one client-facing reply. The handover is the control, so it must leave the directory: an Administrator checks who is asking and never replies through the account's own mail or chat. The rule reads no session mechanism and no `kerberos_enabled`, and holds with `require_mfa` on or off. At least 128 bits, stored as its SHA-256 digest, one per account, 24 hours, spent by one conditional write before the session is stamped. Issued by an Administrator behind step-up, never for the caller's own account, also by an MFA reset of a directory account, and, for a site where no Administrator can issue one, by a host command on `admin-unlock`'s gate, which is not BACKLOG #2711's outage recovery. Rejected: password only, a TOTP-on-directory-password shortcut, an emailed link, a short Argon2 code, gating on the mechanism | **Proposed (2026-10-05)** -- doc-only, no code; drafted on the owner's direction. Merging records it as Proposed and accepts nothing; Accepted needs the owner. Open: the code with `require_mfa` off, a fixed lifetime, a code from an MFA reset, and the host command's reach. Severity conditional per CLAUDE.md section 0 | | [0208](0208-a-theia-desktop-app-gives-interface-analysts-a-simpler-steps-only-editor.md) | **A Theia desktop app gives interface analysts a simpler Steps-only editor** (owner rulings and Manager decisions of 2026-10-07; rewrites the uncommitted 2026-10-02 hosted-Theia drafts after two four-lens reviews) -- two Electron builds sharing one native Theia Steps extension: an analyst build capped at the Steps level, and a developer build that is a full IDE with a Steps / Split / Code switch and no per-user limits. The purpose is a simpler interface, not enforcement. The analyst build signs in to the engine, clears every session gate, proves the session on a non-exempt route at start-up and before each save, and turns on editing only for `code:steps` (new, advisory, read by the editor only) or `code:edit`. That role check is the primary control and a guardrail: the developer build, VS Code and any other tool are outside it. An optional repository check classifies a change as Steps-only by comparing `code` rows, every control header and each typed row's full statement against base, and passes anything else only with a `code:edit` reviewer's approval of the head commit; it holds against a deliberate change only when run from a definition the change cannot edit. Analyst Test takes a generator spec and never passes `--show-phi`. Review R1 is a precondition answered by the R1 fix and a typed-only `lens rewrite` mode the analyst build always sets; both landed in PR 2155 (2026-10-08). Analyst is a built-in role. The analyst build drops three ADR 0076 guardrails (Amendment G). Hosting is deferred to its own ADR. Rejected: a filtered `@theia/plugin-ext` analyst build, per-user limits in the developer build, VS Code for analysts, a Steps editor in the web console. Spec: [`docs/design/theia-analyst-editor.md`](../design/theia-analyst-editor.md) | **Proposed (2026-10-07)** -- doc-only, no editor code. Merging records it as Proposed and accepts nothing; Accepted needs the owner. Licence AGPL-3.0-or-later (owner ruling). Open: spikes S-1 to S-4 (S-1 is met except its Test and language-server legs, measured on a browser target; S-2 is met on the browser build and on Electron, spike S-2b; S-4's classifier half is met by a prototype and its base-ref workflow half is untested), the start-up probe route (an implementation detail), and the differences ADR 0076 Amendment G, G.7, records between PR 2155's lens code and its rules | -| [0210](0210-session-supersession-follows-a-time-bounded-retired-hash-lineage.md) | **Session supersession follows a time-bounded retired-hash lineage** (vault BACKLOG #3028; extends #2146, engine PR 2028) -- each session rotation writes a `session_lineage` link (retired hash, successor hash, `retired_at`) in its own transaction, on SQLite, Postgres and SQL Server. The login supersession, and nothing else, follows those links over any number of hops (at most 32) to the live end and revokes it in the same transaction, for `oidc_flow_ttl_seconds` plus 60 seconds after each rotation; the session reaper purges older links. No session read consults the table, so a retired hash authenticates nothing; a test pins the readers per backend. The links are unsealed, because a forged one can only direct a revoke, which needs no key. One shared walk, so #3029 can reuse it. Rejected: a `previous_hash` column, a revoked tombstone row, a per-sign-in start-time bound. Open: an elevation older than the window, such as a second holder's, is not followed | **Accepted (2026-10-17, on build)** -- design set by the batch 257 Manager's brief; the owner may overrule at review. Severity conditional per CLAUDE.md section 0 | +| [0210](0210-session-supersession-follows-a-time-bounded-retired-hash-lineage.md) | **Session supersession follows a time-bounded retired-hash lineage** (vault BACKLOG #3028; extends #2146, engine PR 2028) -- each session rotation writes a `session_lineage` link (retired hash, successor hash, `retired_at`) in its own transaction, on SQLite, Postgres and SQL Server. The login supersession, and nothing else, follows those links over any number of hops (at most 32) to the live end and revokes it in the same transaction, for `oidc_flow_ttl_seconds` plus 120 seconds after each rotation; the session reaper purges older links. No session read consults the table, so a retired hash authenticates nothing; a test pins the readers per backend. The links are unsealed, because a forged one can only direct a revoke, which needs no key. One shared walk, so #3029 can reuse it. Rejected: a `previous_hash` column, a revoked tombstone row, a per-sign-in start-time bound. Open: an elevation older than the window, such as a second holder's, is not followed | **Accepted (2026-10-17, on build)** -- design set by the batch 257 Manager's brief; the owner may overrule at review. Severity conditional per CLAUDE.md section 0 | ## Retired allocations — numbers taken but never authored diff --git a/messagefoundry/auth/service.py b/messagefoundry/auth/service.py index 326f22cdf2..bcc8cc96d7 100644 --- a/messagefoundry/auth/service.py +++ b/messagefoundry/auth/service.py @@ -185,11 +185,12 @@ #: Vault BACKLOG #3028: what :attr:`AuthService.supersession_lineage_seconds` adds to the flow TTL #: for the sign-in's own work after its last wait. At the federated callback that is the code -#: exchange and a JWKS refresh, each a blocking call under a 10-second socket timeout; on the -#: password and directory legs it is the whole request, whose LDAP calls each sit under -#: ``ad_connect_timeout`` and ``ad_receive_timeout`` (10 seconds by default). A sign-in that runs -#: past the window is a stated limit in docs/SECURITY.md. -_LINEAGE_SIGN_IN_SLACK_SECONDS: Final = 60.0 +#: exchange and a JWKS refresh, each under a 10-second socket timeout, then the directory resolve, +#: whose LDAP calls each sit under ``ad_connect_timeout`` and ``ad_receive_timeout`` (10 seconds +#: by default), then the store writes before the mint. The password and directory legs are one +#: request of the same parts. A socket timeout bounds each blocking call, not the whole, so a +#: sign-in that runs past the window is possible, and docs/SECURITY.md states it as a limit. +_LINEAGE_SIGN_IN_SLACK_SECONDS: Final = 120.0 @cache @@ -8887,8 +8888,12 @@ async def _supersede_session_hash(self, prior_hash: str, *, client: str | None) turn every sign-in into a forced sign-out of every other device. The prior session may belong to a different user, for example on a shared workstation. It is - still ended. Anyone holding that token could already end it with ``POST /auth/logout``, so - this grants no new power. The audit row's actor is the session's OWNER, not whoever signed + still ended. Anyone holding that LIVE token could already end it with ``POST + /auth/logout``, so ending the presented session grants no new power. Following the lineage + does grant one, and it is bounded: for :attr:`supersession_lineage_seconds` after an + elevation, a holder of the RETIRED token can end the session it became through a sign-in + that succeeds, which ``logout`` cannot do. docs/SECURITY.md states it. The audit row's + actor is the session's OWNER, not whoever signed in: ``/me/security-events`` selects rows by actor, so the event belongs in the feed of the user whose session ended, and must not put that user's ids in anyone else's. diff --git a/messagefoundry/store/base.py b/messagefoundry/store/base.py index 02fe9f9105..bede730b86 100644 --- a/messagefoundry/store/base.py +++ b/messagefoundry/store/base.py @@ -2795,7 +2795,8 @@ async def rotate_session( so it must be able to fail closed if the session died underneath it. **A re-key that lands also records a lineage link, in the SAME transaction** (vault BACKLOG - #3028, ADR 0210): ``token_hash`` retired at ``now``, replaced by ``new_token_hash``. The + #3028, ADR 0210): ``token_hash`` retired, replaced by ``new_token_hash``, stamped with + ``now`` or, when that is ``None``, the time the re-key landed, after any wait. The session row itself carries nothing of its old hash, so the old hash still resolves to nothing. The link's one reader is :meth:`supersede_session`; ``store/session_lineage.py`` says why it is unsealed. A re-key that did not land writes no link. @@ -2839,19 +2840,22 @@ async def supersede_session( whether the session was live, and ``revoked_at`` is ``now``. Its ``token_hash`` is the hash the session ended under, which differs from ``token_hash`` when a link was followed. - **ONE store transaction, so it cannot interleave with :meth:`rotate_session` (BACKLOG - #2146).** The login supersession used to read the row and then revoke it by hash, and a - rotation that re-keyed the row between the two left the revoke matching nothing. Here the - store orders the two writes on the row. If this one runs first, the row is revoked and the - rotation's ``revoked_at IS NULL`` guard refuses it, so the rotation fails closed. + **The revoke and the read-back are one write on the row, so they cannot interleave with + :meth:`rotate_session` (BACKLOG #2146).** The login supersession used to read the row and + then revoke it by hash, and a rotation that re-keyed the row between the two left the + revoke matching nothing. Here the store orders the two writes on the row. If this one runs + first, the row is revoked and the rotation's ``revoked_at IS NULL`` guard refuses it, so + the rotation fails closed. **A rotation that committed FIRST is followed, when ``lineage_since`` is given (vault BACKLOG #3028, ADR 0210).** The presented hash then names no row, so the revoke steps along - the links :meth:`rotate_session` wrote, over as many rotations as there were, and ends the - live end, all inside this one transaction. A link retired before ``lineage_since`` is not - followed, so a retired hash leads anywhere only for a bounded time; the caller chooses the - bound, because the store reads no settings. A rotation that commits during the walk costs - the walk one more step, never the revoke: ``store/session_lineage.py`` says why. With + the links :meth:`rotate_session` wrote, up to ``LINEAGE_MAX_HOPS`` of them, and ends the + live end, inside one transaction. A link retired before ``lineage_since`` is not followed, + so a retired hash leads anywhere only for a bounded time; the caller chooses the bound, + because the store reads no settings. On Postgres and SQL Server a rotation can still commit + between two steps; it costs the walk one more step, never the revoke, because each step + revokes before it reads a link. ``store/session_lineage.py`` says why that order is the + control. With ``lineage_since`` left ``None`` no link is followed, and a rotation that committed first leaves this returning ``None``, as before. The lineage lookup lives here and nowhere else: no session READ follows a link, so a retired hash authenticates nothing. diff --git a/messagefoundry/store/postgres.py b/messagefoundry/store/postgres.py index 2fc1e399ba..547abcc45a 100644 --- a/messagefoundry/store/postgres.py +++ b/messagefoundry/store/postgres.py @@ -9766,15 +9766,18 @@ async def rotate_session( The token hash is sealed, so the re-key goes through :meth:`_write_session` (BACKLOG #2592). The lineage link goes in the same transaction, only when the re-key landed (vault - BACKLOG #3028). The answer is read after the transaction has committed. The borrow is + BACKLOG #3028), its time read once the re-key has landed, after any wait for the borrow + or the row lock. The answer is read after the transaction has committed. The borrow is :meth:`_after_commit_txn`'s, for the reason it gives.""" - now = time.time() if now is None else now async with self._after_commit_txn() as conn: rotated = await self._write_session( conn, token_hash, lambda s: replace(s, token_hash=new_token_hash) ) if rotated is not None: - await conn.execute(_PG_SESSION_LINEAGE_INSERT, token_hash, new_token_hash, now) + retired_at = time.time() if now is None else now + await conn.execute( + _PG_SESSION_LINEAGE_INSERT, token_hash, new_token_hash, retired_at + ) return rotated is not None async def revoke_session(self, token_hash: str, *, now: float | None = None) -> None: diff --git a/messagefoundry/store/session_lineage.py b/messagefoundry/store/session_lineage.py index b55d40b364..8b3e3f84a8 100644 --- a/messagefoundry/store/session_lineage.py +++ b/messagefoundry/store/session_lineage.py @@ -51,6 +51,9 @@ async def revoke_through_lineage( does it read the hash's successor and step to it. So a rotation that commits between a ``successor`` read and the next ``revoke`` costs one more step rather than the revoke: the re-keyed row's old hash then has a link of its own, written in the rotation's transaction. + **That order is the control on Postgres and SQL Server, not the transaction:** under READ + COMMITTED a rotation can commit between two steps. Reading the whole chain first and revoking + its end afterwards would aim the revoke at a hash such a rotation had already retired. ``lineage_since`` is the oldest ``retired_at`` a followed link may carry. ``None`` follows no link at all, which is the plain revoke of the hash given (BACKLOG #2146's behaviour). @@ -58,9 +61,9 @@ async def revoke_through_lineage( Returns the row it revoked, whose ``token_hash`` is the hash it ended under, or ``None`` when the walk reached a hash with no unrevoked row and no live link, or ran out of hops.""" current = token_hash - for _ in range(LINEAGE_MAX_HOPS + 1): + for hop in range(LINEAGE_MAX_HOPS + 1): ended = await revoke(current) - if ended is not None or lineage_since is None: + if ended is not None or lineage_since is None or hop == LINEAGE_MAX_HOPS: return ended following = await successor(current, lineage_since) if following is None: diff --git a/messagefoundry/store/sqlserver.py b/messagefoundry/store/sqlserver.py index 1eb4895b82..363abd3a74 100644 --- a/messagefoundry/store/sqlserver.py +++ b/messagefoundry/store/sqlserver.py @@ -14089,11 +14089,12 @@ async def rotate_session( #2592), whose write reads an ``OUTPUT`` rowset and never ``cursor.rowcount`` (BACKLOG #2283). ``token_hash`` is ``NVARCHAR(64)`` and a hex digest is exactly 64 chars, so the clustered-PK row move is width-safe. The lineage link goes in the same transaction, only - when the re-key landed (vault BACKLOG #3028).""" - now = time.time() if now is None else now + when the re-key landed (vault BACKLOG #3028), its time read then, after any wait for the + borrow or the row lock.""" async def _link(cur: Any) -> None: - await cur.execute(_SESSION_LINEAGE_INSERT, (token_hash, new_token_hash, now)) + retired_at = time.time() if now is None else now + await cur.execute(_SESSION_LINEAGE_INSERT, (token_hash, new_token_hash, retired_at)) rotated = await self._write_session_txn( token_hash, lambda s: replace(s, token_hash=new_token_hash), landed=_link diff --git a/messagefoundry/store/store.py b/messagefoundry/store/store.py index 8d60e2b114..40f49178b3 100644 --- a/messagefoundry/store/store.py +++ b/messagefoundry/store/store.py @@ -6980,7 +6980,7 @@ def _dead_target_pairs(rows: Iterable[tuple[Any, Any]]) -> list[tuple[str, str]] CREATE TABLE IF NOT EXISTS session_lineage ( retired_hash TEXT PRIMARY KEY, -- the hash the rotation retired successor_hash TEXT NOT NULL, -- the hash the session carried next - retired_at REAL NOT NULL -- when the rotation committed (wall clock) + retired_at REAL NOT NULL -- when the re-key landed, in its transaction (wall clock) ); CREATE INDEX IF NOT EXISTS ix_session_lineage_retired ON session_lineage(retired_at); @@ -15856,14 +15856,17 @@ async def rotate_session( old hash stops resolving in the same transaction. The token hash is sealed, so the re-key re-seals (BACKLOG #2592). Reports whether a row was re-keyed so the caller can fail closed when the session was revoked out from under it. The lineage link is written in the same - transaction, and only when the re-key landed (vault BACKLOG #3028).""" - now = time.time() if now is None else now + transaction, and only when the re-key landed (vault BACKLOG #3028). Its time is read once + the re-key has landed, so a wait for the lock cannot age the link before it exists.""" async with _writer_txn(self._db, self._lock): rotated = await self._write_session( token_hash, lambda s: replace(s, token_hash=new_token_hash) ) if rotated is not None: - await self._db.execute(_SESSION_LINEAGE_INSERT, (token_hash, new_token_hash, now)) + retired_at = time.time() if now is None else now + await self._db.execute( + _SESSION_LINEAGE_INSERT, (token_hash, new_token_hash, retired_at) + ) await self._commit() return rotated is not None diff --git a/tests/test_session_sql_placeholders.py b/tests/test_session_sql_placeholders.py index 9aa1fdc012..e7c099ae75 100644 --- a/tests/test_session_sql_placeholders.py +++ b/tests/test_session_sql_placeholders.py @@ -233,6 +233,9 @@ async def test_sqlserver_rotate_reads_its_output_rowset_not_the_row_count( # #3028), so a refused rotation links nothing. links = [sql for sql, _ in seen if sql.startswith("INSERT INTO session_lineage")] assert len(links) == (1 if output_rows else 0) + # The generic placeholder test never lands a rotation, so the link's binding is checked here. + for sql, params in seen: + assert sql.count("?") == len(params), f"{len(params)} parameters for:\n{sql}" @pytest.mark.parametrize("revoked", [0, 3]) @@ -255,9 +258,10 @@ async def bump(cur: Any, user_id: str) -> None: async def test_sqlserver_supersede_returns_the_output_row_by_column_name() -> None: - """``supersede_session`` reads its ``OUTPUT inserted.*`` row through ``_execute_output`` - (BACKLOG #2283), which pairs each value with its column name. Read by position instead, a - reordered column list would fill the record from the wrong columns.""" + """``supersede_session`` reads its ``OUTPUT inserted.*`` row through ``_fetch_dicts``, inside + its own transaction (BACKLOG #2283, vault BACKLOG #3028), which pairs each value with its column + name. Read by position instead, a reordered column list would fill the record from the wrong + columns.""" columns = ( "user_id", "token_hash", diff --git a/tests/test_session_supersession_lineage.py b/tests/test_session_supersession_lineage.py index 8900922698..c51dc2c295 100644 --- a/tests/test_session_supersession_lineage.py +++ b/tests/test_session_supersession_lineage.py @@ -21,15 +21,26 @@ from collections.abc import AsyncIterator, Awaitable, Callable from contextlib import closing from pathlib import Path +from urllib.parse import parse_qsl, urlsplit +from uuid import uuid4 import pytest +from messagefoundry.auth import Role +from messagefoundry.auth.oidc import FederatedPrincipal from messagefoundry.auth.service import AuthService from messagefoundry.auth.tokens import hash_token from messagefoundry.store.session_lineage import LINEAGE_MAX_HOPS, revoke_through_lineage from messagefoundry.store.store import MessageStore, SessionRecord from tests._ast_sites import code_names, code_strings, parse_source -from tests.test_session_supersession import PW, _live, _service, _superseded, _token +from tests.test_session_supersession import ( + PW, + _live, + _oidc_settings, + _service, + _superseded, + _token, +) @pytest.fixture @@ -77,6 +88,55 @@ async def test_an_elevation_that_finished_before_the_sign_in_does_not_keep_the_s ] +async def test_an_elevation_during_the_identity_provider_round_trip_is_followed( + store: MessageStore, monkeypatch: pytest.MonkeyPatch +) -> None: + # The federated leg, the widest window: the START leg stages the presented hash, an elevation + # in another tab rotates the session while the browser is at the identity provider, and the + # cookie-less callback supersedes the staged hash. The IdP exchange is replaced at the one seam + # the service calls, as tests/test_session_supersession.py does. + service = await _service(store, **_oidc_settings()) + await service.set_ad_group_map([("cn=mf-ops,dc=x", Role.OPERATOR.value)], actor="admin") + jdoe_id = uuid4().hex + await store.create_user( + user_id=jdoe_id, + username="jdoe", + auth_provider="ad", + directory_object_id="guid-jdoe", + password_generated=False, + ) + await service.bind_federated_subject( + jdoe_id, "S-1-5-21-fed", expected_issuer=None, expected_subject=None, actor="admin" + ) + prior = await _token(service) + + def _exchange(*_a: object, **_k: object) -> FederatedPrincipal: + return FederatedPrincipal( + username="jdoe", + subject="S-1-5-21-fed", + issuer="https://idp.example", + amr=("pwd", "mfa"), + acr=None, + expires_at=time.time() + 600, + auth_time=time.time(), + ) + + monkeypatch.setattr(service, "_exchange_and_validate", _exchange) + flow_id, url = await service.begin_oidc_login( + client=None, public_origin="https://ops.example", prior_session=prior + ) + rotated = await _elevate(service, prior) # finishes while the browser is at the IdP + outcome = await service.complete_oidc_login( + flow_id=flow_id, + state=dict(parse_qsl(urlsplit(url).query))["state"], + code="authcode", + client=None, + public_origin="https://ops.example", + ) + assert outcome.ok, outcome + assert not await _live(service, rotated), "the federated callback did not follow the lineage" + + async def test_the_link_is_followed_over_several_elevations(store: MessageStore) -> None: service = await _service(store) prior = await _token(service) @@ -170,8 +230,10 @@ def _functions_naming_the_lineage(source: str) -> set[str]: @pytest.mark.parametrize("backend", _BACKENDS) def test_only_the_rotation_the_supersession_and_the_purge_touch_the_lineage(backend: str) -> None: - # Closing item 2, structurally: no session READ can consult the lineage, on any backend, - # because no method but these three names the table or a statement on it. + # Closing item 2, structurally, and only this much: in the three backend modules, no method but + # these three spells the table, one of its statements or the shared walk in code. A read through + # a statement under another name, or from a new module, is outside what this proves; the + # behavioural tests above cover the validator itself. source = (_STORE_DIR / backend).read_text(encoding="utf-8") assert _functions_naming_the_lineage(source) == { "rotate_session", From 323c42fe2bf48374cedb1acbe406a1e03e409361 Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Sat, 17 Oct 2026 05:04:04 -0500 Subject: [PATCH 42/57] D5 handover: review round two fixes, PR title and banner text (BACKLOG #3028) Review round two: the PHI.md inventory row's Retention cell is the `n/a -- not PHI` prose form the drift test requires (it was the one red in a full run). SECURITY.md no longer says the retired token's power needs a full sign-in: with MFA required the supersession runs at the password-only mint. New tests: the reaper forwards lineage_seconds, the lifespan passes the service's window, and the Postgres rotation writes its link on the same borrow as its re-key. Proposed PR title: Login supersession follows a time-bounded retired-hash lineage (BACKLOG #3028) Proposed vault ledger banner text for #3028: BUILT on engine branch b257-d5 (stacked on b257-d4b), awaiting merge. Each session rotation writes a session_lineage link (retired hash, successor hash, retired_at) in its own transaction on SQLite, Postgres and SQL Server. The login supersession, and nothing else, follows the links (at most 32 hops) to the live end and revokes it in the same transaction, for oidc_flow_ttl_seconds plus 120 seconds after each rotation; the session reaper purges older links. A retired hash authenticates nothing: no session read consults the table, pinned per backend. ADR 0210 records the design; docs/SECURITY.md states what stays open (a second holder's elevation older than the bound, a sign-in that runs past it, more than 32 elevations, a clock step) and the retired token's one new power. Closing items 1 to 4 are met by this branch. Closing item 5, the re-read of ASVS cell 7.2.4, is vault record work for a Manager's vault Builder after this merges. The Postgres and SQL Server contract legs run only on hosted runners and were not run locally. #3029 is not folded in; supersede_session(lineage_since=...) is the shape it can reuse. --- docs/PHI.md | 2 +- docs/SECURITY.md | 4 +- tests/test_session_supersession_lineage.py | 55 ++++++++++++++++++++++ tests/test_store_pool_acquire_timeout.py | 5 +- 4 files changed, 63 insertions(+), 3 deletions(-) diff --git a/docs/PHI.md b/docs/PHI.md index 5f2706b985..f72a3da944 100644 --- a/docs/PHI.md +++ b/docs/PHI.md @@ -200,7 +200,7 @@ destruction) are documented in [§3](#3-encryption-at-rest) under the matching h | `state.namespace` / `state.key` | all three | **Possibly** — a Handler that keys correlation state on a raw MRN stores that identifier here in the clear | **No** — plaintext by construction: the pair is the composite primary key **and** the AAD input for `state.value`, so it cannot be ciphered without losing the lookup | **PL-4** | Authors must key state on a **surrogate, never a raw identifier**. Covered only by the whole-DB / volume layer. Rides `[retention].state_max_age_days` with its value | ``rides `[retention].state_max_age_days` `` | | `reference.name` / `reference.version` / `reference.key` | all three | **Possibly** — §2's `reference.value` row notes a snapshot row may be patient-keyed; the key column is where that identifier would sit | **No** — plaintext by construction, same reason as `state` | **PL-4** | Same rule: key reference sets on a surrogate. The key columns ride the same delete: `purge_reference_snapshots` removes whole rows of an **undeclared** set, so these go with the `value` they key. A **declared** set is never touched — same orphan-only limit as `reference.value` above | ``orphan-only `[retention].reference_snapshot_days` `` | | `sessions.token_hash` / `client`, `resend_log`, `processed_files`, `pending_approvals.params`, `webauthn_credentials.public_key` | all three | **No** | No (deliberately not ciphered) | **PL-4** | Session tokens are stored as SHA-256 only; `processed_files` holds a hashed derived file key, never a path; approval params carry connection names / channel ids / a config dir by construction; COSE public keys (ADR 0068) are **verification material, not secrets** and are explicitly excluded from the cipher and from rekey | `n/a — not PHI` | -| `session_lineage` (`retired_hash` / `successor_hash` / `retired_at`, vault BACKLOG #3028, ADR 0210) | all three | **No** | No (deliberately not ciphered) | **PL-4** | Two SHA-256 session token hashes and a time, the same class as `sessions.token_hash`: which hash an elevation retired and which replaced it. Read only by the sign-in supersession, never by a session read, so a retired hash authenticates nothing; not sealed, because a forged link can only direct a revoke | the session reaper deletes links past the supersession's window (`oidc_flow_ttl_seconds` plus 120 seconds) | +| `session_lineage` (`retired_hash` / `successor_hash` / `retired_at`, vault BACKLOG #3028, ADR 0210) | all three | **No** | No (deliberately not ciphered) | **PL-4** | Two SHA-256 session token hashes and a time, the same class as `sessions.token_hash`: which hash an elevation retired and which replaced it. Read only by the sign-in supersession, never by a session read, so a retired hash authenticates nothing; not sealed, because a forged link can only direct a revoke. The session reaper deletes links past the window the supersession follows them for (section 8) | `n/a — not PHI` | | `known_login_addresses` (`user_id` / `address` / `first_seen` / `last_seen`, vault BACKLOG #2145) | all three | **No** — a network address tied to an account; identifies a *host*, not a patient | No (metadata, deliberately not ciphered) | **PL-4** | The first-seen sign-in address baseline (BACKLOG #288): one row per host an account finished a sign-in or passed a step-up from, keyed on the account id. **Personal data, not PHI** — the same class and the same decision as `sessions.client` and `audit_log.client`. Read only by the sign-in signal; **no API surface**. Rows past the signal's 90-day lookback are deleted at the account's next sign-in or step-up that records an address. A sign-in that records nothing, such as one that still owes a factor, prunes nothing, so a dormant account, or one that never records a new address, keeps its stale rows until it does. `delete_user` removes every row with the account, and the account's foreign key cascades to any row a racing write leaves | `n/a — not PHI` | | `secret_rotation_meta` (`secret_key` / `fingerprint` / `tracked_since` / `last_rotated`) | **All three backends** (#1186 — SQLite `MessageStore`, `SqlServerStore` and `PostgresStore` each create the table at open and implement the `SecretRotationMetaStore` protocol) | **No** — non-secret rotation state: a **keyed MAC** (DEK-derived, one-way — never the secret value) + ISO dates only | No (deliberately not ciphered — it is neither PHI nor a secret; the MAC is one-way and un-guessable without the DEK) | **PL-4** | ASVS 13.3.4 rotation watcher (BACKLOG #282). `fingerprint` is a keyed MAC so obtaining the rows leaks no secret. Written on a keyed store; absent on a keyless one | `n/a — not PHI` | | SQLite DB file + `-wal` / `-shm` / temp files, and every index | SQLite | **Yes** (mirror the above) | **No** — the app cipher cannot reach them | **PL-5** | WAL/shm hold recently-written PHI outside any app-level encryption. Cover: SQLCipher (whole-DB) and/or FDE on the engine host's data volume. A purge zeroes what it frees, and the WAL truncate is tried hourly, both by default; what they miss is in [§8](#8-retention--purge) (vault BACKLOG #2619) | `n/a — not PHI` | diff --git a/docs/SECURITY.md b/docs/SECURITY.md index 44a49e4523..e75e782e08 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -4481,7 +4481,9 @@ links; the supersession is their one reader. For the length of the bound, a reti power: presented at a sign-in that succeeds, it ends the session it became. That power is new, and it can only end a session, never open or extend one. Whoever held that token before the elevation could already end the session with `POST /auth/logout`; the link lets them do it for the length of -the bound after the elevation too, and only through a sign-in with a credential that passes. A wall +the bound after the elevation too, through a sign-in whose first credential passes; with MFA +required, the supersession runs when the password-only session is minted, before the second factor, +so a password alone is enough. A wall clock that stepped forward at the elevation, or back before the sign-in, lengthens that time by the step; the reaper's clock-step guard covers its own purge, not this. The links are not under the session integrity seal (BACKLOG #2592): a forged link can only direct a revoke, and a revoke needs no key on any backend. diff --git a/tests/test_session_supersession_lineage.py b/tests/test_session_supersession_lineage.py index c51dc2c295..e372997c38 100644 --- a/tests/test_session_supersession_lineage.py +++ b/tests/test_session_supersession_lineage.py @@ -15,6 +15,7 @@ from __future__ import annotations import ast +import asyncio import re import sqlite3 import time @@ -285,6 +286,60 @@ async def test_the_lapsed_credential_revoke_does_not_follow_the_lineage( assert await _live(service, rotated) +async def test_the_reaper_purges_links_by_the_window_it_is_given( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # The lifespan hands the reaper the service's window, and the reaper must pass it to the purge, + # or no link is ever deleted. + from messagefoundry.api import app as api_app + + monkeypatch.setattr(api_app, "_SESSION_REAP_INTERVAL", 0) + monkeypatch.setattr(api_app, "_SESSION_REAP_FIRST_DELAY", 0) + seen: list[float | None] = [] + done = asyncio.Event() + + class _Store: + async def purge_expired_sessions( + self, + *, + now: float | None = None, + idle_seconds: float | None = None, + lineage_seconds: float | None = None, + ) -> None: + seen.append(lineage_seconds) + done.set() + + task = asyncio.create_task( + api_app._session_reaper(_Store(), idle_seconds=60.0, lineage_seconds=420.0) # type: ignore[arg-type] + ) + try: + await asyncio.wait_for(done.wait(), timeout=5) + finally: + task.cancel() + await asyncio.gather(task, return_exceptions=True) + assert seen[0] == 420.0 + + +def test_the_lifespan_hands_the_reaper_the_supersession_window() -> None: + source = (Path(__file__).resolve().parents[1] / "messagefoundry/api/app.py").read_text( + encoding="utf-8" + ) + reapers = [c for c in ast.walk(parse_source(source)) if isinstance(c, ast.Call)] + passing = [ + c + for c in reapers + if isinstance(c.func, ast.Name) + and c.func.id == "_session_reaper" + and any( + k.arg == "lineage_seconds" + and isinstance(k.value, ast.Attribute) + and k.value.attr == "supersession_lineage_seconds" + for k in c.keywords + ) + ] + assert len(passing) == 1, "the lifespan no longer passes the window to the reaper" + + # --- the shared walk, with no database: a rotation that lands mid-walk, and the hop bound ---------- diff --git a/tests/test_store_pool_acquire_timeout.py b/tests/test_store_pool_acquire_timeout.py index 0279bf770e..af1c00c35f 100644 --- a/tests/test_store_pool_acquire_timeout.py +++ b/tests/test_store_pool_acquire_timeout.py @@ -376,8 +376,11 @@ async def test_postgres_post_commit_session_writes_wait_out_a_busy_pool( assert not task.done(), f"{name} gave up on a busy pool after its caller's write committed" pool.gate.set() assert await asyncio.wait_for(task, timeout=5) == expected - # The rotation's lineage link (vault BACKLOG #3028) follows its re-key on the same borrow. assert any(s.startswith("UPDATE sessions") for s in pool.statements), pool.statements + if name == "rotate_session": + # The rotation's lineage link (vault BACKLOG #3028) follows its re-key on the same borrow: + # the stand-in pool hands out one connection, and it recorded both statements. + assert pool.statements[-1].startswith("INSERT INTO session_lineage"), pool.statements async def test_sqlserver_store_releases_a_healthy_borrow() -> None: From 31154fbb46dbe40a9467637ee5f04bdfeae4f187 Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Sat, 17 Oct 2026 05:42:58 -0500 Subject: [PATCH 43/57] CONFIGURATION.md: oidc_flow_ttl_seconds also sets the session lineage window A D5 open item (#3028, ADR 0210): the oidc_flow_ttl_seconds row now says it sets the session lineage window, oidc_flow_ttl_seconds + 120 s, whether or not OIDC is on, and links to SECURITY.md for when a sign-in still misses the session rather than restating those cases. The oidc_enabled row's "Off = byte-identical" names this as its exception. Kept out of the merge commit so the row has its own authoring commit. Proposed PR title for this branch (unchanged from the D5 handover, 323c42fe2): Login supersession follows a time-bounded retired-hash lineage (BACKLOG #3028) Proposed vault ledger banner text for #3028: as in the body of 323c42fe2, and the CONFIGURATION.md row it named as open is now written. --- docs/CONFIGURATION.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index 69191e6778..73444ef73a 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -762,7 +762,7 @@ document ([SECURITY-DOCS-POLICY.md](SECURITY-DOCS-POLICY.md)). | `ad_session_revoke_max_fraction` | float | `0.34` | Circuit breaker, proportional half. A pass exceeding **both** thresholds aborts, revokes nothing, logs at ERROR and writes an `auth.ad_reconcile_aborted` audit row. Requiring both means it fires only on a change simultaneously *large* and *broad* — the signature of a misconfiguration, not of offboarding (3-of-3 or 50-of-300 still applies). Range >0.0–1.0, but with AD on and a non-zero interval the engine refuses `1.0` at load. At `1.0` this half is never exceeded, for the reason the `ad_session_recheck_max_users` row gives, so the whole breaker would never trip. | | `kerberos_enabled` | bool | `false` | Windows SSO (experimental, off by default; needs `ad_enabled`) | | `kerberos_spn` | str | — | service principal, exactly `SERVICE/host`, e.g. `HTTP/host.example.com`. The engine splits it at the `/` into pyspnego's `service` and `hostname`. A malformed value is **refused at load**, including at least one with no `/`, more than one `/`, an empty half, whitespace or control characters, or a realm suffix (`HTTP/host@REALM`); the host's own domain supplies the realm. | -| `oidc_enabled` | bool | `false` | Federated SSO — OIDC auth-code + PKCE relying party ([ADR 0142](adr/0142-federated-sso-oidc-authorization-code-pkce-relying-party-hybrid-ad-backed.md)). A third login for an identity that **already exists in on-prem AD** (needs `ad_enabled`; roles come from LDAP, not the token). Off = byte-identical **for an install with no bound account**. An account an administrator has bound to a federated identity signs in through the IdP only: Windows SSO refuses it whatever this key says, so turning this off leaves a bound account with no sign-in until it is unbound ([SECURITY.md](SECURITY.md#federated-sign-in-oidc-browser-only--adr-0142), vault BACKLOG #2609). Needs `[security].web_console_public_address` (the redirect origin). | +| `oidc_enabled` | bool | `false` | Federated SSO — OIDC auth-code + PKCE relying party ([ADR 0142](adr/0142-federated-sso-oidc-authorization-code-pkce-relying-party-hybrid-ad-backed.md)). A third login for an identity that **already exists in on-prem AD** (needs `ad_enabled`; roles come from LDAP, not the token). Off = byte-identical **for an install with no bound account**, except that `oidc_flow_ttl_seconds` still sets the session lineage window (its row says how). An account an administrator has bound to a federated identity signs in through the IdP only: Windows SSO refuses it whatever this key says, so turning this off leaves a bound account with no sign-in until it is unbound ([SECURITY.md](SECURITY.md#federated-sign-in-oidc-browser-only--adr-0142), vault BACKLOG #2609). Needs `[security].web_console_public_address` (the redirect origin). | | `oidc_issuer` | str | — | https; exact-matched against the id_token `iss`. At most 256 UTF-16 code units, which is 256 characters for an ASCII URL, **refused at load** beyond that: an administrator's bind stores it, and the narrowest issuer column (SQL Server `NVARCHAR(256)`) holds no more (BACKLOG #2331) | | `oidc_client_id` | str | — | also the required `aud`/`azp` | | `oidc_client_secret` | str | — | confidential-client secret — supply via env (`MEFOR_AUTH_OIDC_CLIENT_SECRET`) or `oidc_client_secret_ref`. A value in the file is still accepted, with a WARNING at load; keep it out of the file | @@ -790,7 +790,7 @@ document ([SECURITY-DOCS-POLICY.md](SECURITY-DOCS-POLICY.md)). | `oidc_mfa_amr_values` / `oidc_required_acr_values` | list[str] | `["mfa"]` / `[]` | either family satisfies the gate; both empty with the gate on is refused. Each value is stripped at load and a blank one dropped, in a TOML list as in an env string, so a list of blanks counts as empty ([BACKLOG #2325](BACKLOG.md)) | | `oidc_acr_values` / `oidc_prompt` | str | — | requested authorize params. `oidc_acr_values` is a request only: the gate checks the returned `acr` against `oidc_required_acr_values` alone, and only while `oidc_require_mfa_claim` is on. So while `oidc_enabled` is on, a non-blank `oidc_acr_values` is **refused at load** if `oidc_required_acr_values` names no non-blank value (BACKLOG #2032). It is also refused while `oidc_require_mfa_claim` is off, whatever the required list holds (BACKLOG #2325). A whitespace-only `oidc_acr_values` loads as no request and is not sent. A token whose `amr` matches `oidc_mfa_amr_values` still passes whatever its `acr`; to rely on `acr` alone, also empty `oidc_mfa_amr_values` | | `oidc_jwks_ttl_seconds` / `oidc_jwks_min_refetch_seconds` | int | `3600` / `300` | the JWKS cache TTL + the amplification (min-refetch) bound | -| `oidc_flow_ttl_seconds` / `oidc_flow_cache_max` | int | `300` / `512` | pending-flow TTL + the **reject-when-full** bound | +| `oidc_flow_ttl_seconds` / `oidc_flow_cache_max` | int | `300` / `512` | pending-flow TTL + the **reject-when-full** bound. `oidc_flow_ttl_seconds` also sets the session lineage window, `oidc_flow_ttl_seconds` + 120 s, and it does so whether or not OIDC is on: for that long after an elevation re-keys a session, a sign-in that presents the retired token can still end the session it became, and the session reaper purges older lineage links. A longer TTL lengthens that window; a retired token never authenticates. When a sign-in still misses the session, and why the window is that long, is stated once in [SECURITY.md](SECURITY.md#session-inventory--targeted-revocation-wp-10) ([ADR 0210](adr/0210-session-supersession-follows-a-time-bounded-retired-hash-lineage.md), vault BACKLOG #3028) | | `oidc_callback_min_elapsed_seconds` | float | 1.0 | the least time between a federated start and its callback ([BACKLOG #2301](BACKLOG.md), ASVS 2.4.2). A step-up callback that returns sooner is refused. A sign-in callback is refused the same way, but only when the verified `auth_time` shows the person signed in at the IdP during this flow: an IdP holding a live single sign-on session answers with no human step, and there is nothing to floor. The refusal is the leg's ordinary one (`federated sign-in failed`, or the generic step-up refusal), audited with `reason=too_early`. The default is a **provisional** human-timing floor, derived with `mfa_verify_min_elapsed_seconds`. `0` turns it off; it must be shorter than `oidc_flow_ttl_seconds` | | `oidc_callback_floor_exempt_amr` | list[str] | `[]` | the `amr` values that mark a re-authentication with no human step, such as integrated Windows sign-in or a client certificate ([BACKLOG #2388](BACKLOG.md)). A sign-in or step-up callback whose signature-verified `amr` names one skips `oidc_callback_min_elapsed_seconds`. Every other callback keeps the floor. Values are stripped and a blank one dropped, as for `oidc_mfa_amr_values`. A value `oidc_mfa_amr_values` also accepts is **refused at load** while `oidc_require_mfa_claim` is on. The matched values are recorded as `callback_floor_exempt_amr`, under `evidence` on a sign-in's `auth.login_success` row and at the top of a step-up's `auth.reauth` row. With the list set, a too-early step-up's code is redeemed before it is refused, and a failed exchange is refused with its own reason. Any value is a **named loosening**: it proves a device answered, not that a person acted | | `oidc_session_max_hours` | int | — | caps the federated session below `id_token.exp` if a tighter bound is wanted (ADR 0079 mechanism 1) | From 8b4c2808dc426b7ba2f742e9970f82d56d92a474 Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Sat, 17 Oct 2026 06:37:14 -0500 Subject: [PATCH 44/57] CONFIGURATION.md and the breaker comment: say what the lineage, OIDC-off and refusal text can stand on Three wording fixes on b257-dtail, docs and one code comment only. - oidc_flow_ttl_seconds row: SECURITY.md says when a sign-in still misses the session, but not why the window is that long. That clause now points at ADR 0210 and AuthService.supersession_lineage_seconds. - oidc_enabled row: "Off = byte-identical" now names at least two exceptions: the lineage window, and the oidc_flow_ttl_seconds range check plus the callback-floor check, which run at load with OIDC off. - ad_session_revoke_max row and the settings.py breaker comment: a configuration refusal no longer "judges no account" outright. A refused account is neither struck nor revoked but stays in the breaker's denominator (ReconcilePlan.judged); only a pass whose every answer is a refusal, fault or failure, with a refusal and no referral, aborts as directory_misconfigured before the breaker. BACKLOG #3028 #3010 --- docs/CONFIGURATION.md | 6 +++--- messagefoundry/config/settings.py | 11 +++++++---- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index 73444ef73a..e98f3d086a 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -758,11 +758,11 @@ document ([SECURITY-DOCS-POLICY.md](SECURITY-DOCS-POLICY.md)). | `ad_session_recheck_seconds` | int | `300` | **Directory session reconciliation** ([ADR 0079](adr/0079-kerberos-idp-session-coordination.md) mechanism 2). How often to re-resolve directory principals holding **live** sessions and revoke those AD has disabled or deleted — without it, an AD disable does not take effect until the `[security].max_session_hours` cap (12 h). **`300` (five minutes) is the default** (ADR 0148 GIVEN 1 — the hardened path is the shipped path), floored at **60 s** (a pass costs one LDAP bind per signed-in directory user). `0` disables the loop and is a **loosening** once AD is on — `security_loosenings()` names it. The default is **inert without AD** (`should_reconcile()` also needs an LDAP client), so a non-AD deployment is unaffected; an **explicit** non-zero value without `ad_enabled` is still refused rather than left silently dead. | | `ad_session_recheck_strikes` | int | `2` | Consecutive passes a principal must fail to resolve before its sessions are revoked. *The search matched nothing* cannot tell *deleted* from *moved out of the search base*, so a single ambiguous result must never revoke; a set disabled bit and an unreadable `userAccountControl` strike the same way. A wave of unreadable answers is held instead of revoked, with no setting ([ADR 0195](adr/0195-brake-the-ad-session-reconciler-on-an-undetermined-useraccountcontrol-wave.md)). Range 1–10. | | `ad_session_recheck_max_users` | int | `200` | Per-pass bind budget. Beyond this, remaining users are picked up by later passes (least-recently-probed first), so a large estate degrades to a longer effective interval instead of a bind storm. A pass revokes no more accounts than it probed. So at or below `ad_session_revoke_max`, the breaker can never trip. With AD on and a non-zero interval, the engine then logs a **WARNING** at start that names both settings. It still loads, and revocation still works. To arm the breaker, raise this budget above the cap. | -| `ad_session_revoke_max` | int | `5` | **Mass-revoke circuit breaker**, absolute half. A search base that names the wrong OU, an OU reorganisation that moves the users out of it, or a service account that lost read rights on the entries while it can still read the base answers "not found" for *every* user — indistinguishable from "everyone was deleted". A base that was deleted or moved, or that the service account may not read at all, usually ends the search with a result code instead: a configuration refusal, which judges no account and raises its own `ad_reconcile_aborted` instance with reason `directory_misconfigured` (BACKLOG #3010; [SECURITY.md](SECURITY.md)). A lost read right on `userAccountControl` alone is held instead ([ADR 0195](adr/0195-brake-the-ad-session-reconciler-on-an-undetermined-useraccountcontrol-wave.md)). At `0` the absolute half is gone, so a pass trips once it revokes more than the fraction of the accounts it probed. At or above `ad_session_recheck_max_users`, the breaker can never trip; that row says what the engine does. | +| `ad_session_revoke_max` | int | `5` | **Mass-revoke circuit breaker**, absolute half. A search base that names the wrong OU, an OU reorganisation that moves the users out of it, or a service account that lost read rights on the entries while it can still read the base answers "not found" for *every* user — indistinguishable from "everyone was deleted". A base that was deleted or moved, or that the service account may not read at all, usually ends the search with a result code instead: a configuration refusal. A refused account is neither struck nor revoked, but it still counts among the accounts probed, the breaker's denominator. A pass whose every directory answer is a refusal, a fault or a failure, with at least one refusal and no referral, judges no account it asked about and raises its own `ad_reconcile_aborted` instance with reason `directory_misconfigured` (BACKLOG #3010; [SECURITY.md](SECURITY.md)). A refusal beside other answers alerts, and the rest of the pass is judged and reaches the breaker. A lost read right on `userAccountControl` alone is held instead ([ADR 0195](adr/0195-brake-the-ad-session-reconciler-on-an-undetermined-useraccountcontrol-wave.md)). At `0` the absolute half is gone, so a pass trips once it revokes more than the fraction of the accounts it probed. At or above `ad_session_recheck_max_users`, the breaker can never trip; that row says what the engine does. | | `ad_session_revoke_max_fraction` | float | `0.34` | Circuit breaker, proportional half. A pass exceeding **both** thresholds aborts, revokes nothing, logs at ERROR and writes an `auth.ad_reconcile_aborted` audit row. Requiring both means it fires only on a change simultaneously *large* and *broad* — the signature of a misconfiguration, not of offboarding (3-of-3 or 50-of-300 still applies). Range >0.0–1.0, but with AD on and a non-zero interval the engine refuses `1.0` at load. At `1.0` this half is never exceeded, for the reason the `ad_session_recheck_max_users` row gives, so the whole breaker would never trip. | | `kerberos_enabled` | bool | `false` | Windows SSO (experimental, off by default; needs `ad_enabled`) | | `kerberos_spn` | str | — | service principal, exactly `SERVICE/host`, e.g. `HTTP/host.example.com`. The engine splits it at the `/` into pyspnego's `service` and `hostname`. A malformed value is **refused at load**, including at least one with no `/`, more than one `/`, an empty half, whitespace or control characters, or a realm suffix (`HTTP/host@REALM`); the host's own domain supplies the realm. | -| `oidc_enabled` | bool | `false` | Federated SSO — OIDC auth-code + PKCE relying party ([ADR 0142](adr/0142-federated-sso-oidc-authorization-code-pkce-relying-party-hybrid-ad-backed.md)). A third login for an identity that **already exists in on-prem AD** (needs `ad_enabled`; roles come from LDAP, not the token). Off = byte-identical **for an install with no bound account**, except that `oidc_flow_ttl_seconds` still sets the session lineage window (its row says how). An account an administrator has bound to a federated identity signs in through the IdP only: Windows SSO refuses it whatever this key says, so turning this off leaves a bound account with no sign-in until it is unbound ([SECURITY.md](SECURITY.md#federated-sign-in-oidc-browser-only--adr-0142), vault BACKLOG #2609). Needs `[security].web_console_public_address` (the redirect origin). | +| `oidc_enabled` | bool | `false` | Federated SSO — OIDC auth-code + PKCE relying party ([ADR 0142](adr/0142-federated-sso-oidc-authorization-code-pkce-relying-party-hybrid-ad-backed.md)). A third login for an identity that **already exists in on-prem AD** (needs `ad_enabled`; roles come from LDAP, not the token). Off = byte-identical **for an install with no bound account**, with at least these exceptions: `oidc_flow_ttl_seconds` still sets the session lineage window (its row says how), and its range check, and the check that `oidc_callback_min_elapsed_seconds` is shorter than it, still run at load. An account an administrator has bound to a federated identity signs in through the IdP only: Windows SSO refuses it whatever this key says, so turning this off leaves a bound account with no sign-in until it is unbound ([SECURITY.md](SECURITY.md#federated-sign-in-oidc-browser-only--adr-0142), vault BACKLOG #2609). Needs `[security].web_console_public_address` (the redirect origin). | | `oidc_issuer` | str | — | https; exact-matched against the id_token `iss`. At most 256 UTF-16 code units, which is 256 characters for an ASCII URL, **refused at load** beyond that: an administrator's bind stores it, and the narrowest issuer column (SQL Server `NVARCHAR(256)`) holds no more (BACKLOG #2331) | | `oidc_client_id` | str | — | also the required `aud`/`azp` | | `oidc_client_secret` | str | — | confidential-client secret — supply via env (`MEFOR_AUTH_OIDC_CLIENT_SECRET`) or `oidc_client_secret_ref`. A value in the file is still accepted, with a WARNING at load; keep it out of the file | @@ -790,7 +790,7 @@ document ([SECURITY-DOCS-POLICY.md](SECURITY-DOCS-POLICY.md)). | `oidc_mfa_amr_values` / `oidc_required_acr_values` | list[str] | `["mfa"]` / `[]` | either family satisfies the gate; both empty with the gate on is refused. Each value is stripped at load and a blank one dropped, in a TOML list as in an env string, so a list of blanks counts as empty ([BACKLOG #2325](BACKLOG.md)) | | `oidc_acr_values` / `oidc_prompt` | str | — | requested authorize params. `oidc_acr_values` is a request only: the gate checks the returned `acr` against `oidc_required_acr_values` alone, and only while `oidc_require_mfa_claim` is on. So while `oidc_enabled` is on, a non-blank `oidc_acr_values` is **refused at load** if `oidc_required_acr_values` names no non-blank value (BACKLOG #2032). It is also refused while `oidc_require_mfa_claim` is off, whatever the required list holds (BACKLOG #2325). A whitespace-only `oidc_acr_values` loads as no request and is not sent. A token whose `amr` matches `oidc_mfa_amr_values` still passes whatever its `acr`; to rely on `acr` alone, also empty `oidc_mfa_amr_values` | | `oidc_jwks_ttl_seconds` / `oidc_jwks_min_refetch_seconds` | int | `3600` / `300` | the JWKS cache TTL + the amplification (min-refetch) bound | -| `oidc_flow_ttl_seconds` / `oidc_flow_cache_max` | int | `300` / `512` | pending-flow TTL + the **reject-when-full** bound. `oidc_flow_ttl_seconds` also sets the session lineage window, `oidc_flow_ttl_seconds` + 120 s, and it does so whether or not OIDC is on: for that long after an elevation re-keys a session, a sign-in that presents the retired token can still end the session it became, and the session reaper purges older lineage links. A longer TTL lengthens that window; a retired token never authenticates. When a sign-in still misses the session, and why the window is that long, is stated once in [SECURITY.md](SECURITY.md#session-inventory--targeted-revocation-wp-10) ([ADR 0210](adr/0210-session-supersession-follows-a-time-bounded-retired-hash-lineage.md), vault BACKLOG #3028) | +| `oidc_flow_ttl_seconds` / `oidc_flow_cache_max` | int | `300` / `512` | pending-flow TTL + the **reject-when-full** bound. `oidc_flow_ttl_seconds` also sets the session lineage window, `oidc_flow_ttl_seconds` + 120 s, and it does so whether or not OIDC is on: for that long after an elevation re-keys a session, a sign-in that presents the retired token can still end the session it became, and the session reaper purges older lineage links. A longer TTL lengthens that window; a retired token never authenticates. When a sign-in still misses the session is stated once in [SECURITY.md](SECURITY.md#session-inventory--targeted-revocation-wp-10). Why the window is that long is in [ADR 0210](adr/0210-session-supersession-follows-a-time-bounded-retired-hash-lineage.md) and in `AuthService.supersession_lineage_seconds` in `messagefoundry/auth/service.py` (vault BACKLOG #3028) | | `oidc_callback_min_elapsed_seconds` | float | 1.0 | the least time between a federated start and its callback ([BACKLOG #2301](BACKLOG.md), ASVS 2.4.2). A step-up callback that returns sooner is refused. A sign-in callback is refused the same way, but only when the verified `auth_time` shows the person signed in at the IdP during this flow: an IdP holding a live single sign-on session answers with no human step, and there is nothing to floor. The refusal is the leg's ordinary one (`federated sign-in failed`, or the generic step-up refusal), audited with `reason=too_early`. The default is a **provisional** human-timing floor, derived with `mfa_verify_min_elapsed_seconds`. `0` turns it off; it must be shorter than `oidc_flow_ttl_seconds` | | `oidc_callback_floor_exempt_amr` | list[str] | `[]` | the `amr` values that mark a re-authentication with no human step, such as integrated Windows sign-in or a client certificate ([BACKLOG #2388](BACKLOG.md)). A sign-in or step-up callback whose signature-verified `amr` names one skips `oidc_callback_min_elapsed_seconds`. Every other callback keeps the floor. Values are stripped and a blank one dropped, as for `oidc_mfa_amr_values`. A value `oidc_mfa_amr_values` also accepts is **refused at load** while `oidc_require_mfa_claim` is on. The matched values are recorded as `callback_floor_exempt_amr`, under `evidence` on a sign-in's `auth.login_success` row and at the top of a step-up's `auth.reauth` row. With the list set, a too-early step-up's code is redeemed before it is refused, and a failed exchange is refused with its own reason. Any value is a **named loosening**: it proves a device answered, not that a person acted | | `oidc_session_max_hours` | int | — | caps the federated session below `id_token.exp` if a tighter bound is wanted (ADR 0079 mechanism 1) | diff --git a/messagefoundry/config/settings.py b/messagefoundry/config/settings.py index 6cae8c6c72..3bb52a347e 100644 --- a/messagefoundry/config/settings.py +++ b/messagefoundry/config/settings.py @@ -3202,10 +3202,13 @@ class AuthSettings(_Section): # "not found" for EVERY user — indistinguishable from "everyone was deleted". A base that was # deleted or moved, or that the service account may not read at all, usually ends the search with # a result code instead (noSuchObject, invalidDNSyntax, insufficientAccessRights): a configuration - # refusal, which judges no account and pages as `ad_reconcile_aborted` with reason - # `directory_misconfigured` rather than reaching this breaker (BACKLOG #3010). Without a brake - # the reconciler would sign out the entire estate during exactly the incident when operators need - # the console. A pass that would revoke more than BOTH of these thresholds aborts, + # refusal (BACKLOG #3010). A refused account is neither struck nor revoked, but it stays in this + # breaker's denominator, the accounts probed. A pass whose every directory answer is a refusal, a + # fault or a failure, with at least one refusal and no referral, judges no account it asked about + # and pages as `ad_reconcile_aborted` with reason `directory_misconfigured` before reaching this + # breaker; a refusal beside other answers alerts, and the rest of the pass reaches it. Without a + # brake the reconciler would sign out the entire estate during exactly the incident when + # operators need the console. A pass that would revoke more than BOTH of these thresholds aborts, # revokes nothing, and raises a loud operator-visible alert (log ERROR + an # `auth.ad_reconcile_aborted` audit row). # From 120eedff0eac5626d3f871ff19d80c908f8c02e8 Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Sat, 17 Oct 2026 06:45:36 -0500 Subject: [PATCH 45/57] CONFIGURATION.md and the breaker comment: correct the refusal split the review found Code review of the previous commit found the new refusal text wrong. A refusal beside a referral aborts as directory_referral and never reaches the breaker, and the directory_misconfigured alert fires on any pass with a refusal, not only on an all-refused one. Both texts now say only what is true of every refusal: it alerts on its own source, the refused account is neither struck nor revoked, and it stays in the breaker's denominator. When a pass aborts before the breaker is left to SECURITY.md, which states it once. The settings.py comment keeps the brake rationale next to the "not found" wave it answers. The oidc_enabled row now says the single-key checks on the oidc_* keys run at load with OIDC off, not only the TTL range check. BACKLOG #3028 #3010 --- docs/CONFIGURATION.md | 4 ++-- messagefoundry/config/settings.py | 20 ++++++++++---------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index e98f3d086a..2678237099 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -758,11 +758,11 @@ document ([SECURITY-DOCS-POLICY.md](SECURITY-DOCS-POLICY.md)). | `ad_session_recheck_seconds` | int | `300` | **Directory session reconciliation** ([ADR 0079](adr/0079-kerberos-idp-session-coordination.md) mechanism 2). How often to re-resolve directory principals holding **live** sessions and revoke those AD has disabled or deleted — without it, an AD disable does not take effect until the `[security].max_session_hours` cap (12 h). **`300` (five minutes) is the default** (ADR 0148 GIVEN 1 — the hardened path is the shipped path), floored at **60 s** (a pass costs one LDAP bind per signed-in directory user). `0` disables the loop and is a **loosening** once AD is on — `security_loosenings()` names it. The default is **inert without AD** (`should_reconcile()` also needs an LDAP client), so a non-AD deployment is unaffected; an **explicit** non-zero value without `ad_enabled` is still refused rather than left silently dead. | | `ad_session_recheck_strikes` | int | `2` | Consecutive passes a principal must fail to resolve before its sessions are revoked. *The search matched nothing* cannot tell *deleted* from *moved out of the search base*, so a single ambiguous result must never revoke; a set disabled bit and an unreadable `userAccountControl` strike the same way. A wave of unreadable answers is held instead of revoked, with no setting ([ADR 0195](adr/0195-brake-the-ad-session-reconciler-on-an-undetermined-useraccountcontrol-wave.md)). Range 1–10. | | `ad_session_recheck_max_users` | int | `200` | Per-pass bind budget. Beyond this, remaining users are picked up by later passes (least-recently-probed first), so a large estate degrades to a longer effective interval instead of a bind storm. A pass revokes no more accounts than it probed. So at or below `ad_session_revoke_max`, the breaker can never trip. With AD on and a non-zero interval, the engine then logs a **WARNING** at start that names both settings. It still loads, and revocation still works. To arm the breaker, raise this budget above the cap. | -| `ad_session_revoke_max` | int | `5` | **Mass-revoke circuit breaker**, absolute half. A search base that names the wrong OU, an OU reorganisation that moves the users out of it, or a service account that lost read rights on the entries while it can still read the base answers "not found" for *every* user — indistinguishable from "everyone was deleted". A base that was deleted or moved, or that the service account may not read at all, usually ends the search with a result code instead: a configuration refusal. A refused account is neither struck nor revoked, but it still counts among the accounts probed, the breaker's denominator. A pass whose every directory answer is a refusal, a fault or a failure, with at least one refusal and no referral, judges no account it asked about and raises its own `ad_reconcile_aborted` instance with reason `directory_misconfigured` (BACKLOG #3010; [SECURITY.md](SECURITY.md)). A refusal beside other answers alerts, and the rest of the pass is judged and reaches the breaker. A lost read right on `userAccountControl` alone is held instead ([ADR 0195](adr/0195-brake-the-ad-session-reconciler-on-an-undetermined-useraccountcontrol-wave.md)). At `0` the absolute half is gone, so a pass trips once it revokes more than the fraction of the accounts it probed. At or above `ad_session_recheck_max_users`, the breaker can never trip; that row says what the engine does. | +| `ad_session_revoke_max` | int | `5` | **Mass-revoke circuit breaker**, absolute half. A search base that names the wrong OU, an OU reorganisation that moves the users out of it, or a service account that lost read rights on the entries while it can still read the base answers "not found" for *every* user — indistinguishable from "everyone was deleted". A base that was deleted or moved, or that the service account may not read at all, usually ends the search with a result code instead: a configuration refusal, which raises its own `ad_reconcile_aborted` instance with reason `directory_misconfigured` on any pass that sees one. The refused account is neither struck nor revoked, but it still counts among the accounts the pass probed, the breaker's denominator. When such a pass aborts and revokes nothing before the breaker runs is stated once in [SECURITY.md](SECURITY.md) (BACKLOG #3010). A lost read right on `userAccountControl` alone is held instead ([ADR 0195](adr/0195-brake-the-ad-session-reconciler-on-an-undetermined-useraccountcontrol-wave.md)). At `0` the absolute half is gone, so a pass trips once it revokes more than the fraction of the accounts it probed. At or above `ad_session_recheck_max_users`, the breaker can never trip; that row says what the engine does. | | `ad_session_revoke_max_fraction` | float | `0.34` | Circuit breaker, proportional half. A pass exceeding **both** thresholds aborts, revokes nothing, logs at ERROR and writes an `auth.ad_reconcile_aborted` audit row. Requiring both means it fires only on a change simultaneously *large* and *broad* — the signature of a misconfiguration, not of offboarding (3-of-3 or 50-of-300 still applies). Range >0.0–1.0, but with AD on and a non-zero interval the engine refuses `1.0` at load. At `1.0` this half is never exceeded, for the reason the `ad_session_recheck_max_users` row gives, so the whole breaker would never trip. | | `kerberos_enabled` | bool | `false` | Windows SSO (experimental, off by default; needs `ad_enabled`) | | `kerberos_spn` | str | — | service principal, exactly `SERVICE/host`, e.g. `HTTP/host.example.com`. The engine splits it at the `/` into pyspnego's `service` and `hostname`. A malformed value is **refused at load**, including at least one with no `/`, more than one `/`, an empty half, whitespace or control characters, or a realm suffix (`HTTP/host@REALM`); the host's own domain supplies the realm. | -| `oidc_enabled` | bool | `false` | Federated SSO — OIDC auth-code + PKCE relying party ([ADR 0142](adr/0142-federated-sso-oidc-authorization-code-pkce-relying-party-hybrid-ad-backed.md)). A third login for an identity that **already exists in on-prem AD** (needs `ad_enabled`; roles come from LDAP, not the token). Off = byte-identical **for an install with no bound account**, with at least these exceptions: `oidc_flow_ttl_seconds` still sets the session lineage window (its row says how), and its range check, and the check that `oidc_callback_min_elapsed_seconds` is shorter than it, still run at load. An account an administrator has bound to a federated identity signs in through the IdP only: Windows SSO refuses it whatever this key says, so turning this off leaves a bound account with no sign-in until it is unbound ([SECURITY.md](SECURITY.md#federated-sign-in-oidc-browser-only--adr-0142), vault BACKLOG #2609). Needs `[security].web_console_public_address` (the redirect origin). | +| `oidc_enabled` | bool | `false` | Federated SSO — OIDC auth-code + PKCE relying party ([ADR 0142](adr/0142-federated-sso-oidc-authorization-code-pkce-relying-party-hybrid-ad-backed.md)). A third login for an identity that **already exists in on-prem AD** (needs `ad_enabled`; roles come from LDAP, not the token). Off = byte-identical **for an install with no bound account**, with at least these exceptions: `oidc_flow_ttl_seconds` still sets the session lineage window (its row says how), and the single-key checks on the `oidc_*` keys, such as the `oidc_flow_ttl_seconds` range, still run at load, as does the check that `oidc_callback_min_elapsed_seconds` is shorter than the TTL. An account an administrator has bound to a federated identity signs in through the IdP only: Windows SSO refuses it whatever this key says, so turning this off leaves a bound account with no sign-in until it is unbound ([SECURITY.md](SECURITY.md#federated-sign-in-oidc-browser-only--adr-0142), vault BACKLOG #2609). Needs `[security].web_console_public_address` (the redirect origin). | | `oidc_issuer` | str | — | https; exact-matched against the id_token `iss`. At most 256 UTF-16 code units, which is 256 characters for an ASCII URL, **refused at load** beyond that: an administrator's bind stores it, and the narrowest issuer column (SQL Server `NVARCHAR(256)`) holds no more (BACKLOG #2331) | | `oidc_client_id` | str | — | also the required `aud`/`azp` | | `oidc_client_secret` | str | — | confidential-client secret — supply via env (`MEFOR_AUTH_OIDC_CLIENT_SECRET`) or `oidc_client_secret_ref`. A value in the file is still accepted, with a WARNING at load; keep it out of the file | diff --git a/messagefoundry/config/settings.py b/messagefoundry/config/settings.py index 3bb52a347e..7f39718f38 100644 --- a/messagefoundry/config/settings.py +++ b/messagefoundry/config/settings.py @@ -3199,19 +3199,19 @@ class AuthSettings(_Section): # --- mass-revoke circuit breaker --- # A search base that names the wrong OU, an OU reorganisation that moves the users out of it, or # a service account that lost read rights on the entries while it can still read the base returns - # "not found" for EVERY user — indistinguishable from "everyone was deleted". A base that was - # deleted or moved, or that the service account may not read at all, usually ends the search with - # a result code instead (noSuchObject, invalidDNSyntax, insufficientAccessRights): a configuration - # refusal (BACKLOG #3010). A refused account is neither struck nor revoked, but it stays in this - # breaker's denominator, the accounts probed. A pass whose every directory answer is a refusal, a - # fault or a failure, with at least one refusal and no referral, judges no account it asked about - # and pages as `ad_reconcile_aborted` with reason `directory_misconfigured` before reaching this - # breaker; a refusal beside other answers alerts, and the rest of the pass reaches it. Without a - # brake the reconciler would sign out the entire estate during exactly the incident when - # operators need the console. A pass that would revoke more than BOTH of these thresholds aborts, + # "not found" for EVERY user — indistinguishable from "everyone was deleted". Without a brake + # the reconciler would sign out the entire estate during exactly the incident when operators need + # the console. A pass that would revoke more than BOTH of these thresholds aborts, # revokes nothing, and raises a loud operator-visible alert (log ERROR + an # `auth.ad_reconcile_aborted` audit row). # + # A base that was deleted or moved, or that the service account may not read at all, usually ends + # the search with a result code instead (noSuchObject, invalidDNSyntax, insufficientAccessRights): + # a configuration refusal (BACKLOG #3010). It alerts on its own `ad_reconcile_aborted` source. + # The refused account is neither struck nor revoked, but it stays in this breaker's denominator, + # the accounts probed (`ReconcilePlan.probed`). When such a pass aborts before the breaker runs + # is stated once in docs/SECURITY.md. + # # BOTH must be exceeded to trip, deliberately: the absolute floor stops the breaker firing on a tiny # estate where any proportion is meaningless (3 of 3 genuine offboardings is 100 %), and the # proportion stops a large estate being signed out wholesale. Requiring both means it fires only on From 70f6c45f8af48f2c2529b1d62b831401199d5289 Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Sat, 17 Oct 2026 12:03:44 -0500 Subject: [PATCH 46/57] Bind the step-up floor marker into the refuse partial once Review of the main merge into b257-dchain: the resolution spelled extra=floor_marker on each of the 13 federated step-up refusals past the floor, so a new arm written as refuse(REASON) would silently drop callback_floor_exempt_amr. The marker is now bound into the refuse partial once, right after it is computed, as the address fields are bound above (vault BACKLOG #3145). No arm past that point passes its own extra, and the audit rows are unchanged. BACKLOG #3220. --- messagefoundry/auth/service.py | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/messagefoundry/auth/service.py b/messagefoundry/auth/service.py index 8056d58457..d6b4114add 100644 --- a/messagefoundry/auth/service.py +++ b/messagefoundry/auth/service.py @@ -5111,6 +5111,8 @@ async def complete_oidc_step_up( # BACKLOG #3220: every refusal past the floor carries the marker when the exemption applied, # so a failed exempted step-up is as visible to the loosening's review as a served one. floor_marker = _floor_exempt_marker(floor_exempt_amr) + # Bound once, as the address fields are above, so a new arm cannot leave the marker out. + refuse = partial(refuse, extra=floor_marker) if arrived_too_early and not floor_exempt_amr: # The opted-in site's too-early callback whose amr names no exempt value (BACKLOG #2388). return await refuse(TOO_EARLY) @@ -5125,10 +5127,10 @@ async def complete_oidc_step_up( # stepped up, so the operator is not told "verified" and then signed out. or not session.is_live(now=now, idle_seconds=self.session_idle_seconds) ): - return await refuse("session_gone", lost=True, extra=floor_marker) + return await refuse("session_gone", lost=True) user = await self._load_user(session.user_id) if user is None or user.disabled or user.auth_provider != AuthProvider.AD.value: - return await refuse("session_gone", lost=True, extra=floor_marker) + return await refuse("session_gone", lost=True) if (user.oidc_issuer, user.oidc_subject) != ( principal_claims.issuer, principal_claims.subject, @@ -5141,14 +5143,14 @@ async def complete_oidc_step_up( # BEFORE both freshness tests (BACKLOG #2143), so another person's answer that passed # the claims ladder is filed as a subject mismatch whatever its auth_time. Both tests # below judge this session's identity, so neither is the right name for someone else. - return await refuse(STEP_UP_SUBJECT_MISMATCH, extra=floor_marker) + return await refuse(STEP_UP_SUBJECT_MISMATCH) # FRESHNESS, two tests, and both must pass. max_age=0 and prompt=login ask the IdP to # authenticate the user afresh, so a conforming IdP's auth_time postdates this request. # (a) Against our clock: auth_time is IdP clock and issued_at is ours, so the floor allows # the configured skew for an IdP clock that runs behind. skew = self._settings.oidc_clock_skew_seconds if flow.issued_at <= 0 or principal_claims.auth_time < flow.issued_at - skew: - return await refuse(STEP_UP_NOT_FRESH, extra=floor_marker) + return await refuse(STEP_UP_NOT_FRESH) # (b) Against the IdP's own clock (BACKLOG #2143): auth_time must be LATER than the one the # session holds, which is the sign-in's or the last step-up's. No skew applies, because # both values come from the IdP. This closes most of what (a) alone left: an IdP that @@ -5162,20 +5164,20 @@ async def complete_oidc_step_up( # row and the operator's text say which one refused. held_auth_time = session.idp_auth_time if held_auth_time is None: - return await refuse(STEP_UP_IDP_AUTH_TIME_MISSING, extra=floor_marker) + return await refuse(STEP_UP_IDP_AUTH_TIME_MISSING) if principal_claims.auth_time <= held_auth_time: - return await refuse(STEP_UP_IDP_AUTH_TIME_NOT_LATER, extra=floor_marker) + return await refuse(STEP_UP_IDP_AUTH_TIME_NOT_LATER) # The DIRECTORY still has the account. The password re-bind this leg replaces failed for a # disabled or deleted AD object, and the sign-in leg refuses one as not_in_directory. An IdP # can still sign such a user in (sync lag, or its own credential store), so the engine asks # the directory itself, by the row's immutable id as the sign-in leg does (BACKLOG #1532). if self._ldap is None: - return await refuse("not_configured", extra=floor_marker) + return await refuse("not_configured") object_id = _directory_key(user) if object_id is None: # Never a name-only lookup: a directory may reissue a freed name to someone else. The # sign-in leg refuses such a row for the same reason (BACKLOG #2027). - return await refuse(DIRECTORY_OBJECT_ID_MISSING, extra=floor_marker) + return await refuse(DIRECTORY_OBJECT_ID_MISSING) try: principal = await asyncio.to_thread( self._ldap.resolve_principal, user.username, object_id=object_id @@ -5184,14 +5186,14 @@ async def complete_oidc_step_up( _log.warning( "directory lookup failed during a federated step-up: %s", type(exc).__name__ ) - return await refuse("directory_unavailable", extra=floor_marker) + return await refuse("directory_unavailable") if principal is None: - return await refuse("not_in_directory", extra=floor_marker) + return await refuse("not_in_directory") # BACKLOG #3174: the answer must name the row's own directory object before its groups are # trusted, by the test the sign-in leg and the AD step-up re-bind apply. mismatch = _directory_answer_mismatch(principal, object_id) if mismatch is not None: - return await refuse(mismatch, extra=floor_marker) + return await refuse(mismatch) # BACKLOG #2154: the account's stored roles must all be among the roles its current groups # map to, the same test _directory_step_up_refusal applies on the TOTP and passkey legs # (#2240). Inline rather than through that helper, because its probe would change this @@ -5200,9 +5202,9 @@ async def complete_oidc_step_up( # as before, through the loader (BACKLOG #2592). fresh = await self._load_user(user.id) if fresh is None: - return await refuse("session_gone", lost=True, extra=floor_marker) + return await refuse("session_gone", lost=True) if not set(fresh.role_ids) <= await self._ad_roles_for_groups(principal.groups): - return await refuse(DIRECTORY_ROLES_DEMOTED, extra=floor_marker) + return await refuse(DIRECTORY_ROLES_DEMOTED) purpose = flow.step_up_purpose # The password leg's three ORDER-CRITICAL steps (see :meth:`reauth`), against the hash. # (1) Every stamp against the OLD hash, re-anchoring the session to this client address. From ebabd31a43e8a1433cfbe04d6348637f5c365e30 Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Sat, 17 Oct 2026 12:24:25 -0500 Subject: [PATCH 47/57] Correct the shutdown-drain pointers the dchain merge wrote Review of the merge of b257-dchain into b257-dtail found three statements made false or left false by main's single shutdown budget: - api/approvals.py called DRAIN_TIMEOUT_SECONDS an upper limit on what the lifespans pass. They pass a share of the step's bound instead, which can exceed it; it is only the method's default. - That comment and the one over _BACKGROUND_DRAIN_SECONDS said _SHUTDOWN_BUDGET_SECONDS states the steps and their order and bounds every step. It states the budget arithmetic; _app_drain_steps and the managed teardown hold the steps, and some steps are unbounded. - docs/SECURITY.md said pending lock notices are cut off at the shutdown bound, each with one line. Where the budget cuts the drain first, they are not cancelled and run on. Comments and docs only. BACKLOG #3191, vault BACKLOG #3120. --- docs/SECURITY.md | 7 +++++-- messagefoundry/api/approvals.py | 9 +++++---- messagefoundry/auth/service.py | 11 ++++++----- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/docs/SECURITY.md b/docs/SECURITY.md index 360abf9922..61801b2644 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -5536,8 +5536,11 @@ a `logs:view` reader when some notice failed. nothing bounds how many wait. A task follows a lock, not an attempt, and a lock takes `lockout_threshold` failures on its counter, so the count grows with the accounts being locked at once rather than with the attempts against them. Under locks on many accounts the - tasks wait behind the one-at-a-time read, so mails arrive later, and any still pending at the - shutdown bound are cut off, each with one line. A cap would have to drop a notice or run it on + tasks wait behind the one-at-a-time read, so mails arrive later, and any still pending when the + shutdown drain reaches its own bound are cancelled, each with one line. Where the API lifespan's + shutdown budget cuts that drain first (`_app_drain_steps` in `api/app.py`, BACKLOG #3191), a + pending task is not cancelled: it runs on, and can meet the closed security notifier or the + closing store. A cap would have to drop a notice or run it on the request path, which is what the task exists to avoid. **Client attribution ([ADR 0150](adr/0150-client-address-on-audit-entries.md)).** Every row also diff --git a/messagefoundry/api/approvals.py b/messagefoundry/api/approvals.py index 0c493132ee..ae8899415b 100644 --- a/messagefoundry/api/approvals.py +++ b/messagefoundry/api/approvals.py @@ -106,10 +106,11 @@ #: How long :meth:`ApprovalGate.drain` waits for outcome writes still running at shutdown. It shares #: NSSM's 15 s graceful-stop window (AppStopMethodConsole in scripts/service/install-service.ps1) -#: with every other shutdown step, and engine.stop() still has to run after them. It is an upper -#: limit: the API lifespans hand the drain a share of one shutdown budget, which states the steps, -#: their order and the window arithmetic once (``_SHUTDOWN_BUDGET_SECONDS`` in ``api/app.py``, -#: BACKLOG #3191). +#: with every other shutdown step, and engine.stop() still has to run after them. It is only this +#: method's default: the API lifespans pass the drain a share of its step's bound instead, which can +#: be longer or shorter. ``_SHUTDOWN_BUDGET_SECONDS`` in ``api/app.py`` states the budget and the +#: window arithmetic (BACKLOG #3191); ``_app_drain_steps`` and the managed lifespan's teardown hold +#: the steps and their order. #: One outcome write is a status update and an audit row, so a few seconds is ample. DRAIN_TIMEOUT_SECONDS = 3.0 diff --git a/messagefoundry/auth/service.py b/messagefoundry/auth/service.py index bdd79976d5..4dbbecc19c 100644 --- a/messagefoundry/auth/service.py +++ b/messagefoundry/auth/service.py @@ -2092,11 +2092,12 @@ def _floor_exempt_marker(floor_exempt_amr: Sequence[str]) -> dict[str, object]: #: #: The budget is NSSM's 15 s graceful-stop window (``AppStopMethodConsole`` in #: ``scripts/service/install-service.ps1``), not any one drain's constant (vault BACKLOG #3120). The -#: API lifespans spend it through one shutdown budget, which states the steps, their order and the -#: window arithmetic once and gives each step a bound (``_SHUTDOWN_BUDGET_SECONDS`` in -#: ``api/app.py``, BACKLOG #3191). These two are upper limits: where the lock-notice step's bound -#: there is below their sum, that bound cuts this drain first, as the step's own comment says. A -#: change to either is read against that budget. +#: API lifespans spend it through one shutdown budget. ``_SHUTDOWN_BUDGET_SECONDS`` in +#: ``api/app.py`` states the budget and the window arithmetic (BACKLOG #3191); ``_app_drain_steps`` +#: and the managed lifespan's teardown hold the steps, their order and each bounded step's bound. +#: Where the lock-notice step's bound is below these two together, that bound cuts this drain +#: before it cancels its tasks, and a task it has not cancelled runs on past the drain, as the +#: step's own comment says. A change to either is read against that budget. _BACKGROUND_DRAIN_SECONDS: Final = 2.0 _BACKGROUND_CANCEL_GRACE_SECONDS: Final = 0.5 From 84cfc79872f7edbd7c8c615370c33df68073c6b0 Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Sat, 17 Oct 2026 12:46:50 -0500 Subject: [PATCH 48/57] State the lock-notice shutdown cost once, and correct its threshold Review round two of the batch 257 main merges: - The #2216 residual in docs/SECURITY.md said the engine waits about two seconds and a pending notice is cut off. It now says the drain waits at most two seconds and cancels, that the shutdown budget (BACKLOG #3191) can cut it first, and what a notice left running then costs: the closed security notifier still reports it queued, so the row says mailed: true for a mail never sent. The no-cap bullet points at that paragraph instead of restating it. - The _BACKGROUND_DRAIN_SECONDS comment named the wrong threshold (the drain plus its grace); a bound below the drain alone cuts it before it cancels. It now points at the SECURITY.md residual for the cost, and says the steps hold floors, from which _run_teardown computes each bound. - DRAIN_TIMEOUT_SECONDS in api/approvals.py is described as the default for a caller that passes no timeout, which the lifespans do not use. Comments and docs only. Vault BACKLOG #3120. --- docs/SECURITY.md | 16 +++++++++------- messagefoundry/api/approvals.py | 9 ++++----- messagefoundry/auth/service.py | 9 +++++---- 3 files changed, 18 insertions(+), 16 deletions(-) diff --git a/docs/SECURITY.md b/docs/SECURITY.md index 61801b2644..7c1f4e7165 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -5518,8 +5518,13 @@ source of record is `SecurityEventNotifier._record_undelivered` in second action.* **Residual: the background task adds one way to lose a notice** (BACKLOG #2216). At shutdown the -engine waits about two seconds for pending notices. A notice still pending then is cut off, and may be -neither mailed nor recorded. A failed `auth.lock_notice` row write also changes: it used to fail +engine waits at most two seconds for pending notices, then cancels any still pending, each with one +line. A cancelled notice may be neither mailed nor recorded. The API lifespan's shutdown budget +(BACKLOG #3191) can cut that wait short before it cancels anything, whenever the lock-notice step's +bound falls below those two seconds. A pending notice is then not cancelled and runs on. If it +reaches the security notifier after that notifier has closed, the notifier still takes it and +reports it queued, so its `auth.lock_notice` row says `mailed: true` for a mail that is never sent, +and the throttle holds back that account's next notice of that kind for 24 hours. A failed `auth.lock_notice` row write also changes: it used to fail the request, and now it is logged instead. The mail goes before its row, so that failure costs at most a duplicate mail at the next lock. Each such line names no account and no lock. It still tells a `logs:view` reader when some notice failed. @@ -5536,11 +5541,8 @@ a `logs:view` reader when some notice failed. nothing bounds how many wait. A task follows a lock, not an attempt, and a lock takes `lockout_threshold` failures on its counter, so the count grows with the accounts being locked at once rather than with the attempts against them. Under locks on many accounts the - tasks wait behind the one-at-a-time read, so mails arrive later, and any still pending when the - shutdown drain reaches its own bound are cancelled, each with one line. Where the API lifespan's - shutdown budget cuts that drain first (`_app_drain_steps` in `api/app.py`, BACKLOG #3191), a - pending task is not cancelled: it runs on, and can meet the closed security notifier or the - closing store. A cap would have to drop a notice or run it on + tasks wait behind the one-at-a-time read, so mails arrive later, and any still pending at + shutdown meet the residual above. A cap would have to drop a notice or run it on the request path, which is what the task exists to avoid. **Client attribution ([ADR 0150](adr/0150-client-address-on-audit-entries.md)).** Every row also diff --git a/messagefoundry/api/approvals.py b/messagefoundry/api/approvals.py index ae8899415b..ff1e123365 100644 --- a/messagefoundry/api/approvals.py +++ b/messagefoundry/api/approvals.py @@ -104,11 +104,10 @@ } -#: How long :meth:`ApprovalGate.drain` waits for outcome writes still running at shutdown. It shares -#: NSSM's 15 s graceful-stop window (AppStopMethodConsole in scripts/service/install-service.ps1) -#: with every other shutdown step, and engine.stop() still has to run after them. It is only this -#: method's default: the API lifespans pass the drain a share of its step's bound instead, which can -#: be longer or shorter. ``_SHUTDOWN_BUDGET_SECONDS`` in ``api/app.py`` states the budget and the +#: The default wait of :meth:`ApprovalGate.drain` for outcome writes still running, for a caller +#: that passes no timeout. The API lifespans do not use it: they pass a share of the drain step's +#: bound, which can be longer or shorter, inside NSSM's 15 s graceful-stop window +#: (AppStopMethodConsole in scripts/service/install-service.ps1). ``_SHUTDOWN_BUDGET_SECONDS`` in ``api/app.py`` states the budget and the #: window arithmetic (BACKLOG #3191); ``_app_drain_steps`` and the managed lifespan's teardown hold #: the steps and their order. #: One outcome write is a status update and an audit row, so a few seconds is ample. diff --git a/messagefoundry/auth/service.py b/messagefoundry/auth/service.py index 4dbbecc19c..fe7438f844 100644 --- a/messagefoundry/auth/service.py +++ b/messagefoundry/auth/service.py @@ -2094,10 +2094,11 @@ def _floor_exempt_marker(floor_exempt_amr: Sequence[str]) -> dict[str, object]: #: ``scripts/service/install-service.ps1``), not any one drain's constant (vault BACKLOG #3120). The #: API lifespans spend it through one shutdown budget. ``_SHUTDOWN_BUDGET_SECONDS`` in #: ``api/app.py`` states the budget and the window arithmetic (BACKLOG #3191); ``_app_drain_steps`` -#: and the managed lifespan's teardown hold the steps, their order and each bounded step's bound. -#: Where the lock-notice step's bound is below these two together, that bound cuts this drain -#: before it cancels its tasks, and a task it has not cancelled runs on past the drain, as the -#: step's own comment says. A change to either is read against that budget. +#: and the managed lifespan's teardown hold the steps, their order and each step's floor, from which +#: ``_run_teardown`` computes the step's bound. Where the lock-notice step's bound is below +#: :data:`_BACKGROUND_DRAIN_SECONDS`, the teardown cuts this drain before it cancels anything; +#: ``docs/SECURITY.md``, *Residual: the background task adds one way to lose a notice*, says what +#: that costs. A change to either constant is read against that budget. _BACKGROUND_DRAIN_SECONDS: Final = 2.0 _BACKGROUND_CANCEL_GRACE_SECONDS: Final = 0.5 From 74becfb9711763dedcaf72866da96c57114db7ae Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Sat, 17 Oct 2026 13:41:57 -0500 Subject: [PATCH 49/57] A security notifier closing or closed refuses a notice rather than report it queued (BACKLOG #3120) At shutdown the teardown can cut the lock-notice drain before it cancels a pending notice. That notice ran on, reached the security notifier after aclose(), and was put on a queue nothing drains, so notify() answered True and auth.lock_notice recorded mailed: true for a mail never sent, holding the account's next notice of the kind back for 24 hours. SecurityEventNotifier now sets a closing flag at the start of aclose(), before the stop sentinel goes in, and notify() answers False from then on. The lock notice's row then reads mailed: false. No undelivered audit row is started for such a notice: one started then could outlive aclose() and meet the closed store. The teardown step in api/app.py is unchanged; only its comment is corrected. docs/SECURITY.md: the #2216 residual says what holds now, and the stale "drains its queue after the store has closed" bullet is replaced (stale since #3191). --- changelog.d/3120.security.md | 6 +- docs/SECURITY.md | 23 ++++-- messagefoundry/api/app.py | 4 +- messagefoundry/auth/notifications.py | 4 +- messagefoundry/auth/service.py | 3 +- messagefoundry/pipeline/security_notify.py | 21 +++++- tests/test_lock_notice_off_request_path.py | 88 +++++++++++++++++++++- 7 files changed, 134 insertions(+), 15 deletions(-) diff --git a/changelog.d/3120.security.md b/changelog.d/3120.security.md index caa79d5e4a..7d6b8d84be 100644 --- a/changelog.d/3120.security.md +++ b/changelog.d/3120.security.md @@ -1,8 +1,10 @@ - **A lock notice the relay's full queue drops no longer holds the next one back.** The security notifier now says whether it queued a notice, and the `auth.lock_notice` row records `mailed: false` for one it dropped, so the next lock of the kind mails again rather than staying - quiet for 24 hours. A send that fails later, on the relay's own task, still leaves - `mailed: true`; `docs/SECURITY.md` states that residual. + quiet for 24 hours. The same holds at shutdown: a lock notice that reaches the notifier once it + has begun to close, which a cut shutdown drain allows, is refused and recorded `mailed: false` + rather than reported queued for a mail that is never sent. A send that fails later, on the + relay's own task, still leaves `mailed: true`; `docs/SECURITY.md` states that residual. **The lock notice's throttle read has an index.** A new `ix_audit_actor` index on `audit_log(actor, action, id)`, on SQLite, Postgres and SQL Server, lets the read find one account's rows instead of walking every recent notice row of every account. On Postgres it keys diff --git a/docs/SECURITY.md b/docs/SECURITY.md index 7c1f4e7165..b3e75b6fff 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -5495,8 +5495,8 @@ stays broken, or recovers and breaks again, is reported only by that first line. **A notice the relay loses is audited (BACKLOG #2383).** The `auth.lock_notice` row is written when the notice is handed to the relay. It reads `mailed: true` only when the relay queued the -notice: one its full queue dropped reads `mailed: false`, so the next lock of the kind mails again -(vault BACKLOG #3120). When the relay's queue is full, or the +notice: one its full queue dropped, or one it refused because it had begun to close, reads +`mailed: false`, so the next lock of the kind mails again (vault BACKLOG #3120). When the relay's queue is full, or the send fails, the engine writes one more row, with the notice's recipient as the actor. A lock notice writes `auth.lock_notice_undelivered`, hidden as above. Every other notice kind writes `auth.security_notice_undelivered`, which every `audit:read` reader sees. The detail names the @@ -5505,8 +5505,13 @@ holder. The row never carries the address, the rest of the notice's detail, or t source of record is `SecurityEventNotifier._record_undelivered` in `messagefoundry/pipeline/security_notify.py`. **Still open, at least:** -- At shutdown the notifier drains its queue after the store has closed, so a send that fails then - is not recorded. +- At shutdown the notifier closes while the store is still open (BACKLOG #3191), so a send that + fails in its final drain is recorded. That close is one bounded step of the shutdown, and a + notice still queued when the step is cut is neither sent nor recorded. A lock notice among them + keeps its `auth.lock_notice` row reading `mailed: true`. +- A notice that reaches the notifier once it has begun to close is refused, and the notifier + writes no undelivered row for it. A lock notice's `auth.lock_notice` row reads `mailed: false` + instead. - An offline command such as `provision-admin` wires no audit writer, so it records nothing. - Under a long overload only 64 queue-full records are written at a time, and drops past that are not recorded. @@ -5521,10 +5526,12 @@ source of record is `SecurityEventNotifier._record_undelivered` in engine waits at most two seconds for pending notices, then cancels any still pending, each with one line. A cancelled notice may be neither mailed nor recorded. The API lifespan's shutdown budget (BACKLOG #3191) can cut that wait short before it cancels anything, whenever the lock-notice step's -bound falls below those two seconds. A pending notice is then not cancelled and runs on. If it -reaches the security notifier after that notifier has closed, the notifier still takes it and -reports it queued, so its `auth.lock_notice` row says `mailed: true` for a mail that is never sent, -and the throttle holds back that account's next notice of that kind for 24 hours. A failed `auth.lock_notice` row write also changes: it used to fail +bound falls below those two seconds. A pending notice is then not cancelled and runs on. One that +reaches the security notifier before that notifier begins to close is queued and drained like any +other. One that reaches it later is refused, since nothing would send it (vault BACKLOG #3120), so +no mail goes and its `auth.lock_notice` row says `mailed: false`, which holds back no later notice. +If the store has closed by then, that row write fails and is logged, which costs at most a +duplicate mail at the next lock. A failed `auth.lock_notice` row write also changes: it used to fail the request, and now it is logged instead. The mail goes before its row, so that failure costs at most a duplicate mail at the next lock. Each such line names no account and no lock. It still tells a `logs:view` reader when some notice failed. diff --git a/messagefoundry/api/app.py b/messagefoundry/api/app.py index ef7af8dd91..aecb4041a9 100644 --- a/messagefoundry/api/app.py +++ b/messagefoundry/api/app.py @@ -1257,7 +1257,9 @@ async def _outliving(bound: float) -> None: if auth is not None and owns_auth: # close_background is the shutdown call the service names, with its own bound (its drain # and cancel grace, in auth/service.py). Only a bound below that cuts it before it cancels - # its notices itself; a notice cut that way meets the closed store and logs its own failure. + # its notices itself. A notice cut that way runs on: the security notifier queues it if it + # arrives before that notifier begins to close, and refuses it after, so its row reads + # mailed: false, or meets the closed store and logs its own failure (vault BACKLOG #3120). steps.append( _TeardownStep("the lock-notice drain", 1.0, lambda _b: auth.close_background()) ) diff --git a/messagefoundry/auth/notifications.py b/messagefoundry/auth/notifications.py index c06a1f3b0d..5179470bf8 100644 --- a/messagefoundry/auth/notifications.py +++ b/messagefoundry/auth/notifications.py @@ -236,7 +236,9 @@ class SecurityNotifier(Protocol): must not raise into the auth path (the caller still guards it). ``notify`` returns whether it took the event for sending: ``False`` when it dropped the event - on the spot, for example for want of an address or room on its queue (vault BACKLOG #3120). + on the spot, for example for want of an address or room on its queue, or because it is closing + or closed and would never send it (vault BACKLOG #3120). An implementation that can be closed + answers ``False`` from the start of its close. ``True`` is a hand-off, never a delivery receipt: a send that fails later is not reported back. The ``auth.lock_notice`` row's ``mailed`` field is recorded from this, so a notice dropped here does not hold the next lock's notice back for a day.""" diff --git a/messagefoundry/auth/service.py b/messagefoundry/auth/service.py index fe7438f844..5c4b3a16fc 100644 --- a/messagefoundry/auth/service.py +++ b/messagefoundry/auth/service.py @@ -13709,7 +13709,8 @@ async def _record_lock_notice(self, user: UserRecord, lock: str, *, handed: bool ``mailed`` is true only when the account had an address AND the notifier took the event (``handed``, BACKLOG #2019). A refused hand-off, or one the notifier dropped on a full - queue (vault BACKLOG #3120), then writes ``mailed: false``, which holds back no later + queue or refused once it had begun to close at shutdown (vault BACKLOG #3120), then writes + ``mailed: false``, which holds back no later mailable notice, so the next lock mails again rather than staying quiet for a day. **A send that fails after the hand-off still leaves ``mailed: true``.** It fails on the diff --git a/messagefoundry/pipeline/security_notify.py b/messagefoundry/pipeline/security_notify.py index 76ed4419a8..297cf0f04f 100644 --- a/messagefoundry/pipeline/security_notify.py +++ b/messagefoundry/pipeline/security_notify.py @@ -579,6 +579,10 @@ def __init__( # The queue-full records still being written, held so the loop keeps them alive and so # aclose() can wait for them. self._recording: set[asyncio.Task[None]] = set() + # Set when aclose() begins, never cleared (vault BACKLOG #3120). An event queued after that + # sits behind the stop sentinel, or on a queue nothing reads, and is never sent, so notify() + # refuses it rather than report it queued. + self._closing = False # Logged now, once, rather than at the first send (BACKLOG #1131): the first send may be a # lock notice, and a line appearing then would show a logs:view reader that a lock landed. if use_tls and not tls_verify: @@ -586,7 +590,8 @@ def __init__( async def notify(self, event: SecurityEvent) -> bool: # Returns whether the event was queued for sending, the contract on ``SecurityNotifier``: - # False for no address and for a full queue (vault BACKLOG #3120), never a delivery receipt. + # False for no address, a full queue, or a notifier closing or closed (vault BACKLOG #3120), + # never a delivery receipt. # # No deliverable address means nothing to email. The caller has audited the event; which # events the /me/security-events feed shows is stated once, in auth/notifications.py. @@ -629,6 +634,19 @@ async def notify(self, event: SecurityEvent) -> bool: event.username, ) return False + if self._closing: + # At shutdown: the teardown can cut the lock-notice drain before it cancels a pending + # notice, which then reaches this after the close (vault BACKLOG #3120). No audit row: + # one started now could outlive aclose() and meet the closed store. A lock notice's own + # ``auth.lock_notice`` row says ``mailed: false`` instead, and writes no line here. + if event.event_type not in LOG_SILENT_EVENT_TYPES: + log.warning( + "%s closed; dropping %s for %s", + type(self).__name__, + event.event_type, + event.username, + ) + return False queued = self._enqueue( event, dropped=None @@ -644,6 +662,7 @@ async def notify(self, event: SecurityEvent) -> bool: return queued async def aclose(self) -> None: + self._closing = True # before the sentinel goes in, so nothing is queued behind it try: await super().aclose() finally: diff --git a/tests/test_lock_notice_off_request_path.py b/tests/test_lock_notice_off_request_path.py index 015669e265..e6f901dc73 100644 --- a/tests/test_lock_notice_off_request_path.py +++ b/tests/test_lock_notice_off_request_path.py @@ -16,17 +16,19 @@ import asyncio import json import logging +import threading import time from collections.abc import AsyncIterator, Callable from contextlib import asynccontextmanager from pathlib import Path from typing import Any, Literal +from uuid import uuid4 import pytest from messagefoundry.api import create_managed_app from messagefoundry.auth import service as service_module -from messagefoundry.auth.notifications import ACCOUNT_LOCKED, SecurityEvent +from messagefoundry.auth.notifications import ACCOUNT_LOCKED, PASSWORD_CHANGED, SecurityEvent from messagefoundry.auth.service import AuthService from messagefoundry.config.settings import AlertsSettings, AuthSettings, EgressSettings from messagefoundry.keyed_lock import hold_keyed_lock @@ -817,6 +819,90 @@ async def test_a_notice_dropped_on_a_full_queue_does_not_hold_the_next_one_back( await _close(store, service) +async def test_the_shipped_notifier_queues_nothing_once_it_has_closed() -> None: + """Vault BACKLOG #3120: after ``aclose`` nothing drains the queue, so an event ``notify`` put + there would never be sent. It answers ``False`` instead. The control is the same call on the + same notifier before it closed, which answers ``True``.""" + relay = _relay() + relay.start() + assert await relay.notify(SecurityEvent(ACCOUNT_LOCKED, username="a", email="a@x.test")) + relay._queue.get_nowait() # never sent: the drain loop has not run yet + await relay.aclose() + late = SecurityEvent(ACCOUNT_LOCKED, username="b", email="b@x.test") + assert await relay.notify(late) is False + assert relay._queue.qsize() == 0, "the late event was queued where nothing reads it" + + +async def test_the_shipped_notifier_queues_nothing_while_it_closes( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Vault BACKLOG #3120: once ``aclose`` has begun, an event queued behind its stop sentinel is + never drained, so ``notify`` answers ``False`` from the start of the close, not its end. The + close is held on a send in flight, and the control is that held event, which is sent.""" + started, release = threading.Event(), threading.Event() + sent: list[list[str]] = [] + + def held_send(**kw: Any) -> None: + started.set() + release.wait(_STALL_SECONDS) + sent.append(kw["recipients"]) + + monkeypatch.setattr("messagefoundry.pipeline.security_notify.send_plain_email", held_send) + relay = _relay() + # Room behind the stop sentinel, so a full queue cannot be what refuses the late event. + relay._queue = asyncio.Queue(maxsize=8) + relay.start() + assert await relay.notify(SecurityEvent(PASSWORD_CHANGED, username="a", email="a@x.test")) + assert await asyncio.to_thread(started.wait, _STALL_SECONDS), "the first send never started" + closing = asyncio.create_task(relay.aclose()) + try: + await asyncio.sleep(0) # aclose has begun and is waiting on the drain + late = SecurityEvent(PASSWORD_CHANGED, username="b", email="b@x.test") + assert await relay.notify(late) is False + finally: + release.set() + await closing + assert sent == [["a@x.test"]], "the event taken before the close was not sent" + + +async def test_a_lock_notice_raised_after_the_relay_closed_records_mailed_false() -> None: + """Vault BACKLOG #3120. At shutdown the teardown can cut the lock-notice drain before it cancels + a pending notice, which then runs on and reaches the security notifier after that has closed. + The notifier used to take it and answer ``True``, so the row read ``mailed: true`` for a mail + that is never sent, and the throttle held the account's next notice of the kind back for a day. + The row now reads ``mailed: false``, so that next lock mails again. + + The control is a second account's lock, before the close, whose row reads ``mailed: true``.""" + store, service, _fake, user = await _harness() + relay = _relay(store) + service.attach_security_notifier(relay) + try: + other_id = uuid4().hex + await store.create_user( + user_id=other_id, + username="other-holder", + auth_provider="local", + password_generated=False, + ) + await store.set_user_notify_email(other_id, email="other@example.org") + other_user = await store.get_user(other_id) + assert other_user is not None + await service._record_lock(other_user, "sign_in", _lock(1), client=None) + await service.drain_background() + before = await _lock_notice_rows(store, other_user.username) + assert [json.loads(r["detail"])["mailed"] for r in before] == [True] + relay._queue.get_nowait() # room again, so only the close can refuse the next one + + await relay.aclose() + await service._record_lock(user, "sign_in", _lock(1), client=None) + await service.drain_background() + rows = await _lock_notice_rows(store, user.username) + assert [json.loads(r["detail"]) for r in rows] == [{"lock": "sign_in", "mailed": False}] + assert relay._queue.qsize() == 0, "the late notice was queued where nothing reads it" + finally: + await _close(store, service) + + async def test_after_a_closing_drain_a_notice_runs_inline(monkeypatch: pytest.MonkeyPatch) -> None: """``close_background`` is the shutdown drain. A lock that lands after it, from a request still finishing, runs its notice inline rather than starting a task that would outlive From 6677c8e57370eba3727425a7476429b04a252478 Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Sat, 17 Oct 2026 13:54:04 -0500 Subject: [PATCH 50/57] Review round one on the closing refusal: record the drop, scrub the line, restartable (BACKLOG #3120) - A notice refused while the notifier closes is recorded on its undelivered row with reason "closing", as a full queue's drop is, so the full-queue-at- close window keeps the BACKLOG #2383 record it used to get. - The refusal warning names the kind through notice_kind_log_label and the username through scrub_log_argument, and says "is closing". - start() after a close clears the flag, so a restarted notifier sends again. - Lists of why a lock notice reads mailed: false now include the closing notifier, or point at the SecurityNotifier contract (notifications.py, service.py _notify_security, docs/SECURITY.md). - SECURITY.md: a cut close can also lose or unrecord a send under way, and a mailed: true row then holds the next lock notice back for 24 hours. - Tests close the relay in finally; the end-to-end test runs a started relay with its drain loop, sends the control, and reads the closing record. --- changelog.d/3120.security.md | 3 +- docs/SECURITY.md | 23 +++++----- messagefoundry/auth/notifications.py | 9 ++-- messagefoundry/auth/service.py | 4 +- messagefoundry/pipeline/security_notify.py | 48 ++++++++++++-------- tests/test_lock_notice_off_request_path.py | 52 ++++++++++++++++------ 6 files changed, 90 insertions(+), 49 deletions(-) diff --git a/changelog.d/3120.security.md b/changelog.d/3120.security.md index 7d6b8d84be..8dedf6f128 100644 --- a/changelog.d/3120.security.md +++ b/changelog.d/3120.security.md @@ -3,7 +3,8 @@ `mailed: false` for one it dropped, so the next lock of the kind mails again rather than staying quiet for 24 hours. The same holds at shutdown: a lock notice that reaches the notifier once it has begun to close, which a cut shutdown drain allows, is refused and recorded `mailed: false` - rather than reported queued for a mail that is never sent. A send that fails later, on the + rather than reported queued for a mail that is never sent. The relay also records such a drop, + with the new undelivered-notice reason `closing`. A send that fails later, on the relay's own task, still leaves `mailed: true`; `docs/SECURITY.md` states that residual. **The lock notice's throttle read has an index.** A new `ix_audit_actor` index on `audit_log(actor, action, id)`, on SQLite, Postgres and SQL Server, lets the read find one diff --git a/docs/SECURITY.md b/docs/SECURITY.md index b3e75b6fff..4ee283f2ab 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -5444,7 +5444,7 @@ and header are that route's. relay (`mailed: false`, `reason: no_notifier`) and no address (`mailed: false`) are on the `auth.lock_notice` row. A full queue and a failed send are on an `auth.lock_notice_undelivered` row (BACKLOG #2383), and a full queue also reads `mailed: false` on the `auth.lock_notice` row - (vault BACKLOG #3120). A relay that is down still shows, on every other notice kind. An instance with no relay at all is reported at + (vault BACKLOG #3120). So does a notice the relay refused because it had begun to close. A relay that is down still shows, on every other notice kind. An instance with no relay at all is reported at startup by the serve gate, except under `[security].enforcement = "warn"` with `[alerts].security_notifications_required = false`. Every other notice kind keeps its per-event line (BACKLOG #1139); none of them fires on a refused sign-in. The list is @@ -5496,22 +5496,25 @@ stays broken, or recovers and breaks again, is reported only by that first line. **A notice the relay loses is audited (BACKLOG #2383).** The `auth.lock_notice` row is written when the notice is handed to the relay. It reads `mailed: true` only when the relay queued the notice: one its full queue dropped, or one it refused because it had begun to close, reads -`mailed: false`, so the next lock of the kind mails again (vault BACKLOG #3120). When the relay's queue is full, or the -send fails, the engine writes one more row, with the notice's recipient as the actor. A lock +`mailed: false`, so the next lock of the kind mails again (vault BACKLOG #3120). When the relay's queue is full, the relay refuses the notice because it has +begun to close, or the send fails, the engine writes one more row, with the notice's recipient as +the actor. A lock notice writes `auth.lock_notice_undelivered`, hidden as above. Every other notice kind writes `auth.security_notice_undelivered`, which every `audit:read` reader sees. The detail names the -notice kind and the `reason`, `queue_full` or `send_failed`. An issuer's reminder also names the +notice kind and the `reason`, `queue_full`, `closing` or `send_failed`. An issuer's reminder also names the holder. The row never carries the address, the rest of the notice's detail, or the error. The source of record is `SecurityEventNotifier._record_undelivered` in `messagefoundry/pipeline/security_notify.py`. **Still open, at least:** - At shutdown the notifier closes while the store is still open (BACKLOG #3191), so a send that - fails in its final drain is recorded. That close is one bounded step of the shutdown, and a - notice still queued when the step is cut is neither sent nor recorded. A lock notice among them - keeps its `auth.lock_notice` row reading `mailed: true`. -- A notice that reaches the notifier once it has begun to close is refused, and the notifier - writes no undelivered row for it. A lock notice's `auth.lock_notice` row reads `mailed: false` - instead. + fails in its final drain is recorded. That close is one bounded step of the shutdown. When the + step is cut, a notice still queued is neither sent nor recorded, and a send already under way + may still go out or fail unrecorded. A lock notice among them keeps its `auth.lock_notice` row + reading `mailed: true`, so the throttle holds back that account's next lock notice of the kind + for the rest of its 24 hours. +- A notice that reaches the notifier once it has begun to close is refused and recorded with + `reason: closing`. The close waits only for records started before it finishes. One started + later can meet the closed store, and is then lost like any failed record write. - An offline command such as `provision-admin` wires no audit writer, so it records nothing. - Under a long overload only 64 queue-full records are written at a time, and drops past that are not recorded. diff --git a/messagefoundry/auth/notifications.py b/messagefoundry/auth/notifications.py index 5179470bf8..d99b06bcf7 100644 --- a/messagefoundry/auth/notifications.py +++ b/messagefoundry/auth/notifications.py @@ -127,10 +127,11 @@ #: one, so a line per undelivered lock notice told that reader which candidate was right. #: #: Only ``ACCOUNT_LOCKED`` carries that bit. Every other kind is sent on an authenticated action, a -#: completed sign-in, or a clock, so it keeps its per-event line (BACKLOG #1139). A lock notice with -#: no relay, no address or no room on the notifier's queue is still recorded where only an -#: administrator reads it, as ``mailed: false`` on the ``auth.lock_notice`` row (vault BACKLOG #3120 -#: for the full queue). A full queue or a failed send is also recorded on its own +#: completed sign-in, or a clock, so it keeps its per-event line (BACKLOG #1139). A lock notice the +#: notifier did not take, for at least no relay, no address, no room on its queue or a notifier +#: closing at shutdown, is still recorded where only an administrator reads it, as +#: ``mailed: false`` on the ``auth.lock_notice`` row (vault BACKLOG #3120 for the last two). A full +#: queue, a closing notifier or a failed send is also recorded on its own #: ``auth.lock_notice_undelivered`` row (BACKLOG #2383). A failed send happens after the #: ``auth.lock_notice`` row is written at the hand-off, so that row still reads ``mailed: true``; the #: residual is stated in ``docs/SECURITY.md``. An instance with no relay at all is reported at startup by diff --git a/messagefoundry/auth/service.py b/messagefoundry/auth/service.py index 5c4b3a16fc..82065264ef 100644 --- a/messagefoundry/auth/service.py +++ b/messagefoundry/auth/service.py @@ -13739,8 +13739,8 @@ async def _notify_security( :meth:`_record_suspicious_login` for the two 6.3.5 events. Returns ``True`` only when the notifier took the event without raising (BACKLOG #2019) and - said it took it: the shipped notifier answers ``False`` for an event it dropped for want of - an address or of room on its queue (vault BACKLOG #3120). So a caller that audits the outcome + said it took it: the shipped notifier answers ``False`` for an event it dropped, for the + reasons the ``SecurityNotifier`` contract lists (vault BACKLOG #3120). So a caller that audits the outcome can record what happened. That is a hand-off, not a delivery: a send that fails later, on the notifier's own task, is not reported back. Most callers ignore it. diff --git a/messagefoundry/pipeline/security_notify.py b/messagefoundry/pipeline/security_notify.py index 297cf0f04f..2bb0392616 100644 --- a/messagefoundry/pipeline/security_notify.py +++ b/messagefoundry/pipeline/security_notify.py @@ -579,9 +579,9 @@ def __init__( # The queue-full records still being written, held so the loop keeps them alive and so # aclose() can wait for them. self._recording: set[asyncio.Task[None]] = set() - # Set when aclose() begins, never cleared (vault BACKLOG #3120). An event queued after that - # sits behind the stop sentinel, or on a queue nothing reads, and is never sent, so notify() - # refuses it rather than report it queued. + # Set when aclose() begins, cleared only by a fresh start() (vault BACKLOG #3120). An event + # queued after that sits behind the stop sentinel, or on a queue nothing reads, and is never + # sent, so notify() refuses it; the contract on ``SecurityNotifier`` says why. self._closing = False # Logged now, once, rather than at the first send (BACKLOG #1131): the first send may be a # lock notice, and a line appearing then would show a logs:view reader that a lock landed. @@ -589,9 +589,8 @@ def __init__( warn_smtp_verification_off(cell=ALERTS_SMTP_CELL, host=host) async def notify(self, event: SecurityEvent) -> bool: - # Returns whether the event was queued for sending, the contract on ``SecurityNotifier``: - # False for no address, a full queue, or a notifier closing or closed (vault BACKLOG #3120), - # never a delivery receipt. + # Returns whether the event was queued for sending, the contract on ``SecurityNotifier``, + # which lists why it may answer False. Never a delivery receipt. # # No deliverable address means nothing to email. The caller has audited the event; which # events the /me/security-events feed shows is stated once, in auth/notifications.py. @@ -635,17 +634,18 @@ async def notify(self, event: SecurityEvent) -> bool: ) return False if self._closing: - # At shutdown: the teardown can cut the lock-notice drain before it cancels a pending - # notice, which then reaches this after the close (vault BACKLOG #3120). No audit row: - # one started now could outlive aclose() and meet the closed store. A lock notice's own - # ``auth.lock_notice`` row says ``mailed: false`` instead, and writes no line here. + # At shutdown, for example a lock notice the teardown's cut drain left running (vault + # BACKLOG #3120). Recorded like a full queue's drop. A record started after aclose() + # has finished is not waited for, and may meet the closed store, where it fails and is + # swallowed like any other. if event.event_type not in LOG_SILENT_EVENT_TYPES: log.warning( - "%s closed; dropping %s for %s", + "%s is closing; dropping %s for %s", type(self).__name__, - event.event_type, - event.username, + notice_kind_log_label(event.event_type), + scrub_log_argument(event.username), ) + self._record_dropped(event, "closing") return False queued = self._enqueue( event, @@ -653,14 +653,24 @@ async def notify(self, event: SecurityEvent) -> bool: if event.event_type in LOG_SILENT_EVENT_TYPES else f"{event.event_type} for {event.username}", ) - if not queued and self._audit is not None and len(self._recording) < _MAX_RECORDING: - # Off the caller's path: a notice must never block a sign-in or an admin action, and - # this write would wait on the store under exactly the overload that filled the queue. - task = asyncio.create_task(self._record_undelivered(event, {"reason": "queue_full"})) - self._recording.add(task) - task.add_done_callback(self._recording.discard) + if not queued: + self._record_dropped(event, "queue_full") return queued + def _record_dropped(self, event: SecurityEvent, reason: str) -> None: + # Off the caller's path: a notice must never block a sign-in or an admin action, and this + # write would wait on the store under exactly the overload that filled the queue. + if self._audit is None or len(self._recording) >= _MAX_RECORDING: + return + task = asyncio.create_task(self._record_undelivered(event, {"reason": reason})) + self._recording.add(task) + task.add_done_callback(self._recording.discard) + + def start(self) -> None: + if self._task is None: + self._closing = False # a notifier started again after its close sends again + super().start() + async def aclose(self) -> None: self._closing = True # before the sentinel goes in, so nothing is queued behind it try: diff --git a/tests/test_lock_notice_off_request_path.py b/tests/test_lock_notice_off_request_path.py index e6f901dc73..1307b7ea35 100644 --- a/tests/test_lock_notice_off_request_path.py +++ b/tests/test_lock_notice_off_request_path.py @@ -822,15 +822,23 @@ async def test_a_notice_dropped_on_a_full_queue_does_not_hold_the_next_one_back( async def test_the_shipped_notifier_queues_nothing_once_it_has_closed() -> None: """Vault BACKLOG #3120: after ``aclose`` nothing drains the queue, so an event ``notify`` put there would never be sent. It answers ``False`` instead. The control is the same call on the - same notifier before it closed, which answers ``True``.""" + same notifier before it closed, which answers ``True``. A fresh ``start`` takes events again.""" relay = _relay() relay.start() - assert await relay.notify(SecurityEvent(ACCOUNT_LOCKED, username="a", email="a@x.test")) - relay._queue.get_nowait() # never sent: the drain loop has not run yet - await relay.aclose() + try: + assert await relay.notify(SecurityEvent(ACCOUNT_LOCKED, username="a", email="a@x.test")) + relay._queue.get_nowait() # never sent: the drain loop has not run yet + finally: + await relay.aclose() late = SecurityEvent(ACCOUNT_LOCKED, username="b", email="b@x.test") assert await relay.notify(late) is False assert relay._queue.qsize() == 0, "the late event was queued where nothing reads it" + relay.start() + try: + assert await relay.notify(late) is True, "a notifier started again refused an event" + relay._queue.get_nowait() + finally: + await relay.aclose() async def test_the_shipped_notifier_queues_nothing_while_it_closes( @@ -852,30 +860,43 @@ def held_send(**kw: Any) -> None: # Room behind the stop sentinel, so a full queue cannot be what refuses the late event. relay._queue = asyncio.Queue(maxsize=8) relay.start() - assert await relay.notify(SecurityEvent(PASSWORD_CHANGED, username="a", email="a@x.test")) - assert await asyncio.to_thread(started.wait, _STALL_SECONDS), "the first send never started" - closing = asyncio.create_task(relay.aclose()) + closing: asyncio.Task[None] | None = None try: + assert await relay.notify(SecurityEvent(PASSWORD_CHANGED, username="a", email="a@x.test")) + assert await asyncio.to_thread(started.wait, _STALL_SECONDS), "the send never started" + closing = asyncio.create_task(relay.aclose()) await asyncio.sleep(0) # aclose has begun and is waiting on the drain late = SecurityEvent(PASSWORD_CHANGED, username="b", email="b@x.test") assert await relay.notify(late) is False finally: release.set() - await closing + await (closing if closing is not None else relay.aclose()) assert sent == [["a@x.test"]], "the event taken before the close was not sent" -async def test_a_lock_notice_raised_after_the_relay_closed_records_mailed_false() -> None: +async def test_a_lock_notice_raised_after_the_relay_closed_records_mailed_false( + monkeypatch: pytest.MonkeyPatch, +) -> None: """Vault BACKLOG #3120. At shutdown the teardown can cut the lock-notice drain before it cancels a pending notice, which then runs on and reaches the security notifier after that has closed. The notifier used to take it and answer ``True``, so the row read ``mailed: true`` for a mail that is never sent, and the throttle held the account's next notice of the kind back for a day. - The row now reads ``mailed: false``, so that next lock mails again. + The row now reads ``mailed: false``, so that next lock mails again, and the relay records the + drop with ``reason: closing``. - The control is a second account's lock, before the close, whose row reads ``mailed: true``.""" + The relay is started and its drain loop runs, as at a real shutdown. The control is a second + account's lock before the close, whose notice is sent and whose row reads ``mailed: true``.""" + sent: list[list[str]] = [] + monkeypatch.setattr( + "messagefoundry.pipeline.security_notify.send_plain_email", + lambda **kw: sent.append(kw["recipients"]), + ) store, service, _fake, user = await _harness() - relay = _relay(store) + relay = SecurityEventNotifier( + host="smtp.example.test", port=25, sender="mf@example.test", audit=store + ) service.attach_security_notifier(relay) + relay.start() try: other_id = uuid4().hex await store.create_user( @@ -891,15 +912,20 @@ async def test_a_lock_notice_raised_after_the_relay_closed_records_mailed_false( await service.drain_background() before = await _lock_notice_rows(store, other_user.username) assert [json.loads(r["detail"])["mailed"] for r in before] == [True] - relay._queue.get_nowait() # room again, so only the close can refuse the next one await relay.aclose() + assert sent == [["other@example.org"]], "the control's notice was not sent" await service._record_lock(user, "sign_in", _lock(1), client=None) await service.drain_background() rows = await _lock_notice_rows(store, user.username) assert [json.loads(r["detail"]) for r in rows] == [{"lock": "sign_in", "mailed": False}] assert relay._queue.qsize() == 0, "the late notice was queued where nothing reads it" + await asyncio.gather(*relay._recording) + dropped = await store.list_audit(actor=user.username, action=_LOCK_NOTICE_UNDELIVERED) + assert [json.loads(r["detail"])["reason"] for r in dropped] == ["closing"] + assert sent == [["other@example.org"]], "a mail went out after the close" finally: + await relay.aclose() await _close(store, service) From f7ba6d64f6f39fb2568600ab3c3859c966c4da76 Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Sat, 17 Oct 2026 14:06:43 -0500 Subject: [PATCH 51/57] b257-mailfix handover: PR title unchanged, ledger banner addendum (BACKLOG #3120) No code change. This commit carries the handover text for the draft PR 2387. PR title: unchanged from the b257-dtail head; this limb adds to #3120. Ledger banner addendum for vault BACKLOG #3120: A security notifier that has begun to close, or has closed, now refuses a notice (notify answers False) instead of queueing it where nothing drains it. A lock notice that a cut shutdown drain leaves running therefore records auth.lock_notice mailed: false, not mailed: true, and the relay records the drop on its undelivered row with reason "closing". The teardown in api/app.py is unchanged. docs/SECURITY.md states what holds after the fix and drops the stale "drains its queue after the store has closed" bullet (stale since #3191). Review round two left findings open; they are in the Builder's report. From d89d00e162cf44577a31fe9b36e525f6c12f843f Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Sat, 17 Oct 2026 14:22:37 -0500 Subject: [PATCH 52/57] A closed security notifier refuses to start again; the throttle line says what happens (BACKLOG #3120) Review round two on the closing refusal found two defects the previous limb introduced. 1. After a cut close, start() cleared _closing, but the stop sentinel stayed on the queue, so the restarted drain loop stopped on it and notify() answered True for events nothing sends. start() now raises RuntimeError once aclose() has begun, and the notifier keeps refusing. No engine path restarts a closed notifier: each lifespan builds its own. The test that a fresh start sends again is replaced by one that a closed notifier refuses to start, and a new test cuts a close mid-send, calls start(), and requires that a notice reported queued is sent. It failed on f7ba6d64. 2. The throttle read's fail-open line said the notice "was sent unthrottled". It now says the notice is offered to the notifier unthrottled and the notifier's answer decides whether it is recorded as mailed. docs/SECURITY.md no longer says a failed row write after a refused notice "costs at most a duplicate mail": no mail goes, and no row holds back the next notice. PR title: unchanged from the b257-dtail head; this limb adds to #3120. Ledger banner addendum: a closed security notifier now refuses to start again, so a cut close cannot leave a drain loop that reports notices queued and never sends them. --- docs/SECURITY.md | 5 +- messagefoundry/auth/service.py | 3 +- messagefoundry/pipeline/security_notify.py | 10 +++- tests/test_lock_notice_off_request_path.py | 65 +++++++++++++++++++++- 4 files changed, 74 insertions(+), 9 deletions(-) diff --git a/docs/SECURITY.md b/docs/SECURITY.md index 4ee283f2ab..0c55ce070e 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -5533,8 +5533,9 @@ bound falls below those two seconds. A pending notice is then not cancelled and reaches the security notifier before that notifier begins to close is queued and drained like any other. One that reaches it later is refused, since nothing would send it (vault BACKLOG #3120), so no mail goes and its `auth.lock_notice` row says `mailed: false`, which holds back no later notice. -If the store has closed by then, that row write fails and is logged, which costs at most a -duplicate mail at the next lock. A failed `auth.lock_notice` row write also changes: it used to fail +If the store has closed by then, the throttle read before the refusal fails open and the row write +after it fails, each logged. No mail goes and no row is written, so nothing holds back the next +lock's notice of the kind. A failed `auth.lock_notice` row write also changes: it used to fail the request, and now it is logged instead. The mail goes before its row, so that failure costs at most a duplicate mail at the next lock. Each such line names no account and no lock. It still tells a `logs:view` reader when some notice failed. diff --git a/messagefoundry/auth/service.py b/messagefoundry/auth/service.py index 82065264ef..3f20466732 100644 --- a/messagefoundry/auth/service.py +++ b/messagefoundry/auth/service.py @@ -13689,7 +13689,8 @@ async def _lock_notice_held_back(self, user: UserRecord, lock: str) -> bool: ) except Exception as exc: # noqa: BLE001 - fails open by design, and is logged _log.error( - "a security-notice throttle read failed (%s); the notice was sent unthrottled", + "a security-notice throttle read failed (%s); the notice is offered to the notifier " + "unthrottled, and the notifier's answer decides whether it is recorded as mailed", type(exc).__name__, ) return False diff --git a/messagefoundry/pipeline/security_notify.py b/messagefoundry/pipeline/security_notify.py index 2bb0392616..47d5436594 100644 --- a/messagefoundry/pipeline/security_notify.py +++ b/messagefoundry/pipeline/security_notify.py @@ -579,7 +579,7 @@ def __init__( # The queue-full records still being written, held so the loop keeps them alive and so # aclose() can wait for them. self._recording: set[asyncio.Task[None]] = set() - # Set when aclose() begins, cleared only by a fresh start() (vault BACKLOG #3120). An event + # Set when aclose() begins, and never cleared: start() refuses (vault BACKLOG #3120). An event # queued after that sits behind the stop sentinel, or on a queue nothing reads, and is never # sent, so notify() refuses it; the contract on ``SecurityNotifier`` says why. self._closing = False @@ -667,8 +667,12 @@ def _record_dropped(self, event: SecurityEvent, reason: str) -> None: task.add_done_callback(self._recording.discard) def start(self) -> None: - if self._task is None: - self._closing = False # a notifier started again after its close sends again + # Never again once closed (vault BACKLOG #3120). A close the teardown cut short leaves its + # stop sentinel on the queue, so a drain loop started again would read it and stop, and + # notify() would answer True for events nothing sends. The engine builds a new notifier for + # each lifespan rather than restart one. + if self._closing: + raise RuntimeError(f"{type(self).__name__} has been closed and cannot be started again") super().start() async def aclose(self) -> None: diff --git a/tests/test_lock_notice_off_request_path.py b/tests/test_lock_notice_off_request_path.py index 1307b7ea35..5f793d9465 100644 --- a/tests/test_lock_notice_off_request_path.py +++ b/tests/test_lock_notice_off_request_path.py @@ -14,6 +14,7 @@ from __future__ import annotations import asyncio +import contextlib import json import logging import threading @@ -822,7 +823,7 @@ async def test_a_notice_dropped_on_a_full_queue_does_not_hold_the_next_one_back( async def test_the_shipped_notifier_queues_nothing_once_it_has_closed() -> None: """Vault BACKLOG #3120: after ``aclose`` nothing drains the queue, so an event ``notify`` put there would never be sent. It answers ``False`` instead. The control is the same call on the - same notifier before it closed, which answers ``True``. A fresh ``start`` takes events again.""" + same notifier before it closed, which answers ``True``.""" relay = _relay() relay.start() try: @@ -833,12 +834,70 @@ async def test_the_shipped_notifier_queues_nothing_once_it_has_closed() -> None: late = SecurityEvent(ACCOUNT_LOCKED, username="b", email="b@x.test") assert await relay.notify(late) is False assert relay._queue.qsize() == 0, "the late event was queued where nothing reads it" + + +async def test_a_closed_notifier_refuses_to_start_again() -> None: + """Vault BACKLOG #3120, review round two. A close can leave its stop sentinel on the queue, so a + drain loop started again could stop on it at once. ``start`` after ``aclose`` therefore raises, + and the notifier goes on refusing events. The control is ``start`` on a notifier never closed, + which raises nothing.""" + relay = _relay() + relay.start() # the control + await relay.aclose() + with pytest.raises(RuntimeError, match="closed"): + relay.start() + assert relay._task is None, "a drain loop was started on a closed notifier" + late = SecurityEvent(ACCOUNT_LOCKED, username="b", email="b@x.test") + assert await relay.notify(late) is False + + +async def test_after_a_cut_close_a_notice_reported_queued_is_sent( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Vault BACKLOG #3120, review round two. The teardown bounds the notifier's close, and a close + cut while a send is under way leaves its stop sentinel on the queue. A drain loop started again + then read that sentinel and stopped, and ``notify`` answered ``True`` for an event nothing ever + sent. Whatever ``start`` does after a cut close, a notice reported queued must be sent. + + The control is the event taken before the close: its send was under way when the close was + cut, and once released it goes out and shows in ``sent``.""" + started, release = threading.Event(), threading.Event() + sent: list[list[str]] = [] + + def held_send(**kw: Any) -> None: + if kw["recipients"] == ["a@x.test"]: + started.set() + release.wait(_STALL_SECONDS) + sent.append(kw["recipients"]) + + monkeypatch.setattr("messagefoundry.pipeline.security_notify.send_plain_email", held_send) + relay = _relay() + # Room behind the stop sentinel, so a full queue cannot be what refuses the late event. + relay._queue = asyncio.Queue(maxsize=8) relay.start() + answered = False try: - assert await relay.notify(late) is True, "a notifier started again refused an event" - relay._queue.get_nowait() + assert await relay.notify(SecurityEvent(PASSWORD_CHANGED, username="a", email="a@x.test")) + assert await asyncio.to_thread(started.wait, _STALL_SECONDS), "the send never started" + with pytest.raises(TimeoutError): # the teardown's bound, cutting the close + await asyncio.wait_for(relay.aclose(), 0.05) + assert relay._queue.qsize() == 1, "the cut close left no stop sentinel to test against" + with contextlib.suppress(RuntimeError): # refusing is one way to keep the property below + relay.start() + late = SecurityEvent(PASSWORD_CHANGED, username="b", email="b@x.test") + answered = await relay.notify(late) + if answered: + await relay.aclose() # a live drain loop sends what it took before it stops + assert ["b@x.test"] in sent, "a notice reported queued after a cut close was never sent" finally: + release.set() await relay.aclose() + deadline = time.monotonic() + _STALL_SECONDS + while ["a@x.test"] not in sent and time.monotonic() < deadline: + await asyncio.sleep(0.01) + assert ["a@x.test"] in sent, "the control's send, under way at the cut, never finished" + if not answered: + assert ["b@x.test"] not in sent, "the late notice was refused, yet a mail went for it" async def test_the_shipped_notifier_queues_nothing_while_it_closes( From b0d17045ee4c540ce99487abb1c92e71df9ba9ac Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Sat, 17 Oct 2026 14:33:44 -0500 Subject: [PATCH 53/57] Review on the restart refusal: wording that holds in every order (BACKLOG #3120) The code-review pass at xhigh found the new wording claimed more than holds. A cut close CAN leave its sentinel (not always: a cut during a blocked put leaves none), so the start() comment says "can". The throttle fail-open line no longer claims a hand-off or a recorded outcome, either of which a detach or the same store fault can prevent; it says the throttle did not hold the notice back and the notifier decides. docs/SECURITY.md states the outcome only, without an ordering of the read and the write. The class docstring states the single-use contract. The refusal test's control now asserts the task it starts. --- docs/SECURITY.md | 5 ++--- messagefoundry/auth/service.py | 4 ++-- messagefoundry/pipeline/security_notify.py | 7 +++++-- tests/test_lock_notice_off_request_path.py | 6 +++--- 4 files changed, 12 insertions(+), 10 deletions(-) diff --git a/docs/SECURITY.md b/docs/SECURITY.md index 0c55ce070e..e93257649c 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -5533,9 +5533,8 @@ bound falls below those two seconds. A pending notice is then not cancelled and reaches the security notifier before that notifier begins to close is queued and drained like any other. One that reaches it later is refused, since nothing would send it (vault BACKLOG #3120), so no mail goes and its `auth.lock_notice` row says `mailed: false`, which holds back no later notice. -If the store has closed by then, the throttle read before the refusal fails open and the row write -after it fails, each logged. No mail goes and no row is written, so nothing holds back the next -lock's notice of the kind. A failed `auth.lock_notice` row write also changes: it used to fail +If the store has closed by then, that row write fails and is logged. The notice still went +unmailed, and a missing `mailed: false` row costs nothing at the next lock. A failed `auth.lock_notice` row write also changes: it used to fail the request, and now it is logged instead. The mail goes before its row, so that failure costs at most a duplicate mail at the next lock. Each such line names no account and no lock. It still tells a `logs:view` reader when some notice failed. diff --git a/messagefoundry/auth/service.py b/messagefoundry/auth/service.py index 3f20466732..85cbe3ae9b 100644 --- a/messagefoundry/auth/service.py +++ b/messagefoundry/auth/service.py @@ -13689,8 +13689,8 @@ async def _lock_notice_held_back(self, user: UserRecord, lock: str) -> bool: ) except Exception as exc: # noqa: BLE001 - fails open by design, and is logged _log.error( - "a security-notice throttle read failed (%s); the notice is offered to the notifier " - "unthrottled, and the notifier's answer decides whether it is recorded as mailed", + "a security-notice throttle read failed (%s); the throttle did not hold the notice " + "back, and whether it is mailed is up to the notifier", type(exc).__name__, ) return False diff --git a/messagefoundry/pipeline/security_notify.py b/messagefoundry/pipeline/security_notify.py index 47d5436594..98073f12c7 100644 --- a/messagefoundry/pipeline/security_notify.py +++ b/messagefoundry/pipeline/security_notify.py @@ -541,7 +541,10 @@ def _build_body(event: SecurityEvent) -> str: class SecurityEventNotifier(_BackgroundDispatcher[SecurityEvent]): - """Emails the affected user about a security event, on a bounded background queue.""" + """Emails the affected user about a security event, on a bounded background queue. + + Single use: once :meth:`aclose` has begun, :meth:`notify` refuses every event and :meth:`start` + raises ``RuntimeError`` (vault BACKLOG #3120). Build a new notifier rather than restart one.""" def __init__( self, @@ -667,7 +670,7 @@ def _record_dropped(self, event: SecurityEvent, reason: str) -> None: task.add_done_callback(self._recording.discard) def start(self) -> None: - # Never again once closed (vault BACKLOG #3120). A close the teardown cut short leaves its + # Never again once closed (vault BACKLOG #3120). A close the teardown cut short can leave its # stop sentinel on the queue, so a drain loop started again would read it and stop, and # notify() would answer True for events nothing sends. The engine builds a new notifier for # each lifespan rather than restart one. diff --git a/tests/test_lock_notice_off_request_path.py b/tests/test_lock_notice_off_request_path.py index 5f793d9465..7be5f78003 100644 --- a/tests/test_lock_notice_off_request_path.py +++ b/tests/test_lock_notice_off_request_path.py @@ -14,13 +14,12 @@ from __future__ import annotations import asyncio -import contextlib import json import logging import threading import time from collections.abc import AsyncIterator, Callable -from contextlib import asynccontextmanager +from contextlib import asynccontextmanager, suppress from pathlib import Path from typing import Any, Literal from uuid import uuid4 @@ -843,6 +842,7 @@ async def test_a_closed_notifier_refuses_to_start_again() -> None: which raises nothing.""" relay = _relay() relay.start() # the control + assert relay._task is not None, "the instrument cannot see a started drain loop" await relay.aclose() with pytest.raises(RuntimeError, match="closed"): relay.start() @@ -882,7 +882,7 @@ def held_send(**kw: Any) -> None: with pytest.raises(TimeoutError): # the teardown's bound, cutting the close await asyncio.wait_for(relay.aclose(), 0.05) assert relay._queue.qsize() == 1, "the cut close left no stop sentinel to test against" - with contextlib.suppress(RuntimeError): # refusing is one way to keep the property below + with suppress(RuntimeError): # refusing is one way to keep the property below relay.start() late = SecurityEvent(PASSWORD_CHANGED, username="b", email="b@x.test") answered = await relay.notify(late) From 9fd48a0718409a2033c6024389cc1ad2227e6e45 Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Sat, 17 Oct 2026 15:20:03 -0500 Subject: [PATCH 54/57] A self-revoke that loses the rotation race writes the miss row Review of the fourth main merge. Main's #3029 added a second 404 path to revoke_own_session: the revoke matched no unrevoked row, because a rotation re-keyed the session or a concurrent revoke ended it first. That path returned False with no audit row, while this branch's vault #3260 limb 2 (owner ruling 2026-10-16) has every miss of DELETE /me/sessions/{id} write one auth.session_revoke_missed row, as docs/SECURITY.md states. The race miss now writes that same row, and still no auth.session_revoked row. - service.py: the supersede_session miss joins the unknown and foreign id branch; docstring says so. - tests/test_session_supersession.py: the race test also pins the miss row. - changelog.d/3029.security.md: no longer says the 404 writes no row. - _ui_seam.py: a paragraph names the #3029 logout return type in the digest. BACKLOG #3029, vault BACKLOG #3260. --- changelog.d/3029.security.md | 3 ++- messagefoundry/api/_ui_seam.py | 5 +++++ messagefoundry/auth/service.py | 17 ++++++++++------- tests/test_session_supersession.py | 6 ++++++ 4 files changed, 23 insertions(+), 8 deletions(-) diff --git a/changelog.d/3029.security.md b/changelog.d/3029.security.md index 5c17c92e4c..27dc3f97e3 100644 --- a/changelog.d/3029.security.md +++ b/changelog.d/3029.security.md @@ -7,5 +7,6 @@ `auth.logout` rows carry `detail.outcome`, `revoked` or `nothing_to_revoke`, and a console sign-out's row names the owner of the session it ended instead of a NULL actor. Both sign-out routes still answer the same either way. A self-revoke that - ends nothing answers 404 and writes no row. A sign-out or self-revoke that loses that race still + ends nothing answers 404 and writes no `auth.session_revoked` row, only the + `auth.session_revoke_missed` row an unknown id writes (vault BACKLOG #3260). A sign-out or self-revoke that loses that race still leaves the session live under its new token; ending it there is not attempted. (`BACKLOG #3029`) diff --git a/messagefoundry/api/_ui_seam.py b/messagefoundry/api/_ui_seam.py index ec1b395606..02549e987a 100644 --- a/messagefoundry/api/_ui_seam.py +++ b/messagefoundry/api/_ui_seam.py @@ -416,6 +416,11 @@ #: none of which moved it. The value below covers all four, and vault BACKLOG #3260 above: it was #: re-derived again on the batch 257 branch's merge of those changes. #: +#: BACKLOG #3029: `AuthService.logout` returns a `LogoutOutcome` where it returned `None`. The +#: console's `/ui/logout` ignores the value, but the return type is part of the discovered +#: signature, so the digest moved on main with it. The value below covers it too: it was +#: re-derived on the batch 257 branch's fourth merge of main. +#: #: The digest below covers the surface DISCOVERED from the console's own imports and uses, which is #: strictly larger than the five hand-maintained tuples it replaced -- those had drifted, and the #: proof is that commit 40a4d5d9 added a REQUIRED ``UploadedFileList.scope`` field the console renders diff --git a/messagefoundry/auth/service.py b/messagefoundry/auth/service.py index 70e397f9e2..7045175715 100644 --- a/messagefoundry/auth/service.py +++ b/messagefoundry/auth/service.py @@ -8971,12 +8971,17 @@ async def revoke_own_session( the answer the caller is owed. ``client`` is required, as on :meth:`audit_self_target_refused`, because both planes call this from a request. - Also ``False``, with no audit row, when the revoke matched no unrevoked row (BACKLOG #3029): - at least a rotation that re-keyed the session after the read, which leaves it live under - its new id, or a concurrent revoke that ended it first. The revoke reports whether it - matched, so neither the return nor the trail claims a revoke that did not happen.""" + Also ``False`` when the revoke matched no unrevoked row (BACKLOG #3029): at least a + rotation that re-keyed the session after the read, which leaves it live under its new id, + or a concurrent revoke that ended it first. The revoke reports whether it matched, so + neither the return nor the trail claims a revoke that did not happen. That miss answers the + same 404, so it writes the same miss row and no ``auth.session_revoked`` row.""" session = await self._load_session(session_id) - if session is None or session.user_id != identity.user_id: + if ( + session is None + or session.user_id != identity.user_id + or await self._store.supersede_session(session_id, now=time.time()) is None + ): # The console passes the id as plain text, so only an id-shaped one is copied into the # trail; anything else is recorded as malformed, never as the caller's own characters. shown = session_id[:12] if _SESSION_ID_SHAPE.fullmatch(session_id) else "malformed" @@ -8993,8 +8998,6 @@ async def revoke_own_session( defects=(), ) return False - if await self._store.supersede_session(session_id, now=time.time()) is None: - return False await self._audit( "auth.session_revoked", actor=actor, diff --git a/tests/test_session_supersession.py b/tests/test_session_supersession.py index 02572139b5..6b8b01b43f 100644 --- a/tests/test_session_supersession.py +++ b/tests/test_session_supersession.py @@ -371,6 +371,12 @@ async def test_a_rotation_racing_a_self_revoke_is_not_reported_as_a_revoke( assert rotated is not None and await _live(service, rotated), "the race was not driven" assert revoked is False, "revoke_own_session reported a revoke that did not happen" assert await _superseded(store, "self") == [], "an audit row recorded a revoke that missed" + # Vault BACKLOG #3260, limb 2: the miss answers the same 404 as an unknown id, so it writes the + # same miss row, naming the id the caller sent. + missed = await store.list_audit(action="auth.session_revoke_missed") + assert [(row["actor"], json.loads(str(row["detail"]))) for row in missed] == [ + ("op", {"session": target_hash[:12]}) + ], "the race's 404 left no miss row" async def test_a_self_revoke_that_lands_returns_true_and_audits_it(store: MessageStore) -> None: From 9e65ba505c2ffdd4e8c5d20a360bc86506db7c1b Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Sat, 17 Oct 2026 22:01:29 -0500 Subject: [PATCH 55/57] Reconcile the SQL Server busy-pool test and the drain comment after the merge Review round one of the dchain merge into b257-dtail found the resolution left one semantic conflict: main's SQL Server busy-pool test for the post-commit borrow asserted that rotate_session ends on its UPDATE, and this branch's rotation now ends on the lineage INSERT in the same transaction. The test now asserts what its Postgres twin asserts: an UPDATE of sessions, then the lineage INSERT last for rotate_session. Without this, test_sqlserver_post_commit_session_writes_wait_out_a_busy_pool[rotate_session-True] fails on the merged tree and on neither parent. The DRAIN_TIMEOUT_SECONDS comment the merge wrote is reordered so the "no timeout" clause cannot read as narrowing which writes are drained, and no longer restates the graceful-stop window that _SHUTDOWN_BUDGET_SECONDS states (SDS-3.5). Backlog: vault #3028, #3023, #3193, #3191. --- messagefoundry/api/approvals.py | 13 ++++++------- tests/test_store_pool_acquire_timeout.py | 8 +++++++- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/messagefoundry/api/approvals.py b/messagefoundry/api/approvals.py index 260c5c584a..01d48e88d3 100644 --- a/messagefoundry/api/approvals.py +++ b/messagefoundry/api/approvals.py @@ -105,13 +105,12 @@ } -#: The default wait of :meth:`ApprovalGate.drain` for the gate's writes still running, at least the -#: outcome writes and the request writes (vault BACKLOG #3193), for a caller that passes no -#: timeout. The API lifespans do not use it: they pass a share of the drain step's bound, which can -#: be longer or shorter, inside NSSM's 15 s graceful-stop window (AppStopMethodConsole in -#: scripts/service/install-service.ps1). ``_SHUTDOWN_BUDGET_SECONDS`` in ``api/app.py`` states the -#: budget and the window arithmetic (BACKLOG #3191); ``_app_drain_steps`` and the managed -#: lifespan's teardown hold the steps and their order. +#: The default wait of :meth:`ApprovalGate.drain`, for a caller that passes no timeout. The drain +#: waits for the gate's writes still running, at least the outcome writes and the request writes +#: (vault BACKLOG #3193). The API lifespans do not use this default: they pass a share of the drain +#: step's bound, which can be longer or shorter. ``_SHUTDOWN_BUDGET_SECONDS`` in ``api/app.py`` +#: states the budget and the graceful-stop window arithmetic (BACKLOG #3191); ``_app_drain_steps`` +#: and the managed lifespan's teardown hold the steps and their order. #: Each write is a status change or a new row with its audit row, plus at most a read-back, so a #: few seconds is ample. DRAIN_TIMEOUT_SECONDS = 3.0 diff --git a/tests/test_store_pool_acquire_timeout.py b/tests/test_store_pool_acquire_timeout.py index f88c46c390..7c4c61d422 100644 --- a/tests/test_store_pool_acquire_timeout.py +++ b/tests/test_store_pool_acquire_timeout.py @@ -478,7 +478,13 @@ async def bump(cur: Any, user_id: str) -> None: assert not task.done(), f"{name} gave up on a busy pool after its caller's write committed" pool.gate.set() assert await asyncio.wait_for(task, timeout=5) == expected - assert pool.statements and pool.statements[-1].startswith("UPDATE sessions") + assert any(s.startswith("UPDATE sessions") for s in pool.statements), pool.statements + if name == "rotate_session": + # The rotation's lineage link (vault BACKLOG #3028) follows its re-key in the same + # transaction, on the same post-commit borrow, as in the Postgres twin above. + assert pool.statements[-1].startswith("INSERT INTO session_lineage"), pool.statements + else: + assert pool.statements[-1].startswith("UPDATE sessions"), pool.statements assert pool.released == [pool.conn] and pool.checked_out == [] assert store._acquire_wait.summary().count == 1, "the post-commit wait left the histogram" From 3faa362d8edf8bc4d4ad7356f6dad931cd3148d2 Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Sat, 17 Oct 2026 22:52:31 -0500 Subject: [PATCH 56/57] Review on the dtail merge: the referral-bind test no longer calls an undecodable bind an outage The union kept a tail docstring saying an undecodable response is an outage that pages nobody; since main's FAULT arm (vault #3696 item 2) it pages as a probe fault. Its referral tests now reuse main's _bad_bind_dc and _probe_through_the_reconciler rather than a second copy of the same setup. For BACKLOG #3010. --- tests/test_ldap_decode_boundary.py | 28 +++++++--------------------- 1 file changed, 7 insertions(+), 21 deletions(-) diff --git a/tests/test_ldap_decode_boundary.py b/tests/test_ldap_decode_boundary.py index 70e3f01865..e47eda8b44 100644 --- a/tests/test_ldap_decode_boundary.py +++ b/tests/test_ldap_decode_boundary.py @@ -242,12 +242,9 @@ def _assert_referred_bind(exc: BaseException, operation: str) -> None: def test_a_service_bind_answered_with_a_referral_is_a_referral() -> None: - """Before D3b it read as an undecodable response, an outage the reconciler pages nobody for. - THE PREMISE first: the bare ldap3 bind raises ``KeyError(3)``. The socket is still closed.""" - with _scripted_dc(lambda _raw: [], bind_reply=lambda _raw: (_REFERRAL_BIND, b"")) as ( - dc, - authenticator, - ): + """Before D3b it read as an undecodable response, not as a referral. THE PREMISE first: the + bare ldap3 bind raises ``KeyError(3)``. The socket is still closed.""" + with _bad_bind_dc(_REFERRAL_BIND) as (dc, authenticator): with pytest.raises(KeyError) as bare: _bare_bind(dc.port) assert bare.value.args == (3,) @@ -279,21 +276,10 @@ def bind(raw: bytes) -> _Reply | None: async def test_the_reconciler_reads_a_referred_service_bind_as_referred() -> None: - """So the probe reaches the referral's alert path (BACKLOG #2538), not the outage's.""" - with _scripted_dc(lambda _raw: [], bind_reply=lambda _raw: (_REFERRAL_BIND, b"")) as ( - dc, - authenticator, - ): - store = await MessageStore.open(":memory:") - try: - settings = _ad_settings( - ad_server=f"ldap://127.0.0.1:{dc.port}", ad_allow_insecure_ldap=True - ) - service = AuthService(store, settings, ldap=authenticator) - user = SimpleNamespace(id="u-1", username="alice", directory_object_id=None) - probe = await service._probe_principal(user) # type: ignore[arg-type] - finally: - await store.close() + """So the probe reaches the referral's alert path (BACKLOG #2538), not the outage's or the + probe fault's, and its detail names no referred host.""" + with _bad_bind_dc(_REFERRAL_BIND) as (dc, authenticator): + probe = await _probe_through_the_reconciler(authenticator, dc.port) assert probe.outcome is ProbeOutcome.REFERRED assert probe.detail is not None and _REFERRED not in probe.detail From fd7884fcfee2d1e20407de6fbd52bc7186cb6910 Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Mon, 19 Oct 2026 14:08:35 -0500 Subject: [PATCH 57/57] Review on the main merge: name the configuration refusal where the paging and clear texts missed it Round one of the code review on the merge commit. Text only, no behaviour change: - CONFIGURATION.md and CONNECTIONS.md: a pass with a configuration refusal resolves no reconcile alert and does not re-arm the paging latch, as the code already does (BACKLOG #3010, vault BACKLOG #3696). - SECURITY.md: the configuration refusal marks no alert clear, and a service-account bind refused with one of those codes takes the same path. - service.py: the unjudged-others count names the paging refusal beside an unreachable directory; _track_reconcile_episodes lists misconfigured among the probes that end nothing. - changelog.d/3010.security.md: drops the service-connection unbind bullet. Main's _service_session (BACKLOG #3017) delivers it now, not this branch. - test_ldap_decode_boundary.py: the end-to-end unbind test says it exercises _service_session. Proposed PR title (unchanged): the draft PR 2387 title. Ledger banner text: unchanged from the D3b, D4, D4b and D5 handover commits. --- changelog.d/3010.security.md | 5 +---- docs/CONFIGURATION.md | 5 +++-- docs/CONNECTIONS.md | 2 +- docs/SECURITY.md | 4 +++- messagefoundry/auth/service.py | 7 +++++-- tests/test_ldap_decode_boundary.py | 4 +++- 6 files changed, 16 insertions(+), 11 deletions(-) diff --git a/changelog.d/3010.security.md b/changelog.d/3010.security.md index 5fba8a134d..feab2bbe70 100644 --- a/changelog.d/3010.security.md +++ b/changelog.d/3010.security.md @@ -50,7 +50,4 @@ `directory_referral`, as on any referral. That alert's text names the two search bases and `[auth].ad_bind_dn`, any of which may name another domain of the forest. Before, a sign-in that met one failed with an unhandled error, and the reconciler read it as a - probe fault. - **The service-account connection is unbound** after every sign-in lookup and reconciler probe. - ldap3's own `with` block never unbound it, so each lookup left a directory session open until the - connection was collected. (vault `BACKLOG #3010`) + probe fault. (vault `BACKLOG #3010`) diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index 01b62d57ef..d51c4bae61 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -844,9 +844,10 @@ three on its own, until it restarts again. - An LDAP referral opens its own `ad_reconcile_aborted` instance, with reason `directory_referral` and source `directory-reconciler-referral` (BACKLOG #2538). It resolves once every account signed in at the last referral has since been read in full, present and enabled, on a pass with no - referral. A pass with any referral or probe fault resolves none of these alerts. Neither is + referral. A pass with any referral, probe fault or configuration refusal (BACKLOG #3010) + resolves none of these alerts. None is carried to later passes for the breaker's and the hold's alerts: a later pass whose probe sample - leaves out the referred or faulting account reads its older answer, and can resolve one of + leaves out the referred, faulting or refused account reads its older answer, and can resolve one of those two (BACKLOG #3100, left open). - So an account that never reads in full keeps the referral's alert open while it is signed in. A held account, a disabled one awaiting its strikes, and one that never answers all do this. diff --git a/docs/CONNECTIONS.md b/docs/CONNECTIONS.md index 581f216e89..415c4b68a7 100644 --- a/docs/CONNECTIONS.md +++ b/docs/CONNECTIONS.md @@ -4009,7 +4009,7 @@ reading this page already applies to a file the scan never opened. | Store — SQL Server (`[store].backend = sqlserver`) | three distinct bounds: `[store].connect_timeout` 15 s (**login**, passed as pyodbc `timeout=`), `[store].command_timeout` 30 s (**statement**, applied per acquire as the pyodbc connection attribute) and `[store].acquire_timeout` 30 s (the **pool borrow**); `[store].warm_pool_timeout` 15 s bounds the warm-up | every acquire releases back to the pool; the pool is closed on shutdown; a borrow the pool satisfies after the borrower gave up is released back rather than stranded | a driver error — or a `StoreAcquireTimeout` — propagates to the stage worker and the row stays claimable | stage handoffs re-run idempotently; `reset_stale_inflight` recovers on restart | | Store — Postgres (`[store].backend = postgres`) | `[store].connect_timeout` 15 s (`create_pool(timeout=…)`), `[store].command_timeout` 30 s as `asyncpg`'s per-statement bound, and `[store].acquire_timeout` 30 s on the message-pipeline pool borrows (see the coverage note above) | as SQL Server | as SQL Server | as SQL Server | | Active Directory — login binds (`[auth].ad_server`) | `[auth].ad_connect_timeout` 10 s on the LDAP TCP connect and `[auth].ad_receive_timeout` 10 s on every LDAP response read — threaded into **every** `ldap3` `Server`/`Connection` construction. The nested group search is paged, and **all its pages share one `[auth].ad_group_search_deadline`** 120 s: each socket read of a page after the first waits at most the shorter of `ad_receive_timeout` and what was left when the page was asked for, no page is asked for once it is spent, and the abandon of a refused search gets only what is left (vault BACKLOG #3696). **It is not a hard wall-clock bound**: the cap is set once per page and applies to each read, so a page that arrives over several reads, each inside that cap, can hold the search up to the number of reads times what was left. The page size is the directory's choice, since it may ignore the 500 the engine asks for | the service-account connection is context-managed; the user bind is unbound in a `finally`, so a **rejected** password releases it too (the common adversarial case) | fails closed with `LdapError`; the login is rejected and audited, and so is a group search past its deadline or past 10,000 groups, whatever page size the directory returns (one WARNING per process for each) | **single-shot** — two binds, no retry loop; the group search's pages are one search, not retries | -| Active Directory — session reconciler (`ad_session_recheck_seconds`) | the same `ad_connect_timeout` / `ad_receive_timeout` / `ad_group_search_deadline` | as the login path — context-managed connections | a pass that fails is retried on the next interval; strike state is process-local. A group search refused for its deadline, its 10,000-group cap or its page backstop reads that account as unavailable on every pass. A disable or deletion still revokes it, because the user search runs first, but its roles and channel scope are not re-diffed while that lasts. So it pages, as `ad_reconcile_aborted` under the source `directory-reconciler-paging:`, and audits `auth.ad_reconcile_paging_refused`, once as the account enters that state, never on every pass. It pages again only after a pass judges the account, which an outage, a referral or a probe fault does not, or after a restart (vault BACKLOG #3696). A plain outage still pages nobody | **single-shot per pass**; `ad_session_recheck_strikes` (2) required before a revoke | +| Active Directory — session reconciler (`ad_session_recheck_seconds`) | the same `ad_connect_timeout` / `ad_receive_timeout` / `ad_group_search_deadline` | as the login path — context-managed connections | a pass that fails is retried on the next interval; strike state is process-local. A group search refused for its deadline, its 10,000-group cap or its page backstop reads that account as unavailable on every pass. A disable or deletion still revokes it, because the user search runs first, but its roles and channel scope are not re-diffed while that lasts. So it pages, as `ad_reconcile_aborted` under the source `directory-reconciler-paging:`, and audits `auth.ad_reconcile_paging_refused`, once as the account enters that state, never on every pass. It pages again only after a pass judges the account, which an outage, a referral, a configuration refusal or a probe fault does not, or after a restart (vault BACKLOG #3696). A plain outage still pages nobody | **single-shot per pass**; `ad_session_recheck_strikes` (2) required before a revoke | | Kerberos / SPNEGO SSO (`kerberos_spn`) | **none engine-owned** — the OS provider owns any KDC timeout | the SPNEGO context is per-request | a failed step raises `LdapError` and audits a login reject | **single-shot**, single-leg — no NTLM fallback, no multi-leg handshake | | OIDC IdP — token endpoint (`oidc_token_endpoint`) | 10 s — `auth/oidc/flow.py`'s **own** `exchange_code(timeout=…)` default (`AuthService._oidc_exchange` passes none). The JWKS leg's 10 s is a **separate** literal, `oidc_http.DEFAULT_IDP_TIMEOUT_SECONDS`; the two coincide but are independent | the response is context-managed; the body is size-capped | a non-200 or oversize body fails the login closed | **single-shot** — one POST per login | | OIDC IdP — JWKS fetch (`oidc_jwks_uri`) | 10 s — `oidc_http.DEFAULT_IDP_TIMEOUT_SECONDS`, the constant `jwks_fetcher` is the only consumer of | the response is context-managed; the size cap is enforced on the socket read | a fetch failure fails verification closed | **single-shot**, further damped by the refetch floor | diff --git a/docs/SECURITY.md b/docs/SECURITY.md index eacbb14a48..fc9cd29f6b 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -4708,7 +4708,9 @@ revocation. A pass with any such refusal logs one ERROR naming the code and no a `directory_misconfigured`, under its own source `directory-reconciler-misconfigured`. If every probe is refused, faulted or fails, at least one refused and none referred, the pass aborts as `directory_misconfigured`: it judges no account it asked about and revokes nothing, but it pages. -Like the fault's, that alert has no inverse; an operator resolves it. **The cost:** while the +Like the fault's, that alert has no inverse; an operator resolves it, and a pass with such a +refusal marks no alert clear. A service-account bind the directory refuses with one of those codes +takes the same path, as a malformed `ad_bind_dn` does. **The cost:** while the refusal lasts, the refused accounts' sessions are never revoked by the reconciler, so they run to the absolute cap. Other result codes that are not an answer, such as busy or unavailable, are an outage. A second-factor step-up and a passkey assertion, which probe the directory before they diff --git a/messagefoundry/auth/service.py b/messagefoundry/auth/service.py index a45c4d1d9f..6c0f86b47d 100644 --- a/messagefoundry/auth/service.py +++ b/messagefoundry/auth/service.py @@ -7680,7 +7680,7 @@ def _track_reconcile_episodes( counts, a held one too, so an account that never reads PRESENT does not keep a trip or a referral open for good. For a hold the answer must not be undetermined: a lone account that is held, then strikes, then is held again is one hold, not one per flap. A probe - that reached no answer (unavailable, referred, faulted), a probe sample that misses the + that reached no answer (unavailable, referred, faulted, misconfigured), a probe sample that misses the accounts, and a pass that judged nothing end nothing, so one condition seen across a rotation or an outage stays one episode. @@ -8538,8 +8538,11 @@ def _unjudged_others( if refusals and plan.misconfigured: others = f"{len(plan.misconfigured)} were refused for the configuration. {others}" if plan.unavailable: + # UNAVAILABLE also carries an account whose group search the engine refused for its own + # paging bound (vault BACKLOG #3696), which pages on its own, so this names both. others = ( - f"{plan.unavailable} could not be read (the directory was unreachable). {others}" + f"{plan.unavailable} could not be read (the directory was unreachable, or the " + f"engine refused a group search for its own bound). {others}" ) return others diff --git a/tests/test_ldap_decode_boundary.py b/tests/test_ldap_decode_boundary.py index d816a8fed3..bf7756a224 100644 --- a/tests/test_ldap_decode_boundary.py +++ b/tests/test_ldap_decode_boundary.py @@ -709,7 +709,9 @@ def test_an_unbind_error_never_replaces_the_refusal() -> None: def test_a_probe_answered_before_the_unbind_fails_keeps_its_answer() -> None: - """The same, end to end: the unbind's send fails, and the probe still answers.""" + """The service connection's release, end to end (``_service_session``, BACKLOG #3017): unlike + ``_released`` above after a clean block, an unbind whose send fails is suppressed, and the probe + still answers.""" with _scripted_dc(_groups_answered) as (_dc, authenticator): real = ldap3.Connection.unbind