feat: turn receiving on, and read it in the dashboard (inbound Phase D)

README.md

@@ -26,6 +26,7 @@ and dashboard.
 - **Verify senders** — email addresses and domains, with the DNS records to add
 - **Delivery events** — see whether each message was delivered, bounced, opened or clicked
 - **Webhooks** — get notified when something happens to a message
+- **Receive email** — mail sent to a domain you choose is stored, shown in an Inbox, readable through the API, and announced by webhook
 - **Suppression list** — bounced and unsubscribed addresses are never sent to again
 - **Dashboard** — send a test message, watch deliveries, manage everything
 - **Request SES production access** from the dashboard

apps/api/src/seskit_api/downloads.py

@@ -0,0 +1,134 @@
+"""Serving a stranger's bytes: shared by ``/v1`` and the dashboard.
+
+Both surfaces let a signed-in person fetch an attachment or the original of a
+message somebody sent to them, and both are serving the same hostile thing from the
+same origin. Written once so the two cannot drift - a header added to one and
+forgotten on the other is exactly the failure this exists to prevent, and it would
+look like nothing at all until somebody pointed a browser at the wrong route.
+
+Whatever the sender declared, the response is a download and not a document. A
+browser pointed at one must not render it, sniff it, run script in it, or keep it.
+
+What is here is the part that is the same: the headers, the filename, the order of
+refusal, and building the response. What is *not* here is how a caller is
+authenticated or how a failure is shown - an API key and a JSON envelope on one
+side, a session and a page on the other - because those are what differ.
+"""
+
+from __future__ import annotations
+
+import re
+from urllib.parse import quote
+
+from fastapi import Response
+from seskit_core.config import Settings
+from seskit_core.email.parse import read_attachment
+from seskit_core.errors import APIError, ErrorType
+from seskit_core.models import InboundEmail
+from seskit_core.models.base import utcnow
+from seskit_core.services import InboundStoreFactory, get_connection, stored_credentials
+from seskit_provider_aws_ses import EXPIRED_MESSAGE
+from sqlalchemy.ext.asyncio import AsyncSession
+
+#: Applied to every download.
+#:
+#: ``nosniff`` stops a browser deciding the bytes are HTML because they look like
+#: it. The CSP ``sandbox`` is what is left if one renders it anyway: no script, no
+#: forms, no same-origin access to anything. ``no-store`` because this is somebody's
+#: private mail and should not sit in a shared cache.
+DOWNLOAD_HEADERS = {
+    "X-Content-Type-Options": "nosniff",
+    "Content-Security-Policy": "sandbox; default-src 'none'",
+    "Cache-Control": "private, no-store",
+}
+
+NO_SUCH_ATTACHMENT = "That message has no attachment with that index."
+
+_NOT_SAFE = re.compile(r"[^A-Za-z0-9._ -]")
+
+
+def content_disposition(filename: str) -> str:
+    """A ``Content-Disposition`` that is always a download, for any filename.
+
+    Two forms, because old clients read the plain one and current ones read the
+    encoded one. The plain form is reduced to characters that cannot end the
+    quoted string or start a new header; the encoded form carries the name with
+    everything outside the unreserved set percent-encoded.
+
+    Both are built from the same basename, so neither can carry a path or a name
+    that is only dots, and an empty name becomes ``attachment`` in both. A name
+    that came through the parser already has all of that - this does not rely on
+    it, because it is the last thing between a sender's string and a header, and
+    a client saving the file should not have to sanitise what it was given.
+    """
+    base = filename.replace("\\", "/").rsplit("/", 1)[-1].strip(" .") or "attachment"
+    fallback = _NOT_SAFE.sub("_", base)
+    return f"attachment; filename=\"{fallback}\"; filename*=UTF-8''{quote(base, safe='')}"
+
+
+def require_attachment(row: InboundEmail, index: int) -> None:
+    """Refuse an attachment that does not exist, before anything is fetched.
+
+    A request for an attachment the message does not have should not cost a
+    download of the message to find that out.
+    """
+    if index < 0 or index >= len(row.attachments):
+        raise APIError(ErrorType.NOT_FOUND, NO_SUCH_ATTACHMENT)
+
+
+async def fetch_original(
+    db: AsyncSession,
+    row: InboundEmail,
+    factory: InboundStoreFactory,
+    settings: Settings,
+) -> bytes:
+    """The stored message, or the right error for why not.
+
+    The retention date is checked before anything is fetched. After it the original
+    is gone or about to be, and an S3 call to confirm costs a round trip that can
+    only produce the same answer - so a request past it is refused at once, and one
+    with no known expiry is left to the fetch to settle.
+    """
+    if row.raw_expires_at is not None and row.raw_expires_at <= utcnow():
+        raise APIError(ErrorType.NOT_FOUND, EXPIRED_MESSAGE)
+
+    connection = await get_connection(db, row.project_id)
+    if connection is None or not connection.has_credentials:
+        raise APIError(
+            ErrorType.INVALID_REQUEST,
+            "This project is not connected to AWS, so the original message cannot be "
+            "fetched. Connect it on the AWS page.",
+        )
+
+    store = factory(
+        connection.region, stored_credentials(connection, secret_key=settings.SECRET_KEY)
+    )
+    return await store.fetch_message(bucket=row.storage_bucket, key=row.storage_key)
+
+
+def attachment_response(raw: bytes, index: int) -> Response:
+    """One attachment out of a stored message, as a download.
+
+    Always ``application/octet-stream`` however it was declared. The filename comes
+    from the stored message, not from anything the caller passed.
+    """
+    content = read_attachment(raw, index)
+    if content is None:
+        raise APIError(ErrorType.NOT_FOUND, NO_SUCH_ATTACHMENT)
+    return Response(
+        content=content.data,
+        media_type="application/octet-stream",
+        headers={**DOWNLOAD_HEADERS, "Content-Disposition": content_disposition(content.filename)},
+    )
+
+
+def raw_response(raw: bytes, inbound_id: str) -> Response:
+    """The original message as an ``.eml`` download."""
+    return Response(
+        content=raw,
+        media_type="message/rfc822",
+        headers={
+            **DOWNLOAD_HEADERS,
+            "Content-Disposition": content_disposition(f"{inbound_id}.eml"),
+        },
+    )

apps/api/src/seskit_api/main.py

@@ -34,6 +34,7 @@
     domains,
     emails,
     health,
+    inbox,
     suppressions,
     unsubscribe,
     webhooks,
@@ -258,6 +259,7 @@ async def _unhandled(request: Request, exc: Exception) -> JSONResponse | Respons
     app.include_router(aws.router)
     app.include_router(domains.router)
     app.include_router(emails.router)
+    app.include_router(inbox.router)
     app.include_router(webhooks.router)
     app.include_router(suppressions.router)
     # Public and unauthenticated, unlike everything above it.

apps/api/src/seskit_api/middleware/security.py

@@ -99,11 +99,16 @@ async def dispatch(
         if "Content-Security-Policy" not in response.headers:
             response.headers["Content-Security-Policy"] = policy_for(nonce)
         response.headers["X-Content-Type-Options"] = "nosniff"
-        response.headers["X-Frame-Options"] = "DENY"
+        # Framing and referrer policy follow the same rule as the CSP above: set only if
+        # the route did not. The preview of a received message's HTML is its own
+        # document, framed by the page that shows it, so it has to be able to say
+        # SAMEORIGIN and not be forced back to DENY - and it sends no Referer at all,
+        # which is stricter than the default and must not be loosened to it.
+        response.headers.setdefault("X-Frame-Options", "DENY")
         # same-origin rather than no-referrer: an unsubscribe token or an email
         # id in a Referer sent to another site is a leak, but SESKit's own
         # navigation is worth keeping.
-        response.headers["Referrer-Policy"] = "same-origin"
+        response.headers.setdefault("Referrer-Policy", "same-origin")
 
         if not self._settings.is_local:
             response.headers["Strict-Transport-Security"] = HSTS

apps/api/src/seskit_api/receiving.py

@@ -0,0 +1,136 @@
+"""What the Domains page shows about receiving mail for one domain.
+
+A view model and nothing more: it decides whether receiving can be turned on for a
+domain *right now*, and if not, says why in words a person can act on. The page
+renders it, and the route that turns receiving on asks it again before doing
+anything - the disabled button is a courtesy, and this is the check, because a
+form can be posted by hand.
+
+**Every reason is stated rather than hidden.** A button that is simply absent leaves
+somebody wondering whether the feature exists. Each of these is a precondition they
+can fix, and the page says which.
+
+The MX record is built here and not in a template so that a test can assert on it.
+It is the one thing a user has to do by hand, outside SESKit, and a record with the
+wrong host would send their mail nowhere without an error anywhere.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from seskit_core.config import Settings
+from seskit_core.models import AWSConnection, Identity
+from seskit_provider_aws_ses import supports_receiving
+
+#: SES takes mail at the lowest-preference host. One host, so one value; the number
+#: only matters when a domain lists several, and nobody should be listing others
+#: beside this one.
+MX_PRIORITY = 10
+
+#: Where Amazon SES takes mail in a region. Published by AWS in the General
+#: Reference's "Email Receiving endpoints" table, one per receiving region.
+MX_HOST = "inbound-smtp.{region}.amazonaws.com"
+
+
+@dataclass(frozen=True, slots=True)
+class MxRecord:
+    """The record to add at the user's DNS provider."""
+
+    name: str
+    priority: int
+    value: str
+
+    @property
+    def zone_line(self) -> str:
+        """As one line, for a provider that takes a zone file."""
+        return f"{self.name}  MX  {self.priority} {self.value}"
+
+
+@dataclass(frozen=True, slots=True)
+class ReceivingView:
+    """Receiving for one domain, as the page needs it."""
+
+    enabled: bool
+    #: Why it cannot be turned on right now, or ``None`` if it can. Always ``None``
+    #: when it is already on - a running thing is not blocked, and the controls
+    #: that matter then are the ones to stop it.
+    blocked: str | None
+    mx: MxRecord
+    retention_days: int
+
+
+def mx_record(identity: Identity) -> MxRecord:
+    return MxRecord(
+        name=identity.value,
+        priority=MX_PRIORITY,
+        value=MX_HOST.format(region=identity.region),
+    )
+
+
+def blocked_reason(
+    identity: Identity,
+    connection: AWSConnection | None,
+    settings: Settings,
+    *,
+    taken_elsewhere: bool,
+) -> str | None:
+    """The first thing in the way of turning receiving on, in the order a person
+    would fix them, or ``None`` if nothing is.
+    """
+    if not identity.is_verified:
+        return "Verify this domain first. Receiving needs a domain Amazon SES knows is yours."
+
+    if connection is None or not connection.is_connected or not connection.has_credentials:
+        return "Connect AWS first, on the AWS page."
+
+    if identity.region != connection.region:
+        return (
+            f"This domain is verified in {identity.region}, but this project is connected to "
+            f"{connection.region}. Mail is received in the connection's region."
+        )
+
+    if not supports_receiving(identity.region):
+        return (
+            f"Amazon SES cannot receive mail in {identity.region}. It can in regions such as "
+            "us-east-1, eu-west-1 and ap-southeast-2."
+        )
+
+    if not settings.polls_sqs:
+        return (
+            "Receiving needs this instance to read from SQS, and EVENT_INGESTION is set to "
+            "https. Set it to sqs or both."
+        )
+
+    if taken_elsewhere:
+        return (
+            "This domain already receives mail for another project or region. A domain's MX "
+            "record can only point at one place."
+        )
+
+    return None
+
+
+def receiving_view(
+    identity: Identity,
+    connection: AWSConnection | None,
+    settings: Settings,
+    *,
+    taken_elsewhere: bool = False,
+) -> ReceivingView | None:
+    """``None`` for something that cannot receive at all - an email address has no
+    MX record to point anywhere - so the page shows nothing rather than a control
+    that could never work.
+    """
+    if not identity.is_domain:
+        return None
+
+    enabled = identity.receives_mail
+    return ReceivingView(
+        enabled=enabled,
+        blocked=None
+        if enabled
+        else blocked_reason(identity, connection, settings, taken_elsewhere=taken_elsewhere),
+        mx=mx_record(identity),
+        retention_days=settings.INBOUND_RETENTION_DAYS,
+    )