diff --git a/README.md b/README.md index 215cec1..139add4 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,7 @@ and dashboard. - **Verify senders** — email addresses and domains, with the DNS records to add - **Delivery events** — see whether each message was delivered, bounced, opened or clicked - **Webhooks** — get notified when something happens to a message +- **Receive email** — mail sent to a domain you choose is stored, shown in an Inbox, readable through the API, and announced by webhook - **Suppression list** — bounced and unsubscribed addresses are never sent to again - **Dashboard** — send a test message, watch deliveries, manage everything - **Request SES production access** from the dashboard diff --git a/apps/api/src/seskit_api/downloads.py b/apps/api/src/seskit_api/downloads.py new file mode 100644 index 0000000..7fedcad --- /dev/null +++ b/apps/api/src/seskit_api/downloads.py @@ -0,0 +1,134 @@ +"""Serving a stranger's bytes: shared by ``/v1`` and the dashboard. + +Both surfaces let a signed-in person fetch an attachment or the original of a +message somebody sent to them, and both are serving the same hostile thing from the +same origin. Written once so the two cannot drift - a header added to one and +forgotten on the other is exactly the failure this exists to prevent, and it would +look like nothing at all until somebody pointed a browser at the wrong route. + +Whatever the sender declared, the response is a download and not a document. A +browser pointed at one must not render it, sniff it, run script in it, or keep it. + +What is here is the part that is the same: the headers, the filename, the order of +refusal, and building the response. What is *not* here is how a caller is +authenticated or how a failure is shown - an API key and a JSON envelope on one +side, a session and a page on the other - because those are what differ. +""" + +from __future__ import annotations + +import re +from urllib.parse import quote + +from fastapi import Response +from seskit_core.config import Settings +from seskit_core.email.parse import read_attachment +from seskit_core.errors import APIError, ErrorType +from seskit_core.models import InboundEmail +from seskit_core.models.base import utcnow +from seskit_core.services import InboundStoreFactory, get_connection, stored_credentials +from seskit_provider_aws_ses import EXPIRED_MESSAGE +from sqlalchemy.ext.asyncio import AsyncSession + +#: Applied to every download. +#: +#: ``nosniff`` stops a browser deciding the bytes are HTML because they look like +#: it. The CSP ``sandbox`` is what is left if one renders it anyway: no script, no +#: forms, no same-origin access to anything. ``no-store`` because this is somebody's +#: private mail and should not sit in a shared cache. +DOWNLOAD_HEADERS = { + "X-Content-Type-Options": "nosniff", + "Content-Security-Policy": "sandbox; default-src 'none'", + "Cache-Control": "private, no-store", +} + +NO_SUCH_ATTACHMENT = "That message has no attachment with that index." + +_NOT_SAFE = re.compile(r"[^A-Za-z0-9._ -]") + + +def content_disposition(filename: str) -> str: + """A ``Content-Disposition`` that is always a download, for any filename. + + Two forms, because old clients read the plain one and current ones read the + encoded one. The plain form is reduced to characters that cannot end the + quoted string or start a new header; the encoded form carries the name with + everything outside the unreserved set percent-encoded. + + Both are built from the same basename, so neither can carry a path or a name + that is only dots, and an empty name becomes ``attachment`` in both. A name + that came through the parser already has all of that - this does not rely on + it, because it is the last thing between a sender's string and a header, and + a client saving the file should not have to sanitise what it was given. + """ + base = filename.replace("\\", "/").rsplit("/", 1)[-1].strip(" .") or "attachment" + fallback = _NOT_SAFE.sub("_", base) + return f"attachment; filename=\"{fallback}\"; filename*=UTF-8''{quote(base, safe='')}" + + +def require_attachment(row: InboundEmail, index: int) -> None: + """Refuse an attachment that does not exist, before anything is fetched. + + A request for an attachment the message does not have should not cost a + download of the message to find that out. + """ + if index < 0 or index >= len(row.attachments): + raise APIError(ErrorType.NOT_FOUND, NO_SUCH_ATTACHMENT) + + +async def fetch_original( + db: AsyncSession, + row: InboundEmail, + factory: InboundStoreFactory, + settings: Settings, +) -> bytes: + """The stored message, or the right error for why not. + + The retention date is checked before anything is fetched. After it the original + is gone or about to be, and an S3 call to confirm costs a round trip that can + only produce the same answer - so a request past it is refused at once, and one + with no known expiry is left to the fetch to settle. + """ + if row.raw_expires_at is not None and row.raw_expires_at <= utcnow(): + raise APIError(ErrorType.NOT_FOUND, EXPIRED_MESSAGE) + + connection = await get_connection(db, row.project_id) + if connection is None or not connection.has_credentials: + raise APIError( + ErrorType.INVALID_REQUEST, + "This project is not connected to AWS, so the original message cannot be " + "fetched. Connect it on the AWS page.", + ) + + store = factory( + connection.region, stored_credentials(connection, secret_key=settings.SECRET_KEY) + ) + return await store.fetch_message(bucket=row.storage_bucket, key=row.storage_key) + + +def attachment_response(raw: bytes, index: int) -> Response: + """One attachment out of a stored message, as a download. + + Always ``application/octet-stream`` however it was declared. The filename comes + from the stored message, not from anything the caller passed. + """ + content = read_attachment(raw, index) + if content is None: + raise APIError(ErrorType.NOT_FOUND, NO_SUCH_ATTACHMENT) + return Response( + content=content.data, + media_type="application/octet-stream", + headers={**DOWNLOAD_HEADERS, "Content-Disposition": content_disposition(content.filename)}, + ) + + +def raw_response(raw: bytes, inbound_id: str) -> Response: + """The original message as an ``.eml`` download.""" + return Response( + content=raw, + media_type="message/rfc822", + headers={ + **DOWNLOAD_HEADERS, + "Content-Disposition": content_disposition(f"{inbound_id}.eml"), + }, + ) diff --git a/apps/api/src/seskit_api/main.py b/apps/api/src/seskit_api/main.py index 488ffc9..c881154 100644 --- a/apps/api/src/seskit_api/main.py +++ b/apps/api/src/seskit_api/main.py @@ -34,6 +34,7 @@ domains, emails, health, + inbox, suppressions, unsubscribe, webhooks, @@ -258,6 +259,7 @@ async def _unhandled(request: Request, exc: Exception) -> JSONResponse | Respons app.include_router(aws.router) app.include_router(domains.router) app.include_router(emails.router) + app.include_router(inbox.router) app.include_router(webhooks.router) app.include_router(suppressions.router) # Public and unauthenticated, unlike everything above it. diff --git a/apps/api/src/seskit_api/middleware/security.py b/apps/api/src/seskit_api/middleware/security.py index 0ca630b..3e766e4 100644 --- a/apps/api/src/seskit_api/middleware/security.py +++ b/apps/api/src/seskit_api/middleware/security.py @@ -99,11 +99,16 @@ async def dispatch( if "Content-Security-Policy" not in response.headers: response.headers["Content-Security-Policy"] = policy_for(nonce) response.headers["X-Content-Type-Options"] = "nosniff" - response.headers["X-Frame-Options"] = "DENY" + # Framing and referrer policy follow the same rule as the CSP above: set only if + # the route did not. The preview of a received message's HTML is its own + # document, framed by the page that shows it, so it has to be able to say + # SAMEORIGIN and not be forced back to DENY - and it sends no Referer at all, + # which is stricter than the default and must not be loosened to it. + response.headers.setdefault("X-Frame-Options", "DENY") # same-origin rather than no-referrer: an unsubscribe token or an email # id in a Referer sent to another site is a leak, but SESKit's own # navigation is worth keeping. - response.headers["Referrer-Policy"] = "same-origin" + response.headers.setdefault("Referrer-Policy", "same-origin") if not self._settings.is_local: response.headers["Strict-Transport-Security"] = HSTS diff --git a/apps/api/src/seskit_api/receiving.py b/apps/api/src/seskit_api/receiving.py new file mode 100644 index 0000000..a90931c --- /dev/null +++ b/apps/api/src/seskit_api/receiving.py @@ -0,0 +1,136 @@ +"""What the Domains page shows about receiving mail for one domain. + +A view model and nothing more: it decides whether receiving can be turned on for a +domain *right now*, and if not, says why in words a person can act on. The page +renders it, and the route that turns receiving on asks it again before doing +anything - the disabled button is a courtesy, and this is the check, because a +form can be posted by hand. + +**Every reason is stated rather than hidden.** A button that is simply absent leaves +somebody wondering whether the feature exists. Each of these is a precondition they +can fix, and the page says which. + +The MX record is built here and not in a template so that a test can assert on it. +It is the one thing a user has to do by hand, outside SESKit, and a record with the +wrong host would send their mail nowhere without an error anywhere. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +from seskit_core.config import Settings +from seskit_core.models import AWSConnection, Identity +from seskit_provider_aws_ses import supports_receiving + +#: SES takes mail at the lowest-preference host. One host, so one value; the number +#: only matters when a domain lists several, and nobody should be listing others +#: beside this one. +MX_PRIORITY = 10 + +#: Where Amazon SES takes mail in a region. Published by AWS in the General +#: Reference's "Email Receiving endpoints" table, one per receiving region. +MX_HOST = "inbound-smtp.{region}.amazonaws.com" + + +@dataclass(frozen=True, slots=True) +class MxRecord: + """The record to add at the user's DNS provider.""" + + name: str + priority: int + value: str + + @property + def zone_line(self) -> str: + """As one line, for a provider that takes a zone file.""" + return f"{self.name} MX {self.priority} {self.value}" + + +@dataclass(frozen=True, slots=True) +class ReceivingView: + """Receiving for one domain, as the page needs it.""" + + enabled: bool + #: Why it cannot be turned on right now, or ``None`` if it can. Always ``None`` + #: when it is already on - a running thing is not blocked, and the controls + #: that matter then are the ones to stop it. + blocked: str | None + mx: MxRecord + retention_days: int + + +def mx_record(identity: Identity) -> MxRecord: + return MxRecord( + name=identity.value, + priority=MX_PRIORITY, + value=MX_HOST.format(region=identity.region), + ) + + +def blocked_reason( + identity: Identity, + connection: AWSConnection | None, + settings: Settings, + *, + taken_elsewhere: bool, +) -> str | None: + """The first thing in the way of turning receiving on, in the order a person + would fix them, or ``None`` if nothing is. + """ + if not identity.is_verified: + return "Verify this domain first. Receiving needs a domain Amazon SES knows is yours." + + if connection is None or not connection.is_connected or not connection.has_credentials: + return "Connect AWS first, on the AWS page." + + if identity.region != connection.region: + return ( + f"This domain is verified in {identity.region}, but this project is connected to " + f"{connection.region}. Mail is received in the connection's region." + ) + + if not supports_receiving(identity.region): + return ( + f"Amazon SES cannot receive mail in {identity.region}. It can in regions such as " + "us-east-1, eu-west-1 and ap-southeast-2." + ) + + if not settings.polls_sqs: + return ( + "Receiving needs this instance to read from SQS, and EVENT_INGESTION is set to " + "https. Set it to sqs or both." + ) + + if taken_elsewhere: + return ( + "This domain already receives mail for another project or region. A domain's MX " + "record can only point at one place." + ) + + return None + + +def receiving_view( + identity: Identity, + connection: AWSConnection | None, + settings: Settings, + *, + taken_elsewhere: bool = False, +) -> ReceivingView | None: + """``None`` for something that cannot receive at all - an email address has no + MX record to point anywhere - so the page shows nothing rather than a control + that could never work. + """ + if not identity.is_domain: + return None + + enabled = identity.receives_mail + return ReceivingView( + enabled=enabled, + blocked=None + if enabled + else blocked_reason(identity, connection, settings, taken_elsewhere=taken_elsewhere), + mx=mx_record(identity), + retention_days=settings.INBOUND_RETENTION_DAYS, + ) diff --git a/apps/api/src/seskit_api/routes/domains.py b/apps/api/src/seskit_api/routes/domains.py index 931ffbe..ddb085b 100644 --- a/apps/api/src/seskit_api/routes/domains.py +++ b/apps/api/src/seskit_api/routes/domains.py @@ -21,7 +21,7 @@ from seskit_core.db import get_session from seskit_core.errors import APIError, ErrorType from seskit_core.logging import get_logger -from seskit_core.models import Project +from seskit_core.models import AWSConnection, Identity, Project from seskit_core.redis import get_redis from seskit_core.services import ( InboundProvisionerFactory, @@ -33,7 +33,10 @@ list_projects, refresh_identity, remove_identity, + setup_receiving, + teardown_receiving, ) +from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession from seskit_api.dependencies import ( @@ -45,6 +48,7 @@ require_user, verify_csrf, ) +from seskit_api.receiving import ReceivingView, receiving_view from seskit_api.templating import render logger = get_logger(__name__) @@ -68,6 +72,7 @@ async def _page( status_code: int = 200, ) -> HTMLResponse: connection = await get_connection(db, project.id) + identities = await list_identities(db, project.id) return render( request, "pages/domains.html", @@ -78,11 +83,57 @@ async def _page( project=project, projects=await list_projects(db, current.user.id), connection=connection, - identities=await list_identities(db, project.id), + identities=identities, + receiving=await _receiving_views(request, db, project, connection, identities), error=error, ) +async def _receiving_views( + request: Request, + db: AsyncSession, + project: Project, + connection: AWSConnection | None, + identities: list[Identity], +) -> dict[str, ReceivingView]: + """Receiving for each domain, keyed by identity id. + + Computed here, once, so every handler that re-renders the page - add, refresh, + remove, and the two below - shows the same answer without each having to ask. + """ + settings: Settings = request.app.state.settings + taken = await _receiving_elsewhere(db, project.id, [i.value for i in identities if i.is_domain]) + views: dict[str, ReceivingView] = {} + for identity in identities: + view = receiving_view( + identity, + connection, + settings, + taken_elsewhere=identity.value in taken, + ) + if view is not None: + views[identity.id] = view + return views + + +async def _receiving_elsewhere(db: AsyncSession, project_id: str, values: list[str]) -> set[str]: + """Which of these domains already receive mail through another project. + + The database refuses a second receiver for a domain outright; this is so the + page can say so before somebody presses a button that cannot work. + """ + if not values: + return set() + rows = await db.scalars( + select(Identity.value).where( + Identity.value.in_(values), + Identity.inbound_rule_name.is_not(None), + Identity.project_id != project_id, + ) + ) + return set(rows) + + @router.get("/domains", response_class=HTMLResponse, summary="Sending identities") async def domains_page( request: Request, @@ -241,3 +292,124 @@ def _status_for(error: APIError) -> int: if error.error_type in {ErrorType.AUTHORIZATION_FAILED, ErrorType.AUTHENTICATION_FAILED}: return 400 return error.status_code + + +@router.post( + "/domains/{identity_id}/receiving", + response_class=HTMLResponse, + dependencies=[Depends(verify_csrf)], + summary="Start receiving mail for a domain", +) +async def start_receiving( + request: Request, + identity_id: str, + db: Annotated[AsyncSession, Depends(get_session)], + current: Annotated[CurrentUser, Depends(require_user)], + project: Annotated[Project, Depends(require_project)], + inbound: Annotated[InboundProvisionerFactory, Depends(get_inbound_provisioner_factory)], + settings: Annotated[Settings, Depends(get_app_settings)], +) -> HTMLResponse: + """Create the receiving plumbing in the user's AWS account and add this + domain's rule. + + Refuses on the server for every reason the page shows a disabled button for: + the button is a courtesy, and a form can be posted by hand. + + No rollback on an error, as on the AWS page. A rollback expires every loaded + object and the template then lazy-loads from inside sync Jinja, so the user + sees a 500 instead of the message. ``setup_receiving`` gives the domain back + itself when AWS refuses, which is what makes that safe. + """ + identity = await get_owned_identity(db, identity_id=identity_id, project_id=project.id) + if identity is None: + return await _page(request, db, current, project) + + connection = await get_connection(db, project.id) + taken = await _receiving_elsewhere(db, project.id, [identity.value]) + view = receiving_view(identity, connection, settings, taken_elsewhere=identity.value in taken) + if view is None: + return await _page( + request, + db, + current, + project, + error="Only a domain can receive mail.", + status_code=400, + ) + if view.blocked and connection is not None: + return await _page(request, db, current, project, error=view.blocked, status_code=400) + if connection is None: + return await _page( + request, db, current, project, error=NO_CONNECTION_MESSAGE, status_code=400 + ) + + try: + await setup_receiving( + db, + inbound, + connection, + identity, + resource_prefix=settings.EVENT_RESOURCE_PREFIX, + retention_days=settings.INBOUND_RETENTION_DAYS, + secret_key=settings.SECRET_KEY, + ) + except APIError as error: + return await _page( + request, db, current, project, error=error.message, status_code=_status_for(error) + ) + + await db.commit() + return await _page( + request, + db, + current, + project, + flash=( + f"Receiving is on for {identity.value}. SESKit created a bucket, a topic, a queue " + "and one receipt rule. Add the MX record below to start receiving." + ), + ) + + +@router.post( + "/domains/{identity_id}/receiving/stop", + response_class=HTMLResponse, + dependencies=[Depends(verify_csrf)], + summary="Stop receiving mail for a domain", +) +async def stop_receiving( + request: Request, + identity_id: str, + db: Annotated[AsyncSession, Depends(get_session)], + current: Annotated[CurrentUser, Depends(require_user)], + project: Annotated[Project, Depends(require_project)], + inbound: Annotated[InboundProvisionerFactory, Depends(get_inbound_provisioner_factory)], + settings: Annotated[Settings, Depends(get_app_settings)], +) -> HTMLResponse: + """Remove this domain's receipt rule, and the plumbing if nothing else uses it. + + Mail already received is kept. Stopping receiving is not deleting what arrived. + """ + identity = await get_owned_identity(db, identity_id=identity_id, project_id=project.id) + connection = await get_connection(db, project.id) + if identity is None or connection is None: + return await _page(request, db, current, project) + + try: + await teardown_receiving(db, inbound, connection, identity, secret_key=settings.SECRET_KEY) + except APIError as error: + return await _page( + request, db, current, project, error=error.message, status_code=_status_for(error) + ) + + await db.commit() + return await _page( + request, + db, + current, + project, + flash=( + f"Stopped receiving for {identity.value}. Mail already received is kept. " + "Remember to remove the MX record." + ), + ) diff --git a/apps/api/src/seskit_api/routes/inbox.py b/apps/api/src/seskit_api/routes/inbox.py new file mode 100644 index 0000000..2769936 --- /dev/null +++ b/apps/api/src/seskit_api/routes/inbox.py @@ -0,0 +1,304 @@ +"""The Inbox: mail the project received, in the dashboard. + +Session-authenticated and out of the OpenAPI schema, like the rest of the +dashboard. The read side only - a message is recorded by the worker as it arrives, +and what can be done here is look at it, and fetch an attachment or the original. + +**Everything on these pages was written by somebody else.** Jinja escapes it, and +there is deliberately no ``|safe`` anywhere near a value from a message. The HTML +body is shown as source here; rendering it is a separate, sandboxed document so +that the page a signed-in person is looking at never contains a stranger's markup. + +Ownership is part of every query. An id from another project resolves to the same +answer as one that does not exist, so a stranger cannot probe for real ids. +""" + +from __future__ import annotations + +from typing import Annotated + +from fastapi import APIRouter, Depends, Request, Response +from fastapi.responses import HTMLResponse +from seskit_core.config import Settings +from seskit_core.db import get_session +from seskit_core.errors import APIError, ErrorType +from seskit_core.models import InboundEmail, Project +from seskit_core.services import ( + InboundStoreFactory, + get_received, + list_identities, + list_projects, + list_received, +) +from sqlalchemy.ext.asyncio import AsyncSession + +from seskit_api.dependencies import ( + AuthenticationRequired, + CurrentUser, + get_app_settings, + get_inbound_store_factory, + require_project, + require_user, +) +from seskit_api.downloads import ( + attachment_response, + fetch_original, + raw_response, + require_attachment, +) +from seskit_api.templating import render + +router = APIRouter(tags=["inbox"], include_in_schema=False) + +#: Rows on a page. A cap and not a suggestion, for the reason the Emails page has +#: one: without it a project with a year of mail renders all of it into a template. +PAGE_SIZE = 50 + + +def _status_for(error: APIError) -> int: + """HTTP status for a page that could not complete a download. + + As on the Domains and AWS pages: a credential or permission problem is the + user's AWS configuration, not their SESKit session, and answering 401 or 403 + would read as "you are not signed in". + """ + if error.error_type in {ErrorType.AUTHORIZATION_FAILED, ErrorType.AUTHENTICATION_FAILED}: + return 400 + return error.status_code + + +async def _receiving_domains(db: AsyncSession, project_id: str) -> list[str]: + """The domains this project receives for, for the filter and the empty state.""" + return [i.value for i in await list_identities(db, project_id) if i.receives_mail] + + +async def _list_page( + request: Request, + db: AsyncSession, + current: CurrentUser, + project: Project, + *, + domain: str | None, + before: str | None, + template: str, +) -> HTMLResponse: + domains = await _receiving_domains(db, project.id) + # An unrecognised domain narrows to nothing rather than raising: a stale or + # hand-edited URL should render the page it was clearly asking for. + selected = domain if domain in domains else None + rows, has_more = await list_received( + db, project.id, domain=selected, before=before, limit=PAGE_SIZE + ) + return render( + request, + template, + current=current, + nav_active="inbox", + project=project, + projects=await list_projects(db, current.user.id), + messages=rows, + has_more=has_more, + older=rows[-1].id if has_more and rows else None, + domains=domains, + selected_domain=selected, + ) + + +@router.get("/inbox", response_class=HTMLResponse, summary="Received email") +async def inbox_page( + request: Request, + db: Annotated[AsyncSession, Depends(get_session)], + current: Annotated[CurrentUser, Depends(require_user)], + project: Annotated[Project, Depends(require_project)], + domain: str | None = None, + before: str | None = None, +) -> HTMLResponse: + """The project's received mail, newest first.""" + return await _list_page( + request, db, current, project, domain=domain, before=before, template="pages/inbox.html" + ) + + +@router.get("/partials/inbox", response_class=HTMLResponse, summary="Inbox table fragment") +async def inbox_partial( + request: Request, + db: Annotated[AsyncSession, Depends(get_session)], + current: Annotated[CurrentUser, Depends(require_user)], + project: Annotated[Project, Depends(require_project)], + domain: str | None = None, + before: str | None = None, +) -> HTMLResponse: + """The table on its own, for the filter's HTMX swap. Authenticated like the + page it belongs to - this is a project's mail. + """ + return await _list_page( + request, + db, + current, + project, + domain=domain, + before=before, + template="partials/inbox_table.html", + ) + + +async def _detail_page( + request: Request, + db: AsyncSession, + current: CurrentUser, + project: Project, + message: InboundEmail, + *, + error: str | None = None, + status_code: int = 200, +) -> HTMLResponse: + """One renderer for the detail page, so a failed download lands on the page + the person was looking at and says why, instead of a bare JSON error in a tab. + """ + return render( + request, + "pages/inbox_detail.html", + status_code=status_code, + current=current, + nav_active="inbox", + project=project, + projects=await list_projects(db, current.user.id), + message=message, + error=error, + ) + + +async def _owned(db: AsyncSession, project: Project, inbound_id: str) -> InboundEmail: + message = await get_received(db, project.id, inbound_id) + if message is None: + # The dashboard's own convention for "not yours or not there" - the same + # answer either way, so a stranger cannot probe for real ids. + raise AuthenticationRequired("/inbox") + return message + + +@router.get("/inbox/{inbound_id}", response_class=HTMLResponse, summary="One received email") +async def inbox_detail( + request: Request, + inbound_id: str, + db: Annotated[AsyncSession, Depends(get_session)], + current: Annotated[CurrentUser, Depends(require_user)], + project: Annotated[Project, Depends(require_project)], +) -> HTMLResponse: + """One message in full.""" + message = await _owned(db, project, inbound_id) + return await _detail_page(request, db, current, project, message) + + +@router.get("/inbox/{inbound_id}/attachments/{index}", summary="Download an attachment") +async def inbox_attachment( + request: Request, + inbound_id: str, + index: int, + db: Annotated[AsyncSession, Depends(get_session)], + current: Annotated[CurrentUser, Depends(require_user)], + project: Annotated[Project, Depends(require_project)], + settings: Annotated[Settings, Depends(get_app_settings)], + factory: Annotated[InboundStoreFactory, Depends(get_inbound_store_factory)], +) -> Response: + """One attachment, as a download. + + A GET, because it changes nothing. A failure re-renders the message with the + reason rather than answering with a JSON envelope the browser would show raw. + """ + message = await _owned(db, project, inbound_id) + try: + require_attachment(message, index) + raw = await fetch_original(db, message, factory, settings) + return attachment_response(raw, index) + except APIError as error: + return await _detail_page( + request, + db, + current, + project, + message, + error=error.message, + status_code=_status_for(error), + ) + + +#: Headers for a received message's HTML, served as a document of its own. +#: +#: This is a stranger's markup, and the policy is as tight as one that still shows it can +#: be. ``sandbox`` gives the document an opaque origin with no script, no forms and no +#: top navigation; ``default-src 'none'`` means nothing loads from the network, which +#: also blocks every tracking pixel and remote image; inline styles are the one thing +#: allowed, because without them nearly every message is unreadable; images may only be +#: ``data:``. ``frame-ancestors 'self'`` and ``SAMEORIGIN`` let the message page, and only +#: the message page, put it in a frame. +MAIL_DOCUMENT_HEADERS = { + "Content-Security-Policy": ( + "sandbox; default-src 'none'; style-src 'unsafe-inline'; img-src data:; " + "frame-ancestors 'self'" + ), + "X-Frame-Options": "SAMEORIGIN", + "X-Content-Type-Options": "nosniff", + "Cache-Control": "private, no-store", + "Referrer-Policy": "no-referrer", +} + + +@router.get("/inbox/{inbound_id}/html", summary="A received message's HTML, sandboxed") +async def inbox_html( + request: Request, + inbound_id: str, + db: Annotated[AsyncSession, Depends(get_session)], + current: Annotated[CurrentUser, Depends(require_user)], + project: Annotated[Project, Depends(require_project)], +) -> Response: + """The HTML body as a document of its own, for the message page to frame. + + Served exactly as it was sent - nothing is cleaned or rewritten, because a + sanitiser would be one more thing to get wrong. What contains it is the policy + above and the ``sandbox`` attribute on the frame that shows it, which is the same + belt and braces on both sides: opened directly in a tab it is still sandboxed. + """ + message = await _owned(db, project, inbound_id) + if not message.html_body: + return await _detail_page( + request, + db, + current, + project, + message, + error="This message has no HTML body.", + status_code=404, + ) + return Response( + content=message.html_body, + media_type="text/html; charset=utf-8", + headers=MAIL_DOCUMENT_HEADERS, + ) + + +@router.get("/inbox/{inbound_id}/raw", summary="Download the original message") +async def inbox_raw( + request: Request, + inbound_id: str, + db: Annotated[AsyncSession, Depends(get_session)], + current: Annotated[CurrentUser, Depends(require_user)], + project: Annotated[Project, Depends(require_project)], + settings: Annotated[Settings, Depends(get_app_settings)], + factory: Annotated[InboundStoreFactory, Depends(get_inbound_store_factory)], +) -> Response: + """The original message as an ``.eml`` download.""" + message = await _owned(db, project, inbound_id) + try: + raw = await fetch_original(db, message, factory, settings) + except APIError as error: + return await _detail_page( + request, + db, + current, + project, + message, + error=error.message, + status_code=_status_for(error), + ) + return raw_response(raw, message.id) diff --git a/apps/api/src/seskit_api/routes/v1/domains.py b/apps/api/src/seskit_api/routes/v1/domains.py index 533444e..ec8a5a0 100644 --- a/apps/api/src/seskit_api/routes/v1/domains.py +++ b/apps/api/src/seskit_api/routes/v1/domains.py @@ -14,13 +14,20 @@ from fastapi import APIRouter, Depends, Response from seskit_core.db import get_session +from seskit_core.models import Identity from seskit_core.providers import IdentityType from seskit_core.services import list_identities from sqlalchemy.ext.asyncio import AsyncSession from seskit_api.dependencies import APIContext, require_api_key +from seskit_api.receiving import mx_record from seskit_api.routes.v1.api_keys import API_RESPONSES, apply_rate_limit_headers -from seskit_api.schemas.domains import DomainList, DomainResponse +from seskit_api.schemas.domains import ( + DomainList, + DomainResponse, + MxRecordResponse, + ReceivingResponse, +) router = APIRouter(tags=["domains"]) @@ -42,4 +49,26 @@ async def list_domains( identities = await list_identities(db, context.project.id) domains = [identity for identity in identities if identity.type is IdentityType.DOMAIN] - return DomainList(data=[DomainResponse.model_validate(domain) for domain in domains]) + return DomainList(data=[_response(domain) for domain in domains]) + + +def _response(identity: Identity) -> DomainResponse: + """A domain as a customer sees it, with whether it receives mail. + + ``receiving`` is attached here and not read off the row, because the MX host is + Amazon SES's and the row is core's, which knows nothing about a provider. The + record is withheld until receiving is on, so nobody publishes it early. + """ + response = DomainResponse.model_validate(identity) + record = mx_record(identity) + response.receiving = ReceivingResponse( + enabled=identity.receives_mail, + mx=( + MxRecordResponse( + record_type="MX", name=record.name, priority=record.priority, value=record.value + ) + if identity.receives_mail + else None + ), + ) + return response diff --git a/apps/api/src/seskit_api/routes/v1/inbound.py b/apps/api/src/seskit_api/routes/v1/inbound.py index a406397..c202bb6 100644 --- a/apps/api/src/seskit_api/routes/v1/inbound.py +++ b/apps/api/src/seskit_api/routes/v1/inbound.py @@ -10,19 +10,14 @@ from __future__ import annotations -import re from typing import Annotated, Any -from urllib.parse import quote from fastapi import APIRouter, Depends, Path, Query, Response from seskit_core.config import Settings from seskit_core.db import get_session -from seskit_core.email.parse import read_attachment from seskit_core.errors import APIError, ErrorType from seskit_core.models import InboundEmail -from seskit_core.models.base import utcnow -from seskit_core.services import InboundStoreFactory, get_connection, stored_credentials -from seskit_provider_aws_ses import EXPIRED_MESSAGE +from seskit_core.services import InboundStoreFactory from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession @@ -32,6 +27,12 @@ get_inbound_store_factory, require_api_key, ) +from seskit_api.downloads import ( + attachment_response, + fetch_original, + raw_response, + require_attachment, +) from seskit_api.routes.v1.api_keys import API_RESPONSES, apply_rate_limit_headers from seskit_api.schemas.inbound import ( InboundList, @@ -164,22 +165,10 @@ async def list_inbound( # ------------------------------------------------------------- downloads --- # -# Everything below serves bytes a stranger wrote, from the API's own origin. That -# is the whole risk, and it is handled the same way for both routes: whatever the -# sender declared, the response is a download and not a document. A browser that -# is pointed at one must not render it, sniff it, run script in it, or keep it. - -#: Applied to every download. -#: -#: ``nosniff`` stops a browser deciding the bytes are HTML because they look like -#: it. The CSP ``sandbox`` is what is left if one renders it anyway: no script, no -#: forms, no same-origin access to anything. ``no-store`` because this is somebody's -#: private mail and should not sit in a shared cache. -DOWNLOAD_HEADERS = { - "X-Content-Type-Options": "nosniff", - "Content-Security-Policy": "sandbox; default-src 'none'", - "Cache-Control": "private, no-store", -} +# Everything below serves bytes a stranger wrote, from the API's own origin. How +# that is made safe lives in ``seskit_api.downloads``, shared with the dashboard so +# the two cannot drift; what is here is only what is specific to ``/v1``: the API +# key, the rate-limit headers, and how a refusal is reported. _BINARY_OK: dict[int | str, dict[str, Any]] = { 200: { @@ -188,57 +177,6 @@ async def list_inbound( } } -_NOT_SAFE = re.compile(r"[^A-Za-z0-9._ -]") - - -def content_disposition(filename: str) -> str: - """A ``Content-Disposition`` that is always a download, for any filename. - - Two forms, because old clients read the plain one and current ones read the - encoded one. The plain form is reduced to characters that cannot end the - quoted string or start a new header; the encoded form carries the name with - everything outside the unreserved set percent-encoded. - - Both are built from the same basename, so neither can carry a path or a name - that is only dots, and an empty name becomes ``attachment`` in both. A name - that came through the parser already has all of that - this does not rely on - it, because it is the last thing between a sender's string and a header, and - a client saving the file should not have to sanitise what it was given. - """ - base = filename.replace("\\", "/").rsplit("/", 1)[-1].strip(" .") or "attachment" - fallback = _NOT_SAFE.sub("_", base) - return f"attachment; filename=\"{fallback}\"; filename*=UTF-8''{quote(base, safe='')}" - - -async def _original( - db: AsyncSession, - row: InboundEmail, - factory: InboundStoreFactory, - settings: Settings, -) -> bytes: - """The stored message, or the right error for why not. - - The retention date is checked before anything is fetched. After it the original - is gone or about to be, and an S3 call to confirm costs a round trip that can - only produce the same answer - so a request past it is refused at once, and one - with no known expiry is left to the fetch to settle. - """ - if row.raw_expires_at is not None and row.raw_expires_at <= utcnow(): - raise APIError(ErrorType.NOT_FOUND, EXPIRED_MESSAGE) - - connection = await get_connection(db, row.project_id) - if connection is None or not connection.has_credentials: - raise APIError( - ErrorType.INVALID_REQUEST, - "This project is not connected to AWS, so the original message cannot be " - "fetched. Connect it on the AWS page.", - ) - - store = factory( - connection.region, stored_credentials(connection, secret_key=settings.SECRET_KEY) - ) - return await store.fetch_message(bucket=row.storage_bucket, key=row.storage_key) - @router.get( "/inbound/{inbound_id}/attachments/{index}", @@ -271,21 +209,9 @@ async def download_attachment( row = await _owned(db, inbound_id, context.project.id) if row is None: raise APIError(ErrorType.NOT_FOUND, "No received email with that id.") - if index >= len(row.attachments): - # Before the fetch: a request for an attachment that does not exist should - # not cost a download of the message to find that out. - raise APIError(ErrorType.NOT_FOUND, "That message has no attachment with that index.") - - raw = await _original(db, row, factory, settings) - content = read_attachment(raw, index) - if content is None: - raise APIError(ErrorType.NOT_FOUND, "That message has no attachment with that index.") - - out = Response( - content=content.data, - media_type="application/octet-stream", - headers={**DOWNLOAD_HEADERS, "Content-Disposition": content_disposition(content.filename)}, - ) + require_attachment(row, index) + + out = attachment_response(await fetch_original(db, row, factory, settings), index) apply_rate_limit_headers(out, context) return out @@ -322,12 +248,6 @@ async def download_raw( if row is None: raise APIError(ErrorType.NOT_FOUND, "No received email with that id.") - raw = await _original(db, row, factory, settings) - - out = Response( - content=raw, - media_type="message/rfc822", - headers={**DOWNLOAD_HEADERS, "Content-Disposition": content_disposition(f"{row.id}.eml")}, - ) + out = raw_response(await fetch_original(db, row, factory, settings), row.id) apply_rate_limit_headers(out, context) return out diff --git a/apps/api/src/seskit_api/schemas/domains.py b/apps/api/src/seskit_api/schemas/domains.py index 6ef2e2c..a78199f 100644 --- a/apps/api/src/seskit_api/schemas/domains.py +++ b/apps/api/src/seskit_api/schemas/domains.py @@ -39,6 +39,48 @@ class DnsRecordResponse(BaseModel): ) +class MxRecordResponse(BaseModel): + """The MX record that sends a domain's mail to Amazon SES.""" + + record_type: str = Field(description="`MX`.", examples=["MX"]) + name: str = Field( + description=( + "The record name: the domain itself. Some DNS providers want only the part " + "before your domain, so check whether yours appends it for you." + ), + examples=["example.com"], + ) + priority: int = Field( + description="The preference value. SES takes mail at one host, so this only matters " + "if the domain lists others.", + examples=[10], + ) + value: str = Field( + description="The Amazon SES inbound endpoint for the domain's region.", + examples=["inbound-smtp.us-east-1.amazonaws.com"], + ) + + +class ReceivingResponse(BaseModel): + """Whether SESKit receives mail for a domain, and what to publish for it.""" + + enabled: bool = Field( + description=( + "Whether a receipt rule exists for this domain. Turned on from the Domains page " + "in the dashboard; there is no endpoint for it yet." + ), + ) + mx: MxRecordResponse | None = Field( + default=None, + description=( + "The record to publish at your DNS provider. **Null until receiving is on**: " + "publishing it earlier would send senders' mail to Amazon SES with no rule to " + "take it, and it would be refused. Publishing it replaces wherever the domain " + "receives mail now - consider a subdomain." + ), + ) + + class DomainResponse(BaseModel): """A sending domain and where its verification has got to.""" @@ -89,6 +131,13 @@ class DomainResponse(BaseModel): default=None, description="When SESKit last asked SES about this domain. Null until first checked. UTC.", ) + receiving: ReceivingResponse | None = Field( + default=None, + description=( + "Whether the project receives mail for this domain. Read-only: it reports what " + "the dashboard set up. Mail that arrives is read with `GET /v1/inbound`." + ), + ) created_at: datetime = Field(description="UTC.") diff --git a/apps/api/src/seskit_api/schemas/inbound.py b/apps/api/src/seskit_api/schemas/inbound.py index 242bcf8..4833eef 100644 --- a/apps/api/src/seskit_api/schemas/inbound.py +++ b/apps/api/src/seskit_api/schemas/inbound.py @@ -22,7 +22,6 @@ from pydantic import BaseModel, Field from seskit_core.models import InboundEmail -from seskit_core.models.base import utcnow class InboundVerdicts(BaseModel): @@ -213,10 +212,6 @@ class InboundList(BaseModel): ) -def _raw_available(row: InboundEmail) -> bool: - return row.raw_expires_at is None or row.raw_expires_at > utcnow() - - def summary_of(row: InboundEmail) -> InboundSummary: return InboundSummary( id=row.id, @@ -239,7 +234,7 @@ def summary_of(row: InboundEmail) -> InboundSummary: ), truncated=row.truncated, parse_failed=row.parse_failed, - raw_available=_raw_available(row), + raw_available=row.raw_available, raw_expires_at=row.raw_expires_at, ) diff --git a/apps/api/src/seskit_api/static/css/app.css b/apps/api/src/seskit_api/static/css/app.css index d36aa76..ed7d747 100644 --- a/apps/api/src/seskit_api/static/css/app.css +++ b/apps/api/src/seskit_api/static/css/app.css @@ -29,6 +29,10 @@ and in this theme also the wrong colour: --surface-raised is white, the same white as the card, so a code block had nothing but its border. */ --surface-sunken: #f2f4f7; + /* The canvas a received message is shown on. White in both themes, on purpose: email + is written for a white page, and a message that sets dark text and no background + is unreadable on a dark one. Defined only here, so the dark blocks inherit it. */ + --mail-canvas: #ffffff; --border: #e4e7ec; --border-strong: #d0d5dd; @@ -1105,6 +1109,20 @@ svg { margin-top: var(--space-2); } +/* Mail frame -------------------------------------------------------------- */ + +/* A received message's HTML, in a sandboxed iframe. The frame is the container; what is + inside it is a stranger's document and is styled by nothing here. */ +.frame { + display: block; + width: 100%; + height: 32rem; + border: 1px solid var(--border); + border-radius: var(--radius-lg); + background: var(--mail-canvas); + resize: vertical; +} + /* Code block -------------------------------------------------------------- */ .code { diff --git a/apps/api/src/seskit_api/templates/base.html b/apps/api/src/seskit_api/templates/base.html index 6ff7876..da9bc4c 100644 --- a/apps/api/src/seskit_api/templates/base.html +++ b/apps/api/src/seskit_api/templates/base.html @@ -70,6 +70,7 @@