diff --git a/.github/workflows/backup-d1.yml b/.github/workflows/backup-d1.yml index 6fc9d90fb..00ea814e5 100644 --- a/.github/workflows/backup-d1.yml +++ b/.github/workflows/backup-d1.yml @@ -2,8 +2,19 @@ # # This mirrors `wiki/scripts/backup-d1.sh` (which runs `wrangler d1 export`) # but on GitHub-hosted infrastructure, so backups happen even when no one's -# laptop is on. The export of the `wikilean` D1 database is uploaded as a -# workflow artifact and retained for 30 days. +# laptop is on. The repo is public (artifacts and run logs included), so the +# export of the `wikilean` D1 database is encrypted with age on the runner: +# only the .age file is uploaded as a workflow artifact (retained for 30 +# days), and the plaintext is deleted as soon as it is encrypted. Keys, +# decryption and restore: docs/BACKUP.md. +# +# ── Required repo variable ──────────────────────────────────────────────── +# BACKUP_AGE_RECIPIENTS — the age public key(s) the backups are encrypted +# to, one per line (age1…). Public keys only: it is a variable, not a +# secret, and it is printed in this workflow's public run log. The job +# fails before exporting anything if it is empty or not a valid age +# recipient list. Set it with: +# gh variable set BACKUP_AGE_RECIPIENTS --repo Deicyde/WikiLean # # ── Required repo secrets ───────────────────────────────────────────────── # CLOUDFLARE_API_TOKEN — a Cloudflare API token with D1 read/export scope @@ -21,12 +32,14 @@ # export step fails — the database backup is unaffected (continue-on-error). # # ── Running it manually ─────────────────────────────────────────────────── -# GitHub UI: Actions tab → "Backup D1" → "Run workflow". -# gh CLI: gh workflow run backup-d1.yml +# GitHub UI: Actions tab → "Backup D1" → "Run workflow" (branch: main). +# gh CLI: gh workflow run backup-d1.yml --ref main (never another ref, +# and never re-run a run from before encryption: docs/BACKUP.md) # Download the resulting artifact from the run's summary page. # -# ── Restore (into a fresh empty D1 of the same name) ────────────────────── -# cd wiki && npm ci && ./node_modules/.bin/wrangler d1 execute wikilean --remote --file=wikilean-.sql +# ── Restore ─────────────────────────────────────────────────────────────── +# Decrypt with the offline age identity, rehearse into a local D1, and only +# then touch production: docs/BACKUP.md. name: Backup D1 @@ -46,6 +59,51 @@ jobs: # name via the API (CLOUDFLARE_ACCOUNT_ID below), so this workflow has no # dependency on wiki/wrangler.jsonc being present in the checkout. steps: + # age and the recipient check come first, so a missing or bad key fails + # the job before anything is exported. + - name: Install age (pinned release, SHA-256 verified) + env: + AGE_VERSION: v1.3.2 + # sha256 of age-v1.3.2-linux-amd64.tar.gz: equals the release asset's + # GitHub digest, and the tarball verifies against the release's + # Sigsum proof (docs/BACKUP.md, "Upgrading the pinned age"). + AGE_SHA256: cbe24006683f8eb669266162894b9a522a1af52f2665fbc63a4bb032ed26ac10 + run: | + set -euo pipefail + tarball="$RUNNER_TEMP/age.tar.gz" + curl --fail -sSL --retry 3 --proto '=https' --tlsv1.2 -o "$tarball" \ + "https://github.com/FiloSottile/age/releases/download/${AGE_VERSION}/age-${AGE_VERSION}-linux-amd64.tar.gz" + echo "${AGE_SHA256} ${tarball}" | sha256sum --check --strict + mkdir -p "$RUNNER_TEMP/age-bin" + tar -xzf "$tarball" -C "$RUNNER_TEMP/age-bin" --strip-components=1 age/age + echo "$RUNNER_TEMP/age-bin" >> "$GITHUB_PATH" + + - name: Check backup recipients + env: + BACKUP_AGE_RECIPIENTS: ${{ vars.BACKUP_AGE_RECIPIENTS }} + run: | + set -euo pipefail + if [ -z "${BACKUP_AGE_RECIPIENTS//[[:space:]]/}" ]; then + echo "::error::Repo variable BACKUP_AGE_RECIPIENTS is empty; refusing to export an unencrypted backup (see docs/BACKUP.md)." + exit 1 + fi + printf '%s\n' "$BACKUP_AGE_RECIPIENTS" > "$RUNNER_TEMP/age-recipients.txt" + # age itself is the validator: it rejects malformed keys, identities + # (AGE-SECRET-KEY-…), and plugin recipients whose plugin is absent. + # It only *warns* about SSH key types it can't use (ECDSA, security + # keys, short RSA) and skips them, so any output fails the check too: + # otherwise a spare key could be dropped from every backup silently. + if ! age_out=$(age -R "$RUNNER_TEMP/age-recipients.txt" -o /dev/null < /dev/null 2>&1); then + printf '%s\n' "$age_out" >&2 + echo "::error::Repo variable BACKUP_AGE_RECIPIENTS is not a valid age recipient list; refusing to export (see docs/BACKUP.md)." + exit 1 + fi + if [ -n "$age_out" ]; then + printf '%s\n' "$age_out" >&2 + echo "::error::age warned about BACKUP_AGE_RECIPIENTS (a line it would skip, such as an unsupported SSH key type); refusing to export (see docs/BACKUP.md)." + exit 1 + fi + - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -65,38 +123,64 @@ jobs: id: stamp run: echo "ts=$(date -u +%Y%m%dT%H%M%SZ)" >> "$GITHUB_OUTPUT" - - name: Export D1 (wikilean) + - name: Export D1 (wikilean) and encrypt env: CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} CLOUDFLARE_ACCOUNT_ID: "02e868c47ebe2b175a0609df94e857e6" - run: wiki/node_modules/.bin/wrangler d1 export wikilean --remote --output="wikilean-${{ steps.stamp.outputs.ts }}.sql" + # `d1 export --remote` logs a presigned URL that serves the plaintext + # dump to anyone for an hour. The run log is public, so keep wrangler + # to warnings and errors, discard its stdout, redact URLs from its + # stderr (in case a wrangler upgrade starts logging the URL as a + # warning or in an error), and skip its on-disk debug log (which + # records every message, URL included). + WRANGLER_LOG: warn + WRANGLER_WRITE_LOGS: "false" + run: | + set -euo pipefail + umask 077 + # The plaintext stays outside the workspace and is removed on every + # exit path; only the .age file is left for the upload step. + plain="$RUNNER_TEMP/wikilean-${{ steps.stamp.outputs.ts }}.sql" + trap 'rm -f "$plain"' EXIT + # stdout goes to /dev/null; stderr goes through the redactor. pipefail + # keeps wrangler's exit status. + wiki/node_modules/.bin/wrangler d1 export wikilean --remote --output="$plain" 2>&1 > /dev/null \ + | sed -E 's#[A-Za-z][A-Za-z0-9+.-]*://[^[:space:]]+#[URL redacted]#g' >&2 + # An empty export must fail, not upload as a valid-looking .age. + test -s "$plain" || { echo "::error::wrangler produced an empty export"; exit 1; } + age -R "$RUNNER_TEMP/age-recipients.txt" -o "wikilean-${{ steps.stamp.outputs.ts }}.sql.age" "$plain" + rm -f "$plain" + test "$(head -n 1 "wikilean-${{ steps.stamp.outputs.ts }}.sql.age")" = "age-encryption.org/v1" - - name: Upload backup artifact + - name: Upload encrypted backup artifact uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: wikilean-d1-${{ steps.stamp.outputs.ts }} - path: wikilean-${{ steps.stamp.outputs.ts }}.sql + path: wikilean-${{ steps.stamp.outputs.ts }}.sql.age retention-days: 30 if-no-files-found: error # ── Research export (P2a) ─────────────────────────────────────────── # Second nightly artifact: the pseudonymized annotation_events JSONL # from the live Worker (GET /api/research/export.jsonl, bearer-gated). + # The endpoint is not public, so the artifact is encrypted like the + # backup; the response streams straight into age and never touches disk. # Ordered AFTER the backup upload and continue-on-error on purpose: a # failed/missing export must never fail the database backup. - - name: Fetch research export + - name: Fetch and encrypt research export id: research continue-on-error: true env: PIPELINE_TOKEN: ${{ secrets.PIPELINE_TOKEN }} run: | + set -euo pipefail test -n "$PIPELINE_TOKEN" # fail fast (and quietly) if the secret is missing curl --fail -sS --max-time 300 \ -H "Authorization: Bearer $PIPELINE_TOKEN" \ "https://wikilean.jackmccarthy.org/api/research/export.jsonl" \ - -o "research-export-${{ steps.stamp.outputs.ts }}.jsonl" + | age -R "$RUNNER_TEMP/age-recipients.txt" -o "research-export-${{ steps.stamp.outputs.ts }}.jsonl.age" - - name: Upload research export artifact + - name: Upload encrypted research export artifact # steps.research.outcome is its pre-continue-on-error result, so this # step is skipped (not failed) when the fetch didn't produce a file. if: steps.research.outcome == 'success' @@ -104,6 +188,6 @@ jobs: uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: research-export-${{ steps.stamp.outputs.ts }} - path: research-export-${{ steps.stamp.outputs.ts }}.jsonl + path: research-export-${{ steps.stamp.outputs.ts }}.jsonl.age retention-days: 30 if-no-files-found: error diff --git a/docs/BACKUP.md b/docs/BACKUP.md new file mode 100644 index 000000000..eaa5062d0 --- /dev/null +++ b/docs/BACKUP.md @@ -0,0 +1,266 @@ +# D1 backups: keys, decryption, restore + +WikiLean's canonical data lives in the production D1 database `wikilean`. It +holds articles and annotations, plus the better-auth tables (`users`, +`sessions`, `accounts`) and `flags`. A backup is therefore as sensitive as the +database itself: it contains emails, session IPs and user agents, and OAuth +tokens. + +## What exists + +| Path | Horizon | Where it lives | +|---|---|---| +| D1 Time Travel | 7 days on Workers Free, 30 days on Workers Paid (Cloudflare D1 limits) | Cloudflare, on-platform | +| `Backup D1` workflow (`.github/workflows/backup-d1.yml`) | 30 days | GitHub Actions artifacts, **age-encrypted** | +| `wiki/scripts/backup-d1.sh` (`npm run backup:d1`) | as long as you keep the file | the machine that runs it, **plaintext** (see below) | + +The workflow runs daily (scheduled for 08:27 UTC; GitHub often starts it hours +late) and on demand. Each run uploads two artifacts: + +- `wikilean-d1-`, holding `wikilean-.sql.age`: the full + `wrangler d1 export wikilean --remote`, every table. +- `research-export-`, holding `research-export-.jsonl.age`: the + pseudonymized `annotation_events` stream from the bearer-gated + `GET /api/research/export.jsonl`. + +The repository is public, so any signed-in GitHub account can download these +artifacts and read the run logs. Assume the ciphertext is public. Both files +are encrypted with [age](https://age-encryption.org) to the public key(s) in +the repo variable `BACKUP_AGE_RECIPIENTS`, and only the matching private key +can read them. + +- On the runner, the plaintext D1 dump exists only in the temp directory, from + export to encryption. The research export streams straight into age. +- `wrangler d1 export --remote` logs a presigned URL that serves the plaintext + dump to anyone for an hour. The workflow runs wrangler with + `WRANGLER_LOG=warn`, discards its stdout, and redacts URLs from its stderr, so + that URL stays out of the log even if a wrangler upgrade starts printing it + as a warning or in an error. +- The job fails **before exporting anything** if `BACKUP_AGE_RECIPIENTS` is + empty or is not a valid age recipient list. A pasted private key, a malformed + key and a plugin recipient without its plugin are all rejected. So is any + line age would only warn about and skip, such as an ECDSA or security-key SSH + key, so a spare key can't silently drop out of the list. + +Runs from before this encryption existed uploaded plaintext and logged that +URL. See [One-time cutover](#one-time-cutover-from-plaintext-backups). + +## One-time cutover from plaintext backups + +Before this change, every run uploaded the dump as a plaintext `.sql`, and the +research export as plaintext `.jsonl`, under the same artifact names. Each run +log also printed that one-hour presigned plaintext URL. Any GitHub account +could download those artifacts for 30 days. The URLs have expired, but the +artifacts last until their retention ends. These steps are destructive and +outward-facing, so the maintainer decides and runs them: + +1. Disable the workflow until the variable is set and this change is merged: + `gh workflow disable backup-d1.yml --repo Deicyde/WikiLean`. +2. **Never re-run a run from before this change, and dispatch only from + `main`.** A re-run replays the workflow file of the run's original commit, so + re-running an old run uploads a plaintext dump and logs a live presigned URL + again. GitHub allows re-runs for 30 days after a run. A manual dispatch uses + the workflow file on the ref you pick, and any branch cut before this change + still has the plaintext version. Delete the old runs so they can't be + re-run: + + ```bash + gh run list --workflow backup-d1.yml --repo Deicyde/WikiLean --limit 200 \ + --json databaseId,createdAt,headSha # pick the runs before the first encrypted one + gh run delete --repo Deicyde/WikiLean # once per old run + ``` + + Deleting a run also deletes its log and artifacts. +3. Delete every `wikilean-d1-*` and `research-export-*` artifact created before + the first encrypted run. List them with the `gh api` command under + [Key rotation](#key-rotation), filtering on `created_at`, and delete each id. +4. Treat their contents as exposed: revoke the OAuth app's user tokens, rotate + `BETTER_AUTH_SECRET` (this signs everyone out), and decide whether users + must be notified. + +## One-time setup + +Do this on a trusted machine (`brew install age`). + +1. **Generate the keypair.** The private key never goes to GitHub: not to the + repo, not to a secret, not to a runner. The workflow only encrypts, so it + never needs the private key. + + ```bash + age-keygen -o wikilean-backup.key # prints "Public key: age1…" + age-keygen -y wikilean-backup.key > backup-recipients.txt + ``` + + Keep `wikilean-backup.key` offline with Jack, for example in a password + manager plus one offline copy. Losing it makes every encrypted backup + unreadable. A second keypair held separately is a cheap spare: append its + `age1…` line to `backup-recipients.txt`, and either key can decrypt. + (`age-keygen -pq` makes a post-quantum `age1pq1…` key. The workflow's + pinned age v1.3.2 accepts it, but decrypting it needs age ≥ 1.3. age refuses + to mix post-quantum and classic recipients, so the keys in the list must be + all `age1pq1…` or all classic. A mixed list fails every run.) + +2. **Set the repo variable.** It holds public keys only, one per line, and `#` + comments are allowed. It is a *variable* rather than a secret because it + isn't secret: it is printed in the workflow's public run log. + + ```bash + gh variable set BACKUP_AGE_RECIPIENTS --repo Deicyde/WikiLean < backup-recipients.txt + gh variable get BACKUP_AGE_RECIPIENTS --repo Deicyde/WikiLean # should show only age1… lines + ``` + + If an `AGE-SECRET-KEY-…` line ever lands in the variable, the workflow + refuses it. It is still exposed, though, so rotate (below). + +3. **Re-enable the workflow and prove a run end to end.** + + ```bash + gh workflow enable backup-d1.yml --repo Deicyde/WikiLean + gh workflow run backup-d1.yml --repo Deicyde/WikiLean --ref main + gh run list --workflow backup-d1.yml --repo Deicyde/WikiLean --limit 1 # note the run id + ``` + + Then download and decrypt that run's artifact (next section). A run is good + when the artifact holds a single `.sql.age` file that decrypts to SQL. + +## Download and decrypt + +```bash +mkdir -p ~/wikilean-restore && cd ~/wikilean-restore # not inside a synced folder +gh run download --repo Deicyde/WikiLean -n wikilean-d1- -D . +age -d -i /path/to/wikilean-backup.key -o wikilean-.sql wikilean-.sql.age +``` + +The research export works the same way: download `research-export-` and +decrypt `research-export-.jsonl.age`. + +The decrypted `.sql` is the production database, credentials included. Keep it +on an encrypted disk and out of synced folders, and delete it when you're done. + +## Restore drill: always local first + +Restore into a throwaway local D1 before touching production. `--persist-to` +keeps the drill out of your normal `wrangler dev` state, and an empty directory +avoids collisions with the dump's `CREATE TABLE` statements. Run this from the +repository root: + +```bash +cd wiki && npm ci +npx wrangler d1 execute wikilean --local --persist-to ~/wikilean-restore/d1 \ + --file=$HOME/wikilean-restore/wikilean-.sql +npx wrangler d1 execute wikilean --local --persist-to ~/wikilean-restore/d1 \ + --command "SELECT (SELECT COUNT(*) FROM articles) AS articles, (SELECT COUNT(*) FROM annotation_events) AS events, (SELECT COUNT(*) FROM d1_migrations) AS migrations" +``` + +Record the wall-clock time and the row counts: a measured restore time is one +input to the roadmap's backup RPO/RTO items. Delete `~/wikilean-restore` +afterwards. + +## Restoring production + +Pick the shortest path that covers the damage. Both paths below are +destructive operator actions, and both need Cloudflare credentials. + +**Time Travel (recent damage, database still exists).** It restores the +database in place to any minute inside the retention window above. + +```bash +cd wiki +npx wrangler d1 time-travel info wikilean --timestamp=2026-10-13T08:00:00Z # check the point exists +npx wrangler d1 time-travel restore wikilean --timestamp=2026-10-13T08:00:00Z +``` + +`restore` overwrites all data in the database and prints the previous +bookmark. Save that bookmark: restoring to it (`--bookmark=`) undoes +the restore. + +**From an encrypted dump (older than Time Travel, or the database is gone).** +Decrypt it, rehearse it locally as above, and then import it into a **fresh, +empty** D1. The dump's `CREATE TABLE` statements fail against a populated +database. + +```bash +npx wrangler d1 create wikilean-restore +npx wrangler d1 execute wikilean-restore --remote --file=$HOME/wikilean-restore/wikilean-.sql +``` + +Everything that names the old database has to move with it, or it keeps +reading the abandoned copy without any error: + +- `database_id` in `wiki/wrangler.jsonc`, the Worker's `DB` binding. This is a + Worker change, so it ships through the reviewed release and promotion path. + Commands run from `wiki/` follow this id as long as `database_name` stays + `wikilean`. That covers `wiki/scripts/backup-d1.sh`, + `site/discover_articles.py` and every `wrangler d1 execute wikilean --remote`. +- The database name `wikilean` in `.github/workflows/backup-d1.yml`. The + workflow runs without `wrangler.jsonc` and resolves the database by name + through the API. While the old database exists, the workflow keeps backing it + up and stays green. +- The Brain D1 snapshot acquisition. It pins the id in + `brain/acquire_d1_snapshot.py`, `brain/d1_snapshot_bundle.py` and the request + descriptor `brain/authority/requests/d1-community-snapshot-v1.json`. Both + Python files also pin that descriptor's sha256, so moving the id takes a new, + reviewed request version. An in-place edit will not work. + +Then decrypt the next backup and confirm that it contains post-restore data. + +**After either path:** a restore rewinds `articles.version`, so new edits +reuse version numbers whose rendered HTML is still cached under +`render:v::` in `RENDER_CACHE` for up to 30 days. Before +reopening writes, bump the render-cache prefix (a Worker change) or purge those +keys. Otherwise readers can be served pages from the discarded timeline. The +`page:*` entries expire within an hour on their own. + +## Key rotation + +Rotate when a key holder changes, when the private key may have leaked, or on a +routine schedule. + +1. Generate the new keypair (setup step 1) and update the variable to the new + public key(s) (step 2). The next run uses them, with no workflow change. +2. Artifacts made before the change are still encrypted to the old key, and + they expire 30 days after their run. Keep the old private key until then, + or delete those artifacts. +3. **If the old private key leaked**, every artifact encrypted to it is + readable by whoever holds it. Delete them all right away: + + ```bash + gh api --paginate repos/Deicyde/WikiLean/actions/artifacts \ + --jq '.artifacts[] | select(.name | test("^(wikilean-d1|research-export)-")) | "\(.id) \(.name) \(.created_at)"' + gh api -X DELETE repos/Deicyde/WikiLean/actions/artifacts/ + ``` + + Then treat their contents as exposed: revoke the OAuth app's user tokens, + rotate `BETTER_AUTH_SECRET`, and consider whether users must be notified. + +## Upgrading the pinned age + +The workflow downloads `age--linux-amd64.tar.gz` from the upstream +GitHub release. It checks the tarball against `AGE_SHA256` before extracting, +and installs only the `age` binary. To move to a new release: + +```bash +V=v1.x.y # the new release tag +gh release download "$V" --repo FiloSottile/age -p "age-$V-linux-amd64.tar.gz*" +shasum -a 256 "age-$V-linux-amd64.tar.gz" +gh api "repos/FiloSottile/age/releases/tags/$V" \ + --jq '.assets[] | select(.name == "age-'"$V"'-linux-amd64.tar.gz") | .digest' +``` + +The two hashes must match. Also verify the tarball's Sigsum proof as described +in age's [SIGSUM.md](https://github.com/FiloSottile/age/blob/main/SIGSUM.md), +then update `AGE_VERSION` and `AGE_SHA256` together. + +## The local script + +`wiki/scripts/backup-d1.sh` (`npm run backup:d1`) does **not** encrypt yet. It +writes a plaintext `wikilean-.sql` to `wiki/backups/`, which is gitignored +but otherwise unprotected. Until it encrypts, treat its output like the +decrypted dump above, or encrypt it straight away: + +```bash +age -R backup-recipients.txt -o wikilean-.sql.age wikilean-.sql && rm wikilean-.sql +``` + +Its header also suggests restoring straight into production with `--remote`. +Follow the local-first drill above instead.