diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 810ff15..8dfc393 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -266,7 +266,7 @@ jobs: - name: Assert no decrypted artefact is tracked run: | fail=0 - for pattern in '.env' '.rendered/' '.purge-secrets.txt' 'certificates/'; do + for pattern in '.env' '.rendered/' '.purge-secrets.txt' 'certificates/' 'backups/'; do if git ls-files | grep -E "(^|/)${pattern//./\\.}" | grep -v '\.env\.example'; then echo "::error::tracked file matching '${pattern}' — it must be gitignored" fail=1 diff --git a/.gitignore b/.gitignore index da99dfa..9bd7756 100644 --- a/.gitignore +++ b/.gitignore @@ -33,12 +33,18 @@ certificates/ stacks/*/snmp-exporter/mibs/ # ---- Backups ---- -# `make backup` writes stack volumes here; `make backup-firewall` writes the -# SOPS-encrypted pfSense config here. Neither belongs in git. The firewall -# config in particular carries the WAN address, the full rule set and user +# `make backup` writes age-encrypted volume archives to backups/volumes//, +# and `make restore` writes a pre-restore snapshot of what it is about to +# replace to backups/volumes/.pre-restore-/. `make backup-firewall` +# writes the SOPS-encrypted pfSense config to backups/firewall/. None of it +# belongs in git. +# +# The firewall config carries the WAN address, the full rule set and user # password hashes — docs/security.md says those are deliberately unpublished, # and encryption does not change that a public repository is the wrong place -# for them. See scripts/backup-firewall.sh. +# for them. grafana-data is the same argument: it holds the admin password +# hash, every API token and every datasource credential. See +# scripts/backup-firewall.sh. backups/ # ---- Runtime state produced by the stack ---- diff --git a/Makefile b/Makefile index 0ac2c97..3fb7a7c 100644 --- a/Makefile +++ b/Makefile @@ -262,21 +262,36 @@ screenshots: ## Render the dashboards to docs/images/ (stack must be up) @# what to look for. ./scripts/capture-screenshots.sh $(STACK) -.PHONY: backup +.PHONY: backup-firewall backup-firewall: ## Pull morpheus's pfSense config and encrypt it to ./backups/ @# The single largest unmitigated failure in the estate is morpheus dying @# with no config export. Output is gitignored and never committed — see @# the header of scripts/backup-firewall.sh for why. ./scripts/backup-firewall.sh $(ARGS) -backup: ## Back up the stack's volumes to ./backups/ - @mkdir -p backups - @for v in prometheus-data loki-data grafana-data alertmanager-data; do \ - printf 'backing up %s\n' "$$v"; \ - docker run --rm -v $(STACK)_$$v:/data -v "$(PWD)/backups:/backup" \ - alpine tar czf "/backup/$$v.tar.gz" -C /data . ; \ - done - @printf '\033[0;32mwrote backups/\033[0m\n' +.PHONY: backup +backup: ## Quiesce the stack, archive its volumes to ./backups/ and verify + @# Thin on purpose. This target used to BE the implementation, and every + @# defect in #64 followed from that: one fixed output filename that tar + @# truncated at open, so the only way to lose a backup was to take one; a + @# hardcoded volume list that had silently skipped alloy-data since Alloy + @# was added; an unpinned `alpine`; a hot copy of an open TSDB; and no + @# verification beyond tar's exit status. + @# + @# The volume list and the services to stop are now derived from + @# compose.yaml, so a sixth volume cannot be forgotten. STACK goes in the + @# environment rather than positionally: the script's arguments are flags. + STACK=$(STACK) ./scripts/backup-volumes.sh $(ARGS) + +.PHONY: restore +restore: ## Restore the stack's volumes from a backup set (ARGS="--from ") + @# Deliberately a separate script from `backup`. One script that both writes + @# archives and overwrites live volumes is one mistyped flag from an outage, + @# and scripts/backup-firewall.sh — the model for both — is non-destructive + @# throughout. The volume inventory is not duplicated: restore-volumes.sh + @# reads `backup-volumes.sh --inventory`, the way every SNMP tool reads + @# scripts/snmp-targets.sh. + STACK=$(STACK) ./scripts/restore-volumes.sh $(ARGS) .PHONY: purge-history-dry-run purge-history-dry-run: ## Preview the git-history secret purge (safe) diff --git a/README.md b/README.md index cd141e2..e785dc7 100644 --- a/README.md +++ b/README.md @@ -65,9 +65,9 @@ incident. - **Supply chain pinned by digest.** Every image carries both a tag and a `sha256:` digest, so a moved tag cannot change what deploys. CI enforces it; `make pin-digests` re-resolves them from the registry. -- **Documented decisions and runbooks.** Eight ADRs covering what was chosen and - what was rejected — including the costs accepted knowingly; nine runbooks for - the operations that are easy to get wrong at 1am. +- **Documented decisions and runbooks.** Eleven ADRs covering what was chosen + and what was rejected — including the costs accepted knowingly; twelve + runbooks for the operations that are easy to get wrong at 1am. ## Architecture @@ -164,7 +164,8 @@ rack; a dashed border means egress only. Full topology and data flow in │ ├── observability.md security.md roadmap.md │ ├── adr/ # 11 architecture decision records │ └── runbooks/ # deploy, add device, rotate creds, certs, key backup, -│ # purge, restore the firewall, ship firewall logs, +│ # purge, restore the firewall, restore the stack, +│ # ship firewall logs, verify the alert path, │ # enable suricata, fit the UPS battery └── Makefile # make help ``` @@ -195,7 +196,8 @@ $ make help secrets-edit Edit the encrypted secrets in $EDITOR secrets-verify-backup Check a backup age key decrypts the secrets validate Run every check CI runs - backup Back up the stack's volumes to ./backups/ + backup Quiesce the stack, archive its volumes to ./backups/ and verify + restore Restore the stack's volumes from a backup set ... ``` diff --git a/docs/observability.md b/docs/observability.md index bf47411..a9dc4bd 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -236,9 +236,11 @@ make ps # container status make logs SERVICE=grafana # tail one service make reload # hot-reload Prometheus, Alertmanager, snmp-exporter make validate # everything CI runs -make backup # tar the data volumes into ./backups/ +make backup # quiesce, archive, encrypt and verify the data volumes +make restore ARGS=--list # the backup sets that exist make down # stop, keep data -make nuke # stop, destroy data (prompts) +make nuke # stop, destroy data (prompts) — recoverable, see + # docs/runbooks/restore-the-stack.md ``` Prometheus and Alertmanager are started with lifecycle endpoints enabled, so diff --git a/docs/roadmap.md b/docs/roadmap.md index 7c317b2..e42ce36 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -93,7 +93,11 @@ what left this one unfireable for months. off `prometheus`, and buy a spare ProDesk.** A backup on the same shelf as the thing it protects is not a backup, and [`restore-the-firewall.md`](runbooks/restore-the-firewall.md) stays a - hypothesis until it has been restored onto a spare once. + hypothesis until it has been restored onto a spare once. The volume sets + `make backup` writes have exactly the same defect: they sit on the host they + protect. Unlike the firewall, they have now been restored — the whole stack + was brought up on a restored set on 2026-08-29 and verified. Getting a copy + off this host is the part that is still missing. - **[#93](https://github.com/Gerrrt/HomeLab/issues/93) Replace the UPS battery.** An APCRBC115 went into `mjolnir` on 2026-08-28 and passed its self-test the same day: `upsTestResultsSummary` `4` → `1`, `upsBatteryVoltage` off its @@ -141,7 +145,10 @@ what left this one unfireable for months. - **[#77](https://github.com/Gerrrt/HomeLab/issues/77) Schedule something.** Nothing runs `make backup`, `make backup-firewall`, `make check-digests` or `make secrets-verify-backup` on a timer. The last of those is the only proof - the secrets are recoverable. + the secrets are recoverable, and `make restore ARGS=--dry-run` is now the + only proof the volumes are — `make backup ARGS='--verify-only --all'` is the + cheap version to put on the timer, because it is what catches bit-rot in a + set nobody has touched for a week. ## Decided but not built diff --git a/docs/runbooks/deploy-stack.md b/docs/runbooks/deploy-stack.md index 24a7e00..7c5f363 100644 --- a/docs/runbooks/deploy-stack.md +++ b/docs/runbooks/deploy-stack.md @@ -86,7 +86,8 @@ make up ``` Data volumes survive `make down` and `make up`. Only `make nuke` destroys them, -and it prompts. +it prompts, and it is recoverable from a backup set — see +[`restore-the-stack.md`](restore-the-stack.md). ## Troubleshooting @@ -103,9 +104,19 @@ and it prompts. ## Backups ```bash -make backup # tars each data volume into ./backups/ +make backup # quiesce, archive, encrypt, verify +make backup ARGS=--list # the sets that exist +make restore ARGS="--dry-run --from latest" # prove the newest one is readable ``` -Prometheus and Loki data is reproducible-ish (it re-accumulates), but Grafana's -volume holds annotations and users. The dashboards themselves are in git, so a -lost Grafana volume is an inconvenience rather than a loss. +`make backup` stops the services for the length of the copy — about ninety +seconds — because a copy of a live store is not a backup. It writes one +timestamped, age-encrypted, verified archive per volume into +`backups/volumes//`, keeps the seven newest complete sets, and prunes +only after the new one has verified. + +Prometheus and Loki data re-accumulates, so losing it is a hole in the record +rather than a loss of function. `grafana-data` is the exception and the reason +this matters: the dashboards are in git, but the users, the annotations, the +admin password and every UI edit that was never exported exist only in that +volume. Restoring any of it is [`restore-the-stack.md`](restore-the-stack.md). diff --git a/docs/runbooks/restore-the-stack.md b/docs/runbooks/restore-the-stack.md new file mode 100644 index 0000000..b4e1f4f --- /dev/null +++ b/docs/runbooks/restore-the-stack.md @@ -0,0 +1,372 @@ +# Runbook: Restore the observability stack + +**Target:** the five Docker data volumes on `prometheus` (10.0.99.20), VLAN 99 +**Time:** 10 minutes for one volume; 30 for the whole set on a rebuilt host +**You will need:** a backup set, the age private key, and the stack stopped — +the restore script stops it for you + +Nothing in the house breaks when this stack is down, and that is exactly what +makes it easy to lose. Metrics and logs re-accumulate, the dashboards are in +git, and the alert rules are in git, so for four of the five volumes the loss is +a hole in the record rather than a loss of function. The record is the thing +that tells you whether the incident in front of you has happened before, and it +is the one part of this repository that cannot be rebuilt from the repository. +`grafana-data` is different in kind again: it holds the users, the annotations, +the admin password and every dashboard edit made in the UI and never exported. +It is the only state here with no copy in git. + +> [!CAUTION] +> Restoring is destructive and its worst failure is silent. A stack brought back +> with a stale `grafana.db` shows the right dashboards over the right data and +> is wrong about who logged in, what was annotated, what is silenced and what +> the admin password is. Nothing will tell you. Read §4 before you start §2. + +--- + +## 0. Before anything breaks + +Three things must be true, and none of them is automatic. + +**A current set exists.** + +```bash +make backup # quiesce, archive, encrypt, verify +make backup ARGS=--list +``` + +The output lands in `backups/volumes//`, which is gitignored. Each +archive is age-encrypted to the same recipient as everything else in +[`.sops.yaml`](../../.sops.yaml). A set is complete only when it has a +`MANIFEST`; the backup writes that last, and a set without one is the wreckage +of a failed run. + +The default stops the services for the length of the copy — about ninety seconds +— because a copy of a live store is not a backup. `--hot` skips the stop and +records `mode hot` in the manifest, and everything below treats such a set as +unproven, because it is. + +**It lives somewhere other than the machine that made it.** A set on +`prometheus` protects against a bad upgrade and a mistyped command. It protects +against nothing that happens to `prometheus`. Copy it to the backup target and +offsite. This is the step that gets skipped — see [`roadmap.md`](../roadmap.md). + +**The age key is backed up.** A volume archive you cannot decrypt is a disk you +cannot read. See [`back-up-the-age-key.md`](back-up-the-age-key.md) and +`make secrets-verify-backup`. + +And it has been dry-run restored at least once: + +```bash +make restore ARGS="--dry-run --from latest" +``` + +A set that has never been decrypted is a set you are hoping about. + +--- + +## 1. Decide which failure you have + +| Symptom | Likely cause | Go to | +| --- | --- | --- | +| Grafana loads, every panel empty, nothing older than this morning | The TSDB is gone — usually `make nuke` in the wrong window | §3 | +| Grafana will not start, or starts with no users and no saved dashboards | `grafana.db` damaged, most often after an upgrade migrated it | §2, `grafana-data` | +| Everything normal, every alert silence vanished | `alertmanager-data` lost | §2, `alertmanager-data` | +| Loki answers but returns nothing older than the last restart | `loki-data` lost or partly written | §2, `loki-data` | +| Duplicate log lines flooding Loki after a restart | `alloy-data` positions lost; Alloy re-read from the top | §2, `alloy-data` | +| The host's disk is gone, or the filesystem is read-only | Hardware | §3, after rebuilding the host | +| Files under `/var/lib/docker/volumes` deleted or encrypted | Ransomware, or a mis-aimed `rm -rf` | §3 — and **not** from a set on this host | + +Run `make ps` and `docker volume ls` before anything else. A stack that looks +like it lost its data is far more often a container that failed to start than a +volume that is gone, and the first five rows above cost one volume where §3 +costs five. + +Check the volume name carefully. This host has carried volumes under two compose +project prefixes, and `prometheus_grafana-data` is not +`observability_grafana-data`. The backup and restore scripts derive the prefix +from the `name:` key in `compose.yaml` rather than guessing it, which is why they +refuse to run against a volume that does not exist instead of quietly creating +an empty one. + +Most of this table is not an emergency. Metrics and logs refill on their own, and +restoring them costs everything collected since the backup. Restore the volume +you lost, not the set it came in. + +--- + +## 2. Restore one volume + +Verify first, and read what it prints: + +```bash +make restore ARGS="--dry-run --from 20260829T064124Z --only grafana-data" +``` + +That decrypts the archive, reads it end to end, checks it against the manifest's +SHA-256, and prints the live volume's current size beside the archive's. Nothing +is touched. If the two sizes are wildly different, stop and work out why before +going on. + +Then, without `--dry-run`: + +```bash +make restore ARGS="--from 20260829T064124Z --only grafana-data" +``` + +It stops the whole stack, writes a pre-restore snapshot of the current contents +to `backups/volumes/.pre-restore-/`, empties the volume and extracts the +archive into it. It asks you to type the stamp first; that is deliberate, and +there is no `--yes`. It then reports the owning uid it found against the one the +service needs, and it does not silently correct a mismatch — a volume restored +byte-for-byte that its service cannot write to is a restore that failed. + +It leaves the stack **stopped**. Bring it up yourself and then run §4: + +```bash +make up +``` + +--- + +## 3. Restore the whole set + +Same, without `--only`. On a rebuilt host, do it in this order: + +1. Restore the age key and run `make render`, or nothing will start. +2. `make restore ARGS="--from "` — volumes that do not exist yet are + created rather than replaced, so a bare host is a valid target. +3. `make up`. + +> [!CAUTION] +> On a rebuilt host, restore **before** the first `make up`, not after. Starting +> Grafana once against an empty volume writes a fresh `grafana.db` that the +> restore then discards, which is merely wasteful. Starting Prometheus once and +> then restoring an older TSDB over it interleaves two sets of blocks, which is +> not. + +One more thing bites only on a rebuilt host, and it is not about the volumes. + +> [!CAUTION] +> **Grafana will not start if the host has no internet.** `GF_INSTALL_PLUGINS` +> makes its background installer contact `grafana.com` on every start, and a +> failure there is fatal — it crash-loops, even though both plugins are already +> in the volume you just restored. If the uplink is part of what you are +> recovering from, start Grafana with that variable emptied and put it back +> once the network is up. Every other service starts offline. + +--- + +## 4. Verify + +```bash +# 1. Did it come back at all? +make up && make ps + +# 2. prometheus-data — ask for a value the running stack could not have +# scraped since boot: an hour before the backup stamp. +curl -sG http://localhost:9090/api/v1/query \ + --data-urlencode 'query=count(up)' \ + --data-urlencode "time=$(date -u -d ' -1 hour' +%s)" \ + | jq '.data.result | length' # zero means that block did not come back + +# 3. grafana-data — the dashboards are provisioned from git and prove nothing; +# under unified storage they do not even appear in the legacy dashboard +# table. What exists only in the volume is the user table. Read it straight +# out of the database and skip the credential entirely: +docker run --rm -v observability_grafana-data:/d:ro \ + "$(./scripts/image-for.sh archiver)" cat /d/grafana.db > /tmp/g.db +python3 -c "import sqlite3;print(sqlite3.connect('/tmp/g.db').execute( + \"select login, created from user\").fetchall())" +# The admin's `created` must PREDATE the restore. If it is today's date, +# grafana.db did not come back and Grafana provisioned itself a fresh one. +# This lab has no annotations, so the annotation timestamps some runbooks +# suggest are a vacuous check here — the user table is the one that works. + +# 4. alertmanager-data — silences live in the volume, not the config. +docker exec alertmanager amtool silence query --alertmanager.url=http://localhost:9093 + +# 5. loki-data — query the restored window, not the last five minutes. +curl -sG http://localhost:3100/loki/api/v1/query_range \ + --data-urlencode 'query={host=~".+"}' \ + --data-urlencode "start=" \ + --data-urlencode "end=" | jq '.data.result | length' +``` + +Then the assertions that must **fail**. These are the ones that catch a restore +which looks perfect and did nothing. + +```bash +# 6a. Prometheus must have a GAP between the backup stamp and the restart. +# Continuous data across that window means you are looking at fresh scrapes +# and the restore quietly no-opped. +curl -sG http://localhost:9090/api/v1/query \ + --data-urlencode 'query=count(up)' \ + --data-urlencode "time=" \ + | jq '.data.result | length' # must be zero + +# 6b. Alloy WILL re-ship, and this measures how much. Restoring alloy-data puts +# the log positions back to their offsets at the stamp, so Alloy re-reads +# every file from there and pushes the lines again — carrying their +# ORIGINAL timestamps, so the duplicates land inside a window that closed +# before the restore. Count that window per host, twice, ten minutes apart. +# A climb is expected. It is not a failed restore; it is the cost of one, +# and it tells you how far Alloy still has to catch up. +# +# Measured during the 2026-08-29 rehearsal: about 4,600 duplicate lines +# inside the hour before the stamp within four minutes of start-up. +# +# The lines that must NOT climb are the ones carrying the ORIGINAL host +# label. Those are the restored data. If that number moves, something is +# writing into your restored history. + +# 6c. Grafana's admin password must NOT be the one in the rendered .env. +# Grafana applies GF_SECURITY_ADMIN_PASSWORD only when it creates the admin +# user, so if the value in .env logs you in, grafana.db did not come back +# and Grafana provisioned itself a fresh one. +``` + +> [!NOTE] +> Step 6 matters more than it looks, and 6c is weaker than it looks. A restore +> that puts back *something* leaves a stack that starts, serves dashboards and +> answers queries — because the dashboards, the rules and the datasources all +> come from git and never needed the volume at all. Everything visible works. +> The only evidence a volume came back is data that predates the restore, so +> check the age of what you are looking at, not that you are looking at +> something. And 6c proves nothing if the backed-up database was created with +> the password now in `.env`: identical values make the test vacuous while +> looking like it passed — and in this lab it IS the case, so 6c currently +> proves nothing at all. Use the `created` column on the admin row from step 3 +> instead: a user created before the stamp cannot have been provisioned by the +> restart, and that check does not depend on the password differing or on any +> annotation existing. + +--- + +## 5. Afterwards + +- **Take a fresh backup now.** The restored volumes are the live ones, and the + pre-restore snapshot is the only remaining copy of what you replaced. Keep it + deliberately or delete it deliberately; it holds the same secrets as any other + set and lives under the same rules. +- **Prometheus ages restored blocks from their own timestamps, not from today.** + Thirty days of retention against a twenty-day-old set is ten days of history, + shrinking. +- **Review the silences.** Anything created after the stamp is gone, and + anything live at the stamp is suppressing again. +- If the cause was `make nuke`, the confirmation worked and someone typed it. + Record it in [`roadmap.md`](../roadmap.md) if the design should have made that + harder. + +--- + +## What is proven, and what is not + +The whole-stack restore in §3 was performed on 2026-08-29. It is no longer a +hypothesis. The set was restored into a scratch project and the entire stack was +started on the result; §4 was run against it, including the negative assertions. + +**What it established.** + +- **The Prometheus TSDB restores and serves.** Every block was reported healthy, + the WAL replayed in about a second, and instant queries returned series from a + day and a week before the stamp. +- **There is a genuine gap after the stamp.** Queries at the stamp plus thirty + minutes, one hour and three hours all returned nothing, which is the assertion + that catches a restore that quietly did nothing. The restored stack was + simultaneously scraping its own targets, so it was serving restored history + and collecting new data with a clean seam between them. +- **`grafana.db` restores intact.** Compared table by table against the live + database: users, datasources, orgs, annotations and preferences all matched, + and the admin row's `created` was twelve days before the restore. A fresh + provisioning would have stamped it that day. +- **Loki serves the restored store.** Label values came back, and a range query + over an hour that closed before the stamp returned the lines in it. +- **`nflog` and `silences` come back** with their original modification times. +- **Ownership survives.** The volumes came back owned by 65534, 10001 and 472 — + what Prometheus, Loki and Grafana need in order to write to them. + +**What it found, which per-volume testing had not.** + +- **Grafana would not start without internet access.** `GF_INSTALL_PLUGINS` + made the background installer contact `grafana.com` on every start, and a + failure there was fatal — Grafana crash-looped, even though both plugins were + already present in the restored volume. On a host that has lost its uplink, + which is a perfectly ordinary disaster, the restore succeeded and Grafana + still would not come up. Neither plugin was used by any dashboard, and one of + them was an Angular plugin this Grafana refuses to load anyway, so the + declaration was removed rather than repaired. Grafana's own bundled apps ship + inside the image and need no network, which is why an offline start works now + — verified on both fresh and restored volumes. +- **Alloy replays, and now there is a number for it.** Restoring `alloy-data` + put the log positions back to their offsets at the stamp, and Alloy re-read + from there and re-shipped the lines with their original timestamps — about + 4,600 duplicates inside the hour before the stamp, within four minutes of + start-up, after which the count held steady across three samples. The restored + lines themselves never moved. Duplicates in Loki after a restore are expected + behaviour, not a symptom. + +**What is still not proven.** The rehearsal ran under a scratch project name +with an overlay, and it used `docker compose up -d` rather than `make up` — +`make up` renders config and hot-reloads, and takes no overlay. Restoring **in +place, over the live project, and then running `make up`** has still never been +done. That is a smaller gap than the one this closed, but it is not zero: it is +the difference between "the data restores and a stack runs on it" and "this +stack, on this host, comes back". + +## Rehearsing a restore without touching the live stack + +Both scripts honour `COMPOSE_PROJECT_NAME` the way compose itself does, so a set +can be restored into a scratch project and started beside the live one. Write +this overlay — `container_name` is not namespaced by project, so without it +every service collides with the running stack: + +```yaml +services: + prometheus: { container_name: rehearse-prometheus } + alertmanager: { container_name: rehearse-alertmanager } + loki: { container_name: rehearse-loki } + snmp-exporter: { container_name: rehearse-snmp-exporter } + alloy: { container_name: rehearse-alloy } + renderer: { container_name: rehearse-renderer } + grafana: + container_name: rehearse-grafana + environment: + GF_INSTALL_PLUGINS: "" +networks: + observability: + internal: true +``` + +`internal: true` is the safety argument, not a detail. Without it the rehearsal +Alertmanager sends to the real notification channels and pings the real +dead-man's-switch heartbeat, and the rehearsal snmp-exporter polls production +devices. It also means published ports do not route, so reach the services with +`docker exec` rather than from the host — and it is why `GF_INSTALL_PLUGINS` +has to be emptied above. + +```bash +export COMPOSE_PROJECT_NAME=rehearse BIND_ADDR=127.0.0.1 \ + PROMETHEUS_PORT=19090 ALERTMANAGER_PORT=19093 LOKI_PORT=13100 \ + GRAFANA_PORT=13000 ALLOY_PORT=12346 SYSLOG_PORT=11514 \ + ALLOY_HOSTNAME=rehearse-alloy + +./scripts/restore-volumes.sh --from +docker compose -f stacks/observability/compose.yaml -f rehearse.yaml up -d +# ... run section 4 against it with docker exec ... +docker compose -f stacks/observability/compose.yaml -f rehearse.yaml down --volumes +``` + +Set `ALLOY_HOSTNAME` to something distinct. It is what lets you tell restored +log lines from ones the rehearsal stack produced, which is the whole of check 6b. + +The cheap check, worth running whenever a set is written somewhere new: + +```bash +make restore ARGS="--dry-run --from latest" +``` + +That proves the key on this host decrypts every file in the set, that each +stream survives its gzip CRC end to end, that each unpacks to a complete tar +holding the volume its filename claims, and that the set contains all five +volumes rather than the four somebody noticed were missing later. It proves +nothing about whether the stack runs on the result — for that, rehearse. diff --git a/scripts/backup-volumes.sh b/scripts/backup-volumes.sh new file mode 100755 index 0000000..d030d92 --- /dev/null +++ b/scripts/backup-volumes.sh @@ -0,0 +1,836 @@ +#!/usr/bin/env bash +# +# Quiesce the observability stack, archive its data volumes, encrypt them, and +# prove every archive is readable before calling the run a success. +# +# WHAT THIS REPLACES +# +# `make backup` used to be seven lines inline in the Makefile, and every defect +# in #64 followed from that. It wrote backups/.tar.gz — one fixed name, +# no timestamp, no rotation — and tar truncates at open(2), so a run that failed +# had already destroyed the last good backup: the only way to lose a backup was +# to take one. It hardcoded four volume names and had silently skipped +# alloy-data since Alloy was added. It ran an unpinned `alpine`. It tarred +# /prometheus while Prometheus was writing to it. It verified nothing — tar's +# exit status was the whole of the quality control. And it bind-mounted +# backups/ into a container running as root, so every archive came out +# root-owned and could not be rotated without sudo. +# +# WHY IT STOPS THE STACK +# +# The default is to `docker compose stop` the services that own the volumes, +# archive, then start them again. A copy of a live store is not a backup, it is +# a file that looks like one: +# +# prometheus-data the head block is mmap'd and the WAL is append-only +# mid-record. Prometheus replays and discards a torn tail, +# so this usually survives — but "usually" is not a restore +# procedure. +# grafana-data grafana.db is SQLite. A copy taken mid-transaction, with +# no journal to go with it, is the classic corruption case: +# the file opens, and is quietly missing writes. This is +# the volume a hot copy is most likely to ruin. +# alertmanager-data nflog and silences are snapshots written on the +# maintenance tick or at shutdown. SIGTERM is what makes +# them exist and be current. +# +# A trap restarts whatever was stopped on every exit path, including Ctrl-C and +# an error mid-archive. A backup script must never leave the monitoring stack +# down; that turns a routine job into an outage with nothing left watching. +# +# WHY age AND NOT sops +# +# Every other artefact here is encrypted with sops, and backup-firewall.sh pipes +# straight into it. sops holds the whole document in memory and stores it +# base64-encoded inside YAML: free for a 6 KB config.xml, a gigabyte of RSS and +# a 1.4 GB output file for a 1 GB TSDB. `age -r` streams. The recipient is still +# read from .sops.yaml, so there is still exactly one key — rotating it there +# rotates it here. +# +# The recipient is passed in argv and is therefore visible in `ps`. It is a +# public key; it can only encrypt. Do not "fix" this. +# +# Verification decrypts, so the private key must be on this host. That is no new +# exposure — render-config.sh already needs it — but it is a choice, and it +# forecloses a write-only design where the host can produce backups it cannot +# read. Recorded here so a future reader knows it was decided rather than +# overlooked. +# +# WHY THE OUTPUT IS NOT COMMITTED +# +# backups/ is gitignored, `make validate` asserts nothing under it is tracked +# and CI asserts the same — see the header of scripts/backup-firewall.sh for the +# argument. It applies here with more force: grafana.db carries the admin +# password hash, every API token and every datasource credential. +# +# Usage: +# scripts/backup-volumes.sh quiesce, archive, verify +# scripts/backup-volumes.sh --hot skip the stop; UNPROVEN +# scripts/backup-volumes.sh --list show the sets that exist +# scripts/backup-volumes.sh --inventory print the derived volume table +# scripts/backup-volumes.sh --project print the compose project name +# scripts/backup-volumes.sh --verify-only re-verify the newest set +# scripts/backup-volumes.sh --verify-only --all re-verify every retained set +# scripts/backup-volumes.sh --verify-only --set [--only vol,vol] +# scripts/backup-volumes.sh --prune apply retention only +# +# Environment: +# STACK default observability selects stacks//compose.yaml +# COMPOSE_PROJECT_NAME overrides the volume prefix, as it does for compose +# KEEP default 7 complete sets to retain +# STOP_TIMEOUT default 60 seconds before SIGKILL on stop +# SOPS_AGE_KEY_FILE default ~/.config/sops/age/keys.txt +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +STACK="${STACK:-observability}" +STACK_DIR="${REPO_ROOT}/stacks/${STACK}" +COMPOSE_FILE="${STACK_DIR}/compose.yaml" +OUT_DIR="${REPO_ROOT}/backups/volumes" +SOPS_POLICY="${REPO_ROOT}/.sops.yaml" +AGE_IDENTITY="${SOPS_AGE_KEY_FILE:-${HOME}/.config/sops/age/keys.txt}" +KEEP="${KEEP:-7}" +STOP_TIMEOUT="${STOP_TIMEOUT:-60}" + +# An archive smaller than this is not a backup. Measured on this host: an empty +# volume encrypts to 306 bytes, and the smallest real one (alertmanager-data, +# 12 KB of mostly-sparse nflog and silences) to 872. 512 sits between them, and +# is the same floor backup-firewall.sh:116 uses on a smaller artefact. The +# structural guard is the entry count in verify(); this is belt and braces. +MIN_BYTES=512 + +# The archives are age-encrypted, so this is defence in depth rather than the +# control. grafana-data is in here; it is cheap. +umask 077 + +red() { printf '\033[0;31m%s\033[0m\n' "$*" >&2; } +green() { printf '\033[0;32m%s\033[0m\n' "$*"; } +info() { printf '\033[0;34m--\033[0m %s\n' "$*"; } +warn() { printf '\033[0;33mwarning:\033[0m %s\n' "$*" >&2; } +die() { red "$*"; exit 1; } + +need() { command -v "$1" >/dev/null 2>&1 || die "missing dependency: $1"; } + +# The age recipient is read from .sops.yaml rather than duplicated here. One +# source of truth for the key; rotating it in .sops.yaml rotates it here too. +recipient() { + grep -oE 'age1[0-9a-z]{50,}' "${SOPS_POLICY}" | head -1 +} + +human() { numfmt --to=iec --suffix=B "$1" 2>/dev/null || printf '%sB' "$1"; } + +# --------------------------------------------------------------------------- +# Inventory +# +# The volume list is DERIVED, not written down. alloy-data was missing from the +# old recipe precisely because the list was hardcoded; adding a fifth entry +# would have fixed today's symptom and left the mechanism in place. Text +# parsing rather than PyYAML or `docker compose config`, for the reasons in the +# header of scripts/image-for.sh: this must work before either is guaranteed +# present, and needs no .env for the ${VAR:?} guards. +# +# A named volume is discriminated from a bind mount by its source not starting +# with . or / — which is what compose itself uses. +# --------------------------------------------------------------------------- + +# Volume -> the one entry that proves an archive holds THAT volume. +# +# This is the analogue of backup-firewall.sh's `` grep, and it carries +# more weight than it looks: `age -r` has no associated data, so +# loki-data.tar.gz.age and prometheus-data.tar.gz.age are interchangeable as far +# as age is concerned. The sentinel is the only thing binding a filename to its +# content, so it has to discriminate rather than merely be present. +# +# ./wal is deliberately NOT the sentinel for Prometheus or Loki even though it +# is the most reliably present entry in both — both have it, so a crossed +# mapping, the exact failure this exists to catch, would sail through. +declare -A SENTINEL=( + [prometheus-data]="./chunks_head" + [loki-data]="./chunks" + [grafana-data]="./grafana.db" + [alertmanager-data]="./nflog" + [alloy-data]="./alloy_seed.json" +) + +# Reported when absent, never fatal. These cover the fresh-volume case, where +# the discriminating entry above may not exist yet. +declare -A COMPANIONS=( + [prometheus-data]="./wal ./lock ./queries.active" + [loki-data]="./wal ./index ./compactor" + [grafana-data]="./plugins ./dashboards ./png" + [alertmanager-data]="./silences" + [alloy-data]="./remotecfg" +) + +VOLUMES=() +SERVICES=() +declare -A VOL_SERVICE=() +declare -A VOL_MOUNT=() +PROJECT="" + +parse_compose() { + awk ' + /^services:/ { in_services = 1; in_volumes = 0; next } + /^volumes:/ { in_services = 0; in_volumes = 1; next } + /^[^[:space:]#]/ { in_services = 0; in_volumes = 0 } + + /^name:/ && !printed_name { printf "project\t%s\n", $2; printed_name = 1 } + + in_volumes && /^ [A-Za-z0-9_.-]+:/ { + n = $0; sub(/^ /, "", n); sub(/:.*/, "", n) + printf "declared\t%s\n", n + next + } + + in_services && /^ [A-Za-z0-9_.-]+:/ { + s = $0; sub(/^ /, "", s); sub(/:.*/, "", s) + svc = s; inlist = 0 + next + } + in_services && /^ volumes:[[:space:]]*$/ { inlist = 1; next } + in_services && /^ [A-Za-z_<]/ { inlist = 0 } + in_services && inlist && /^ - / { + if ($2 ~ /^[.\/]/) next + split($2, p, ":") + if (p[1] == "" || p[2] == "") next + printf "mount\t%s\t%s\t%s\n", p[1], svc, p[2] + } + ' "${COMPOSE_FILE}" +} + +load_inventory() { + [[ -f ${COMPOSE_FILE} ]] || die "no compose file at ${COMPOSE_FILE}" + + local -a declared=() + local kind a b c + while IFS=$'\t' read -r kind a b c; do + case "${kind}" in + project) PROJECT="${a}" ;; + declared) declared+=("${a}") ;; + mount) + [[ -z ${VOL_SERVICE[$a]:-} ]] \ + || die "volume ${a} is mounted by both ${VOL_SERVICE[$a]} and ${b} — this script cannot say which service to stop" + VOL_SERVICE["${a}"]="${b}" + VOL_MOUNT["${a}"]="${c}" + ;; + esac + done < <(parse_compose) + + # The project name is what prefixes the volumes, and it is NOT $STACK. STACK + # is a directory name; this host still carries orphan prometheus_grafana-data + # and prometheus_loki-data volumes from when the two diverged. + # + # COMPOSE_PROJECT_NAME wins over the file's name: key, because that is the + # order docker compose itself resolves them in. Getting this backwards would + # archive one project's volumes while compose ran another's — and the restore + # would then overwrite the wrong ones. It is also what makes a rehearsal + # possible: COMPOSE_PROJECT_NAME=restoretest restores a set into a scratch + # stack instead of over the live one. See docs/runbooks/restore-the-stack.md. + PROJECT="${COMPOSE_PROJECT_NAME:-${PROJECT}}" + [[ -n ${PROJECT} ]] || die "no top-level name: in ${COMPOSE_FILE} — cannot derive the volume prefix" + ((${#declared[@]} > 0)) || die "no named volumes declared in ${COMPOSE_FILE}" + + local v + for v in "${declared[@]}"; do + # A volume nothing mounts is a volume this script cannot attribute, and a + # volume it cannot attribute is one it would silently skip. That is exactly + # how alloy-data went missing. + [[ -n ${VOL_SERVICE[$v]:-} ]] \ + || die "volume ${v} is declared in ${COMPOSE_FILE} but no service mounts it — refusing to run" + # Derivation solves one inventory; the sentinel table is a second one. + # Making its absence fatal means adding a sixth volume produces a named + # error rather than an archive nothing can verify. + [[ -n ${SENTINEL[$v]:-} ]] \ + || die "no sentinel defined for ${v} in $(basename "$0") — add one; this script will not write a backup it cannot verify" + VOLUMES+=("${v}") + done + + mapfile -t SERVICES < <(printf '%s\n' "${VOL_SERVICE[@]}" | sort -u) +} + +print_inventory() { + local v + for v in "${VOLUMES[@]}"; do + printf '%s\t%s\t%s\n' "${v}" "${VOL_SERVICE[$v]}" "${VOL_MOUNT[$v]}" + done +} + +# --------------------------------------------------------------------------- +# Sets +# --------------------------------------------------------------------------- + +# A set is complete when its MANIFEST exists; the manifest is written last. +# Sorted by name, not mtime — mtime can be touched, and the stamp is UTC +# ISO-8601 basic form, so a name sort IS a chronological sort. +complete_sets() { + find "${OUT_DIR}" -mindepth 2 -maxdepth 2 -name MANIFEST -printf '%h\n' 2>/dev/null | sort -r +} + +all_sets() { + find "${OUT_DIR}" -mindepth 1 -maxdepth 1 -type d -name '2*' -printf '%p\n' 2>/dev/null | sort -r +} + +newest_complete() { complete_sets | head -1; } + +manifest_field() { + awk -F'\t' -v k="$2" '$1 == k { print $2; exit }' "$1/MANIFEST" 2>/dev/null +} + +newest_quiesced() { + local d + while read -r d; do + [[ -n ${d} ]] || continue + [[ "$(manifest_field "${d}" mode)" == quiesced ]] && { printf '%s\n' "${d}"; return 0; } + done < <(complete_sets) + return 0 +} + +list_sets() { + local d state mode count n=0 + if [[ ! -d ${OUT_DIR} ]] || [[ -z "$(all_sets)" ]]; then + info "no sets in ${OUT_DIR}" + return 0 + fi + while read -r d; do + [[ -n ${d} ]] || continue + if [[ -f ${d}/MANIFEST ]]; then + state=complete + mode="$(manifest_field "${d}" mode)" + # A complete set whose manifest will not parse is not a proven-quiesced + # set. Say so rather than printing a blank column. + [[ -n ${mode} ]] || mode=unreadable + else + state=INCOMPLETE + mode=unknown + fi + count=$(find "${d}" -maxdepth 1 -name '*.tar.gz.age' | wc -l) + printf '%s\t%s\t%s\t%s archive(s)\n' "$(basename "${d}")" "${state}" "${mode}" "${count}" + n=$((n + 1)) + done < <(all_sets) + info "${n} set(s), keeping ${KEEP}" +} + +# --------------------------------------------------------------------------- +# Verification — the tarball analogue of backup-firewall.sh:62-76 +# +# Three escalating assertions, one streaming pass, nothing extracted: +# (a) it decrypts at all. age's STREAM construction is AEAD per chunk with a +# final-chunk flag, so this also rejects a truncated, bit-flipped or +# tampered file — strictly stronger than the sops case. +# (b) it is the right KIND of thing: the whole gzip stream parses as a tar +# (CRC32 and ISIZE are checked at end of stream) and the volume's own +# sentinel is present. +# (c) report semantic content back to the operator — the analogue of +# backup-firewall.sh printing the config version and rule count. +# --------------------------------------------------------------------------- +verify() { + local f="$1" vol="$2" lenient="${3:-0}" + local bytes out entries found foreign missing must companions all v + + [[ -f ${f} ]] || { red "no such archive: ${f}"; return 1; } + + bytes=$(stat -c %s "${f}") + if ((bytes < MIN_BYTES)); then + red "$(basename "${f}") is ${bytes} bytes — implausibly small; that is not a backup" + return 1 + fi + + must="${SENTINEL[$vol]:-}" + companions="${COMPANIONS[$vol]:-}" + [[ -n ${must} ]] || { red "no sentinel for ${vol}"; return 1; } + + # Every volume's sentinel is handed to awk, not just this one's. Finding + # somebody else's is how a crossed mapping is caught, and that has to be fatal + # in both modes — see below. + all="" + for v in "${!SENTINEL[@]}"; do all+=" ${v}|${SENTINEL[$v]}"; done + + # The listing is NOT piped through head, grep -q or grep -m1. Any reader that + # exits early SIGPIPEs tar, tar dies on signal 13, the shell reports 141, and + # `set -o pipefail` turns a perfectly good archive into a failed verification + # — a backup script reporting corruption it invented. awk consumes every line + # to EOF and does the matching itself. + # + # Whole-line comparison rather than $NF, and -tzf rather than -tzvf, because + # Grafana's plugin and dashboard trees contain filenames with spaces. Matching + # whole lines also means only top-level entries can satisfy a sentinel. + if ! out="$(age --decrypt -i "${AGE_IDENTITY}" "${f}" \ + | tar -tzf - \ + | awk -v vol="${vol}" -v all="${all# }" -v companions="${companions}" ' + BEGIN { + n = split(all, pairs, " ") + for (i = 1; i <= n; i++) { + split(pairs[i], kv, "|") + owner[kv[2]] = kv[1] + } + m = split(companions, c, " ") + for (i = 1; i <= m; i++) want[c[i]] = 1 + } + { + entries++ + line = $0 + sub(/\/$/, "", line) # tar suffixes directories + if (line in owner) hit[owner[line]] = 1 + if (line in want) seen[line] = 1 + } + END { + foreign = "" + for (v in hit) if (v != vol) foreign = foreign " " v + missing = "" + for (k in want) if (!(k in seen)) missing = missing " " k + # "-" rather than "" for an empty list. Tab is an IFS + # WHITESPACE character, so bash collapses a run of them + # into one delimiter — an empty field here silently + # shifts every later field left, and `missing` arrives in + # `foreign` as a phantom crossed-mapping report. + if (foreign == "") foreign = "-" + if (missing == "") missing = "-" + printf "%d\t%d\t%s\t%s\n", entries, (vol in hit), foreign, missing + }')"; then + red "FAILED to decrypt or read $(basename "${f}")" + return 1 + fi + + IFS=$'\t' read -r entries found foreign missing <<<"${out}" + + if ((entries <= 1)); then + red "$(basename "${f}") unpacks to ${entries} entries — that is an empty archive, not a backup" + return 1 + fi + + # Fatal in BOTH modes. --hot forgives a sentinel that is merely absent, but an + # archive carrying another volume's sentinel is not a fresh volume — it is the + # wrong file under this name. age -r has no associated data, so nothing but + # this check binds a filename to its content. + if [[ ${foreign} != "-" ]]; then + red "${vol}: this archive carries the sentinel of${foreign} — it is not a ${vol} backup" + return 1 + fi + + if ((found == 0)); then + if ((lenient)); then + # --hot only. alertmanager-data has no entry that survives a fresh, + # never-cleanly-stopped Alertmanager: nflog and silences are snapshots + # written on the maintenance tick or at SIGTERM — and SIGTERM is exactly + # what quiescing sends. So this falls out of the mode rather than needing + # a per-volume exception. + warn "${vol}: ${must} is not in the archive. A hot copy of a service that has never shut down cleanly may legitimately lack it, but nothing here proves this archive is ${vol}." + warn "checked $(basename "${f}") — ${entries} entries, $(human "${bytes}"), ${must} MISSING" + else + red "${vol}: ${must} is not in the archive — this does not look like a ${vol} backup" + return 1 + fi + else + green "verified $(basename "${f}") — ${entries} entries, $(human "${bytes}"), ${must} present" + fi + + [[ ${missing} != "-" ]] && info " not present: ${missing# }" + return 0 +} + +verify_set() { + local d="$1"; shift + local lenient=0 vol failed=0 + local -a want=("$@") + ((${#want[@]})) || want=("${VOLUMES[@]}") + [[ -d ${d} ]] || die "no such set: ${d}" + [[ "$(manifest_field "${d}" mode)" == hot ]] && lenient=1 + info "verifying $(basename "${d}")" + for vol in "${want[@]}"; do + if [[ ! -f ${d}/${vol}.tar.gz.age ]]; then + red "${vol}: no archive in $(basename "${d}")" + failed=1 + continue + fi + verify "${d}/${vol}.tar.gz.age" "${vol}" "${lenient}" || failed=1 + done + return "${failed}" +} + +# --------------------------------------------------------------------------- +# Quiesce +# --------------------------------------------------------------------------- +STOPPED=() + +quiesce() { + # Only the services that are actually running. If prometheus is deliberately + # down for maintenance, this must not quietly bring it back up. + mapfile -t STOPPED < <( + "${COMPOSE[@]}" ps --status running --services 2>/dev/null \ + | grep -Fx -f <(printf '%s\n' "${SERVICES[@]}") || true + ) + if ((${#STOPPED[@]} == 0)); then + info "nothing to stop — the stack is already down" + return 0 + fi + + # -t 60, not the 10s default. Prometheus flushing its head block and + # Alertmanager writing its nflog and silences snapshots are exactly what + # quiescing is for; a SIGKILL at ten seconds skips both and leaves the WAL to + # carry state the archive was meant to capture cleanly. + info "stopping ${STOPPED[*]} (timeout ${STOP_TIMEOUT}s)" + "${COMPOSE[@]}" stop -t "${STOP_TIMEOUT}" "${STOPPED[@]}" +} + +cleanup() { + local rc=$? + ((${#STOPPED[@]})) || return "${rc}" + info "restarting ${STOPPED[*]}" + # `start`, not `up -d`: the containers still exist, and up -d would recreate + # them from a compose file that may need a rendered .env. errexit is off + # inside a trap on purpose — the restart must be attempted whatever failed. + if ! "${COMPOSE[@]}" start "${STOPPED[@]}" >/dev/null 2>&1 \ + && ! "${COMPOSE[@]}" up -d "${STOPPED[@]}" >/dev/null 2>&1; then + red "############################################################" + red "THE STACK IS DOWN. backup-volumes.sh stopped these services" + red "and could not start them again:" + red " ${STOPPED[*]}" + red "Run: make up" + red "############################################################" + STOPPED=() + return 1 + fi + STOPPED=() + return "${rc}" +} + +# --------------------------------------------------------------------------- +# Archiving +# --------------------------------------------------------------------------- +TORN=() + +archive_one() { + local vol="$1" out="$2" tar_rc age_rc + local -a rcs=() + + # tar writes to stdout and age writes the file as the operator. The plaintext + # never touches this disk — the property backup-firewall.sh gets by piping ssh + # into sops — and backups/ is never bind-mounted into a container, which is + # what used to make every archive root-owned and unrotatable. + # + # :ro on the source: a backup must not be able to write to the thing it is + # backing up. --network none and --read-only because tar to stdout needs + # neither. --numeric-owner because these volumes belong to uids the container + # has no names for (65534, 10001, 472) and the restore path needs them back + # verbatim. + # + # errexit is lifted for the pipeline because BOTH statuses are needed, and + # `rc=$?` afterwards would clobber PIPESTATUS. + set +e + docker run --rm --network none --read-only \ + --security-opt no-new-privileges \ + -v "${PROJECT}_${vol}:/data:ro" \ + "${TAR_IMAGE}" \ + tar --numeric-owner -czf - -C /data . \ + | age --recipient "${AGE_RECIPIENT}" --output "${out}" + # Copied in one go: reading PIPESTATUS is itself a command, and the first + # assignment would reset it before the second could see index 1. + rcs=("${PIPESTATUS[@]}") + set -e + tar_rc="${rcs[0]}"; age_rc="${rcs[1]}" + + ((age_rc == 0)) || { red "${vol}: age failed (rc ${age_rc})"; return 1; } + + case "${tar_rc}" in + 0) ;; + # GNU tar exits 1 — not 2 — for "file changed as we read it", which is + # precisely the hot-copy hazard. BusyBox tar cannot report it at all, which + # is why the archiver image is Debian; see compose.yaml. + 1) + if ((HOT)); then + warn "${vol}: a file changed while tar was reading it — this archive is torn" + TORN+=("${vol}") + else + red "${vol}: a file changed while tar was reading it, with the service stopped — something else is writing to this volume" + return 1 + fi + ;; + *) red "${vol}: tar failed (rc ${tar_rc})"; return 1 ;; + esac + return 0 +} + +# --------------------------------------------------------------------------- +# Retention +# --------------------------------------------------------------------------- +prune() { + local -a sets=() + mapfile -t sets < <(complete_sets) + + local d incomplete + incomplete=$(comm -23 <(all_sets | sort) <(printf '%s\n' "${sets[@]}" | sort) | wc -l) + if ((incomplete > 0)); then + # Reported, never deleted. Deleting data on a failure path is the exact bug + # class #64 is about. + warn "${incomplete} incomplete set(s) in ${OUT_DIR} — a failed run left these. Inspect, then remove them by hand." + fi + + if ((${#sets[@]} <= KEEP)); then + info "${#sets[@]} complete set(s), keeping ${KEEP} — nothing to prune" + return 0 + fi + + local keep_quiesced + keep_quiesced="$(newest_quiesced)" + + for d in "${sets[@]:KEEP}"; do + # Never construct an rm -rf target from an unvalidated variable. + if [[ -z ${d} || ${d} != "${OUT_DIR}/"[0-9]* || ! -f ${d}/MANIFEST ]]; then + red "refusing to prune ${d}" + continue + fi + # A run of --hot backups must not evict the last archive anyone has actually + # proven restorable. + if [[ ${d} == "${keep_quiesced}" ]]; then + info "keeping $(basename "${d}") — the newest quiesced set" + continue + fi + info "pruning $(basename "${d}")" + rm -rf -- "${d}" + done +} + +# --------------------------------------------------------------------------- +# Arguments +# +# --hot is a modifier that still runs the main path, so a bare case on $1 will +# not do. The loop keeps the recognisable shape — an explicit "" arm and a +# rejecting * arm — and refuses a second mode rather than letting the last one +# win silently. +# --------------------------------------------------------------------------- +usage() { sed -n 's|^# \{0,1\}||; /^Usage:/,/^$/p' "$0" | head -20; } + +MODE=backup +MODE_SET="" +HOT=0 +ALL=0 +SET_ARG="" +ONLY="" + +set_mode() { + [[ -z ${MODE_SET} ]] || die "conflicting modes: --${MODE_SET} and --$1" + MODE="$1"; MODE_SET="$1" +} + +while (($#)); do + case "$1" in + --hot) HOT=1 ;; + --all) ALL=1 ;; + --set) SET_ARG="${2:?--set needs a stamp}"; shift ;; + --only) ONLY="${2:?--only needs a comma-separated volume list}"; shift ;; + --list) set_mode list ;; + --inventory) set_mode inventory ;; + --project) set_mode project ;; + --verify-only) set_mode verify ;; + --prune) set_mode prune ;; + -h|--help) usage; exit 0 ;; + "") ;; + *) die "unknown argument: $1" ;; + esac + shift +done + +COMPOSE=(docker compose -f "${COMPOSE_FILE}") + +load_inventory + +# Resolved after the inventory so a typo is checked against the real volume list +# rather than silently restoring nothing. +ONLY_VOLUMES=() +if [[ -n ${ONLY} ]]; then + IFS=',' read -r -a ONLY_VOLUMES <<<"${ONLY}" + for v in "${ONLY_VOLUMES[@]}"; do + [[ -n ${SENTINEL[$v]:-} && -n ${VOL_SERVICE[$v]:-} ]] \ + || die "unknown volume: ${v} (have: ${VOLUMES[*]})" + done +fi + +case "${MODE}" in + inventory) + print_inventory + exit 0 + ;; + # The compose project name is what prefixes the volumes. Exposed so + # restore-volumes.sh does not have to derive it a second time. + project) + printf '%s\n' "${PROJECT}" + exit 0 + ;; + list) + list_sets + exit 0 + ;; +esac + +need age +[[ -f ${AGE_IDENTITY} ]] \ + || die "no age identity at ${AGE_IDENTITY} — verification decrypts what it just wrote, and an unverified backup is not a backup" + +case "${MODE}" in + verify) + failed=0 + if ((ALL)); then + mapfile -t targets < <(complete_sets) + elif [[ -n ${SET_ARG} ]]; then + targets=("${OUT_DIR}/${SET_ARG}") + else + mapfile -t targets < <(newest_complete) + fi + if ((${#targets[@]} == 0)) || [[ -z ${targets[0]} ]]; then + die "no complete sets in ${OUT_DIR}" + fi + for t in "${targets[@]}"; do + verify_set "${t}" "${ONLY_VOLUMES[@]}" || failed=1 + done + ((failed == 0)) || die "verification FAILED" + exit 0 + ;; + prune) + prune + exit 0 + ;; +esac + +# --------------------------------------------------------------------------- +# Main path +# +# Everything that can fail is made to fail BEFORE anything is stopped. +# --------------------------------------------------------------------------- +need docker +need tar +need awk +need flock +need numfmt +docker info >/dev/null 2>&1 || die "cannot reach the docker daemon" + +mkdir -p "${OUT_DIR}" + +# Without this a cron run and a manual run overlap: one stops the stack while +# the other is mid-archive, and the first to finish restarts it under the second. +exec 9>"${OUT_DIR}/.lock" +flock -n 9 || die "another $(basename "$0") is already running" + +AGE_RECIPIENT="$(recipient)" +[[ -n ${AGE_RECIPIENT} ]] || die "no age recipient found in ${SOPS_POLICY}" + +"${COMPOSE[@]}" config -q >/dev/null 2>&1 \ + || die "docker compose config failed — the \${VAR:?} guards need a rendered .env. Run: make render" + +# `docker run -v missing_volume:/data` CREATES an empty volume, so a wrong +# prefix produces five plausible-looking 45-byte archives and exit 0. This host +# already carries orphan prometheus_* volumes from an older project name. +for v in "${VOLUMES[@]}"; do + docker volume inspect "${PROJECT}_${v}" >/dev/null 2>&1 \ + || die "no volume ${PROJECT}_${v} — the stack has not been started under project '${PROJECT}', or it was created under a different one (docker volume ls)" +done + +TAR_IMAGE="$("${REPO_ROOT}/scripts/image-for.sh" archiver)" +docker image inspect "${TAR_IMAGE}" >/dev/null 2>&1 || { + info "pulling ${TAR_IMAGE}" + docker pull -q "${TAR_IMAGE}" >/dev/null +} + +# The sizing pass doubles as proof the archiver image actually runs, before the +# stack goes down. This is what stops the classic "stack is down, and now the +# registry is unreachable". +info "sizing ${#VOLUMES[@]} volume(s)" +total_kb=0 +for v in "${VOLUMES[@]}"; do + kb="$(docker run --rm --network none -v "${PROJECT}_${v}:/data:ro" "${TAR_IMAGE}" \ + du -sk /data | awk '{print $1}')" + total_kb=$((total_kb + kb)) +done +avail_kb="$(df --output=avail -k "${OUT_DIR}" | tail -1 | tr -d ' ')" +if ((avail_kb < total_kb * 11 / 10)); then + die "only $(human $((avail_kb * 1024))) free at ${OUT_DIR}, and the volumes hold $(human $((total_kb * 1024))) uncompressed — refusing to start" +fi + +STAMP="$(date -u +%Y%m%dT%H%M%SZ)" +SET_DIR="${OUT_DIR}/${STAMP}" +# No -p: a duplicate stamp is a named error, not a merge. +mkdir "${SET_DIR}" || die "set ${STAMP} already exists" + +if ((HOT)); then + warn "--hot: the stack keeps running, so nothing here is proven restorable." + warn "grafana.db in particular is SQLite; a copy taken mid-transaction, with no" + warn "journal to go with it, opens fine and is silently missing writes." +else + # EXIT alone does not cover an uncaught SIGINT. Turning the signal into a + # normal exit is what guarantees the EXIT trap runs exactly once, on every + # path a backup can die on. SIGKILL cannot be trapped, and note that + # `restart: unless-stopped` does NOT help — a container stopped by + # `docker compose stop` is one Docker has been told to leave alone. + trap cleanup EXIT + trap 'exit 130' INT + trap 'exit 143' TERM +fi + +started=${SECONDS} +((HOT)) || quiesce +downtime_start=${SECONDS} + +failed=0 +declare -A BYTES=() SHA=() +for v in "${VOLUMES[@]}"; do + info "archiving ${v} (${VOL_SERVICE[$v]}${VOL_MOUNT[$v]})" + part="${SET_DIR}/${v}.tar.gz.age.part" + if ! archive_one "${v}" "${part}"; then + rm -f "${part}" + failed=1 + continue + fi + if ! verify "${part}" "${v}" "${HOT}"; then + failed=1 + continue + fi + BYTES["${v}"]="$(stat -c %s "${part}")" + SHA["${v}"]="$(sha256sum "${part}" | awk '{print $1}')" + mv "${part}" "${SET_DIR}/${v}.tar.gz.age" +done + +downtime=$((SECONDS - downtime_start)) + +# Restart before writing the manifest: the stack matters more than the paperwork. +if ((HOT == 0)); then + trap - EXIT INT TERM + cleanup || failed=1 +fi + +if ((failed)); then + red "one or more volumes failed — no manifest written, nothing pruned" + red "the incomplete set is at ${SET_DIR}" + exit 1 +fi + +# Written last: its presence is what marks the set complete. Extension-free on +# purpose — .yamllint.yaml has no backups/ ignore and markdownlint globs +# **/*.md, so MANIFEST.yaml or MANIFEST.md would be linted by `make lint`. +{ + printf '# %s set %s\n' "$(basename "$0")" "${STAMP}" + printf 'stack\t%s\n' "${STACK}" + printf 'project\t%s\n' "${PROJECT}" + printf 'mode\t%s\n' "$( ((HOT)) && echo hot || echo quiesced )" + # Which key is needed to read this set, and so which sets survived a rotation. + printf 'recipient\t%s\n' "${AGE_RECIPIENT}" + printf 'archiver\t%s\n' "${TAR_IMAGE}" + printf 'downtime\t%s\n' "${downtime}" + ((${#TORN[@]})) && printf 'torn\t%s\n' "${TORN[*]}" + printf '#volume\tservice\tmount\tbytes\tsha256\n' + for v in "${VOLUMES[@]}"; do + printf '%s\t%s\t%s\t%s\t%s\n' \ + "${v}" "${VOL_SERVICE[$v]}" "${VOL_MOUNT[$v]}" "${BYTES[$v]}" "${SHA[$v]}" + done +} > "${SET_DIR}/MANIFEST" + +prune + +set_bytes=0 +for v in "${VOLUMES[@]}"; do set_bytes=$((set_bytes + BYTES[$v])); done + +printf '\n' +green "wrote ${SET_DIR#"${REPO_ROOT}"/} — ${#VOLUMES[@]} volumes, $(human "${set_bytes}"), $( ((HOT)) && echo "stack never stopped (UNPROVEN)" || echo "stack down ${downtime}s" ), $((SECONDS - started))s total" +printf '\n' +info "This is on the same host as everything it protects." +info "Copy the set to the backup target and offsite — see docs/roadmap.md #92." +info "Nothing runs this on a timer yet — docs/roadmap.md #77." +info "Restoring it: docs/runbooks/restore-the-stack.md" diff --git a/scripts/restore-volumes.sh b/scripts/restore-volumes.sh new file mode 100755 index 0000000..a5e6b77 --- /dev/null +++ b/scripts/restore-volumes.sh @@ -0,0 +1,403 @@ +#!/usr/bin/env bash +# +# Put a backup set back into the stack's data volumes. +# +# This is the destructive half of scripts/backup-volumes.sh, and it is a +# separate script on purpose: one program that both writes archives and +# overwrites live volumes is one mistyped flag away from an outage, and +# scripts/backup-firewall.sh — the model for both — is non-destructive +# throughout. The volume inventory is not duplicated either; it comes from +# `backup-volumes.sh --inventory`, the way every SNMP tool reads +# scripts/snmp-targets.sh. +# +# THE ORDER MATTERS MORE THAN ANYTHING ELSE HERE +# +# Every selected archive is decrypted and read end to end BEFORE a single byte +# of live data is removed. A restore that destroys the current volume and then +# discovers the backup will not decrypt has taken a recoverable situation and +# made it final. That is the one property this script exists to guarantee, and +# --dry-run runs exactly that phase and stops. +# +# WHY THERE IS NO DEFAULT SET +# +# --from is mandatory. docs/runbooks/restore-the-firewall.md already states the +# principle: "a config that predates the change you are trying to recover from +# restores you into the problem". Here it inverts — the NEWEST set is the one +# most likely to contain the corruption you are recovering from. A bad Grafana +# upgrade at 22:00 and a backup at 03:00 means the newest grafana.db is the +# broken one. `--from latest` exists, resolves the stamp, prints it and its age, +# and still asks you to confirm against the resolved stamp, so "newest" is +# something you chose rather than something that happened to you. +# +# WHY IT DOES NOT RESTART THE STACK +# +# It stops the stack and leaves it stopped. Bringing it back up is `make up`, +# followed by section 4 of the runbook. A script that restarts the stack itself +# invites reading "it came back up" as "the restore worked", and those are +# different claims — the dashboards, rules and datasources all come from git and +# render perfectly over a volume that never came back. +# +# Usage: +# scripts/restore-volumes.sh --list +# scripts/restore-volumes.sh --dry-run --from verify, touch nothing +# scripts/restore-volumes.sh --from restore the whole set +# scripts/restore-volumes.sh --from --only grafana-data[,loki-data] +# scripts/restore-volumes.sh --from latest +# +# Options: +# --no-safety-net skip the pre-restore snapshot of the current contents +# +# Environment: +# STACK default observability +# COMPOSE_PROJECT_NAME overrides the volume prefix, as it does for compose. +# This is how a set is rehearsed into a scratch stack +# rather than restored over the live one — see the last +# section of docs/runbooks/restore-the-stack.md. +# SOPS_AGE_KEY_FILE default ~/.config/sops/age/keys.txt +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +STACK="${STACK:-observability}" +STACK_DIR="${REPO_ROOT}/stacks/${STACK}" +COMPOSE_FILE="${STACK_DIR}/compose.yaml" +COMPOSE=(docker compose -f "${COMPOSE_FILE}") +OUT_DIR="${REPO_ROOT}/backups/volumes" +SOPS_POLICY="${REPO_ROOT}/.sops.yaml" +AGE_IDENTITY="${SOPS_AGE_KEY_FILE:-${HOME}/.config/sops/age/keys.txt}" +BACKUP="${REPO_ROOT}/scripts/backup-volumes.sh" + +umask 077 + +red() { printf '\033[0;31m%s\033[0m\n' "$*" >&2; } +green() { printf '\033[0;32m%s\033[0m\n' "$*"; } +info() { printf '\033[0;34m--\033[0m %s\n' "$*"; } +warn() { printf '\033[0;33mwarning:\033[0m %s\n' "$*" >&2; } +die() { red "$*"; exit 1; } + +need() { command -v "$1" >/dev/null 2>&1 || die "missing dependency: $1"; } + +human() { numfmt --to=iec --suffix=B "$1" 2>/dev/null || printf '%sB' "$1"; } + +recipient() { grep -oE 'age1[0-9a-z]{50,}' "${SOPS_POLICY}" | head -1; } + +manifest_field() { + awk -F'\t' -v k="$2" '$1 == k { print $2; exit }' "$1/MANIFEST" 2>/dev/null +} + +manifest_sha() { + awk -F'\t' -v v="$2" '$1 == v { print $5; exit }' "$1/MANIFEST" 2>/dev/null +} + +manifest_bytes() { + awk -F'\t' -v v="$2" '$1 == v { print $4; exit }' "$1/MANIFEST" 2>/dev/null +} + +# 20260829T062231Z -> "3 day(s) old". Best-effort: a set whose name will not +# parse is still restorable, it just does not get an age printed next to it. +stamp_age() { + local s="$1" epoch now + epoch="$(date -u -d "${s:0:4}-${s:4:2}-${s:6:2} ${s:9:2}:${s:11:2}:${s:13:2} UTC" +%s 2>/dev/null)" || return 0 + [[ -n ${epoch} ]] || return 0 + now="$(date +%s)" + printf '%s day(s) old' "$(( (now - epoch) / 86400 ))" +} + +# The uid each service runs as, and therefore the uid its volume's contents must +# come back owned by. A restore that returns every byte and leaves Grafana +# unable to write its own database is a restore that failed. +declare -A EXPECT_UID=( + [prometheus-data]=65534 + [loki-data]=10001 + [grafana-data]=472 +) + +FROM="" +ONLY="" +SNAP_DIR="" +DRY=0 +SAFETY=1 +MODE=restore + +while (($#)); do + case "$1" in + --from) FROM="${2:?--from needs a stamp, or 'latest'}"; shift ;; + --only) ONLY="${2:?--only needs a comma-separated volume list}"; shift ;; + --dry-run) DRY=1 ;; + --no-safety-net) SAFETY=0 ;; + --list) MODE=list ;; + -h|--help) sed -n 's|^# \{0,1\}||; /^Usage:/,/^set -euo/p' "$0" | sed '$d'; exit 0 ;; + "") ;; + *) die "unknown argument: $1" ;; + esac + shift +done + +if [[ ${MODE} == list ]]; then + exec "${BACKUP}" --list +fi + +need docker +need age +need tar +need numfmt +need sha256sum +docker info >/dev/null 2>&1 || die "cannot reach the docker daemon" +[[ -f ${AGE_IDENTITY} ]] || die "no age identity at ${AGE_IDENTITY} — the archives cannot be read without it" + +[[ -n ${FROM} ]] || die "--from is required. Run --list to see the sets, and read docs/runbooks/restore-the-stack.md before picking the newest one by reflex." + +# --------------------------------------------------------------------------- +# Inventory, from backup-volumes.sh rather than restated here +# --------------------------------------------------------------------------- +VOLUMES=() +declare -A VOL_SERVICE=() +while IFS=$'\t' read -r v s _; do + [[ -n ${v} ]] || continue + VOLUMES+=("${v}") + VOL_SERVICE["${v}"]="${s}" +done < <("${BACKUP}" --inventory) +((${#VOLUMES[@]})) || die "could not read the volume inventory from ${BACKUP}" + +PROJECT="$("${BACKUP}" --project)" +[[ -n ${PROJECT} ]] || die "could not read the compose project name" + +TARGETS=() +if [[ -n ${ONLY} ]]; then + IFS=',' read -r -a TARGETS <<<"${ONLY}" + for v in "${TARGETS[@]}"; do + [[ -n ${VOL_SERVICE[$v]:-} ]] || die "unknown volume: ${v} (have: ${VOLUMES[*]})" + done +else + TARGETS=("${VOLUMES[@]}") +fi + +# --------------------------------------------------------------------------- +# Resolve the set +# --------------------------------------------------------------------------- +if [[ ${FROM} == latest ]]; then + SET_DIR="$(find "${OUT_DIR}" -mindepth 2 -maxdepth 2 -name MANIFEST -printf '%h\n' 2>/dev/null | sort -r | head -1)" + [[ -n ${SET_DIR} ]] || die "no complete sets in ${OUT_DIR}" + STAMP="$(basename "${SET_DIR}")" + info "--from latest resolves to ${STAMP}" +else + STAMP="${FROM}" + SET_DIR="${OUT_DIR}/${STAMP}" +fi + +[[ -d ${SET_DIR} ]] || die "no such set: ${SET_DIR}" +[[ -f ${SET_DIR}/MANIFEST ]] \ + || die "${STAMP} has no MANIFEST — it is an incomplete set left by a failed backup, and this script will not restore from one" + +SET_MODE="$(manifest_field "${SET_DIR}" mode)" +SET_RECIPIENT="$(manifest_field "${SET_DIR}" recipient)" +ARCHIVER="$(manifest_field "${SET_DIR}" archiver)" + +if [[ -n ${SET_RECIPIENT} && ${SET_RECIPIENT} != "$(recipient)" ]]; then + warn "this set was encrypted to ${SET_RECIPIENT}, which is not the recipient currently in .sops.yaml." + warn "the key has been rotated since. Decryption below will tell you whether the old identity is still on this host." +fi + +# The archiver image is what does the extraction. Prefer the one recorded in the +# manifest, so a set is put back by the same tar that took it. +if [[ -z ${ARCHIVER} ]]; then + ARCHIVER="$("${REPO_ROOT}/scripts/image-for.sh" archiver)" +fi +docker image inspect "${ARCHIVER}" >/dev/null 2>&1 || { + info "pulling ${ARCHIVER}" + docker pull -q "${ARCHIVER}" >/dev/null +} + +# --------------------------------------------------------------------------- +# Phase 1 — read-only. Nothing below this section touches a live volume. +# --------------------------------------------------------------------------- +info "phase 1: proving every selected archive is readable before anything is destroyed" + +failed=0 +for v in "${TARGETS[@]}"; do + f="${SET_DIR}/${v}.tar.gz.age" + if [[ ! -f ${f} ]]; then + red "${v}: no archive in ${STAMP}" + failed=1 + continue + fi + want="$(manifest_sha "${SET_DIR}" "${v}")" + if [[ -n ${want} ]]; then + got="$(sha256sum "${f}" | awk '{print $1}')" + if [[ ${got} != "${want}" ]]; then + red "${v}: sha256 does not match the manifest — the file has changed since it was written" + red " manifest: ${want}" + red " on disk: ${got}" + failed=1 + continue + fi + else + warn "${v}: no sha256 in the manifest to check against" + fi +done +((failed == 0)) || die "phase 1 failed — nothing was touched" + +# The decrypt-and-read pass is backup-volumes.sh's own verify(), not a second +# implementation of it. One sentinel table, one set of assertions, used by the +# script that writes archives and the script that consumes them. +"${BACKUP}" --verify-only --set "${STAMP}" ${ONLY:+--only "${ONLY}"} \ + || die "phase 1 failed — the archives did not verify, and nothing was touched" + +# --------------------------------------------------------------------------- +# The plan +# --------------------------------------------------------------------------- +printf '\n' +printf 'Restoring set %s — %s, %s\n\n' "${STAMP}" "${SET_MODE:-unknown mode}" "$(stamp_age "${STAMP}")" +printf ' %-32s %14s %14s\n' "volume" "live now" "in the set" +for v in "${TARGETS[@]}"; do + live="?" + if docker volume inspect "${PROJECT}_${v}" >/dev/null 2>&1; then + kb="$(docker run --rm --network none -v "${PROJECT}_${v}:/data:ro" "${ARCHIVER}" du -sk /data | awk '{print $1}')" + live="$(human $((kb * 1024)))" + else + live="does not exist" + fi + printf ' %-32s %14s %14s\n' "${PROJECT}_${v}" "${live}" "$(human "$(manifest_bytes "${SET_DIR}" "${v}")")" +done +printf '\n' + +if ((DRY)); then + green "--dry-run: every selected archive decrypts and reads clean. Nothing was changed." + printf '\n' + info "That proves the key works, the ciphertext is intact, each archive unpacks to a" + info "complete tar, and each holds the volume its name claims. It does NOT prove that" + info "Prometheus will open the restored TSDB, that Grafana will read the restored" + info "grafana.db, or that ownership survives the round trip. See" + info "docs/runbooks/restore-the-stack.md." + exit 0 +fi + +# --------------------------------------------------------------------------- +# Confirmation — before the stop, because stopping the stack is itself +# disruptive and must not happen on a set that is about to be rejected. +# --------------------------------------------------------------------------- +[[ -t 0 ]] || die "restore needs a terminal — it will not run unattended, and there is deliberately no --yes" + +printf '\033[0;33mThis destroys the current contents of the %s volume(s) above.\033[0m\n' "${#TARGETS[@]}" +((SAFETY)) && printf 'A pre-restore snapshot is written first.\n' +read -r -p "Type '${STAMP}' to continue: " reply +[[ ${reply} == "${STAMP}" ]] || die "not confirmed — nothing was touched" + +if [[ ${SET_MODE} == hot ]]; then + # A hot copy of Prometheus or Loki can hold a torn WAL. Prometheus will either + # refuse to start or silently drop the head block, and the second is worse. + printf '\033[0;33m%s was taken with --hot: the services were running, so nothing in it is proven restorable.\033[0m\n' "${STAMP}" + read -r -p "Type 'unproven' to continue anyway: " reply2 + [[ ${reply2} == unproven ]] || die "not confirmed — nothing was touched" +fi + +# --------------------------------------------------------------------------- +# Phase 2 — destructive +# --------------------------------------------------------------------------- +info "stopping the stack" +"${COMPOSE[@]}" stop -t 60 + +for v in "${TARGETS[@]}"; do + holders="$(docker ps -q --filter "volume=${PROJECT}_${v}")" + [[ -z ${holders} ]] \ + || die "a container is still running against ${PROJECT}_${v} — refusing to replace a volume in use" +done + +if ((SAFETY)); then + SNAP_DIR="${OUT_DIR}/.pre-restore-${STAMP}" + mkdir -p "${SNAP_DIR}" + AGE_RECIPIENT="$(recipient)" + [[ -n ${AGE_RECIPIENT} ]] || die "no age recipient in ${SOPS_POLICY} — cannot write the safety snapshot" + info "snapshotting the current contents to ${SNAP_DIR#"${REPO_ROOT}"/}" + snapped=() + for v in "${TARGETS[@]}"; do + docker volume inspect "${PROJECT}_${v}" >/dev/null 2>&1 || continue + docker run --rm --network none --read-only --security-opt no-new-privileges \ + -v "${PROJECT}_${v}:/data:ro" "${ARCHIVER}" \ + tar --numeric-owner -czf - -C /data . 2>/dev/null \ + | age --recipient "${AGE_RECIPIENT}" --output "${SNAP_DIR}/${v}.tar.gz.age" + snapped+=("${v}") + done + # The snapshot gets a manifest of its own, in the same shape, so rolling back + # is `--from .pre-restore-` rather than a hand-written docker command + # typed under pressure. The leading dot keeps it out of backup-volumes.sh's + # set listing and therefore out of retention: nothing prunes it but you. + { + printf '# pre-restore snapshot taken before restoring %s\n' "${STAMP}" + printf 'stack\t%s\n' "${STACK}" + printf 'project\t%s\n' "${PROJECT}" + printf 'mode\tquiesced\n' + printf 'recipient\t%s\n' "${AGE_RECIPIENT}" + printf 'archiver\t%s\n' "${ARCHIVER}" + printf '#volume\tservice\tmount\tbytes\tsha256\n' + for v in "${snapped[@]}"; do + printf '%s\t%s\t-\t%s\t%s\n' "${v}" "${VOL_SERVICE[$v]}" \ + "$(stat -c %s "${SNAP_DIR}/${v}.tar.gz.age")" \ + "$(sha256sum "${SNAP_DIR}/${v}.tar.gz.age" | awk '{print $1}')" + done + } > "${SNAP_DIR}/MANIFEST" + + # Restoring onto a bare host replaces nothing, so there is nothing to snapshot + # and an empty directory claiming to hold "the only copy" of what was replaced + # is worse than no directory at all. + if ((${#snapped[@]} == 0)); then + rm -rf -- "${SNAP_DIR}" + SNAP_DIR="" + info "no existing volumes to snapshot — nothing is being replaced" + fi +fi + +for v in "${TARGETS[@]}"; do + info "restoring ${PROJECT}_${v}" + # Empty and extract in ONE container invocation, so a half-emptied volume + # needs the container to die between the two. It cannot be made properly + # transactional — Docker cannot rename a volume, so "extract to a scratch + # volume and swap" is really "wipe the real one and copy", the same window and + # twice the disk. The honest answer to an unavoidable window is the snapshot + # above, not pretending the window is closed. + # + # Extracted as root, with --numeric-owner, so tar can put back the uids the + # services run as (65534, 10001, 472) rather than remapping them. + age --decrypt -i "${AGE_IDENTITY}" "${SET_DIR}/${v}.tar.gz.age" \ + | docker run --rm -i --network none -v "${PROJECT}_${v}:/data" "${ARCHIVER}" \ + sh -c 'set -e; cd /data && find . -mindepth 1 -delete && tar --numeric-owner -xzf - -C /data' +done + +# --------------------------------------------------------------------------- +# Ownership — reported, never silently corrected +# --------------------------------------------------------------------------- +printf '\n' +for v in "${TARGETS[@]}"; do + want="${EXPECT_UID[$v]:-}" + [[ -n ${want} ]] || continue + got="$(docker run --rm --network none -v "${PROJECT}_${v}:/data:ro" "${ARCHIVER}" \ + stat -c '%u' /data/. 2>/dev/null || echo '?')" + if [[ ${got} == "${want}" ]]; then + green "${v}: owned by uid ${got}, as ${VOL_SERVICE[$v]} expects" + else + warn "${v}: owned by uid ${got}, but ${VOL_SERVICE[$v]} runs as ${want}." + warn " ${VOL_SERVICE[$v]} will not be able to write to it. To correct, deliberately:" + warn " docker run --rm -v ${PROJECT}_${v}:/data ${ARCHIVER} chown -R ${want}:${want} /data" + fi +done + +printf '\n' +green "restored ${#TARGETS[@]} volume(s) from ${STAMP}" +printf '\n' +if ((SAFETY)) && [[ -n ${SNAP_DIR} ]]; then + info "What you replaced is in ${SNAP_DIR#"${REPO_ROOT}"/} — the only copy. To roll back:" + info " make restore ARGS=\"--from .pre-restore-${STAMP}\"" +fi +info "The stack is STOPPED. Bring it up and then verify:" +info " make up" +info " docs/runbooks/restore-the-stack.md section 4" +printf '\n' +warn "Four things a restore gets right mechanically and wrong operationally:" +warn " Grafana's admin password now comes from the restored grafana.db, not .env —" +warn " GF_SECURITY_ADMIN_PASSWORD only applies when the admin user is created." +warn " Silences live at ${STAMP} are back and are suppressing alerts nobody remembers silencing." +warn " Prometheus ages restored blocks from their own timestamps, so 30d of retention" +warn " against a 20-day-old set is 10 days of history, shrinking." +warn " Alloy's log positions went back to their old offsets — expect either duplicate" +warn " lines in Loki, or a silent gap where the files have since rotated." +printf '\n' +warn "This put data back. It did not verify the data is correct. Run section 4." diff --git a/stacks/observability/compose.yaml b/stacks/observability/compose.yaml index 4b15814..d966e2b 100644 --- a/stacks/observability/compose.yaml +++ b/stacks/observability/compose.yaml @@ -180,7 +180,28 @@ services: GF_AUTH_ANONYMOUS_ENABLED: "false" GF_ANALYTICS_REPORTING_ENABLED: "false" GF_ANALYTICS_CHECK_FOR_UPDATES: "false" - GF_INSTALL_PLUGINS: grafana-clock-panel,grafana-piechart-panel + # No plugins are installed here, deliberately. This was + # GF_INSTALL_PLUGINS: grafana-clock-panel,grafana-piechart-panel and both + # were dead weight — no dashboard under grafana/dashboards/ uses either + # panel type, and grafana-piechart-panel is an Angular plugin, which this + # Grafana refuses to initialise: it was downloaded and then rejected on + # every single start. + # + # It was also an availability bug, which is the reason it is gone rather + # than merely tidied. The background installer contacts grafana.com on + # every start, and a failure there is FATAL to the process — Grafana + # crash-looped on a host with no uplink, even though both plugins were + # already present in grafana-data. That turns an ordinary outage into a + # failed disaster recovery, and it was found by doing one: see the last + # section of docs/runbooks/restore-the-stack.md. + # + # Grafana still preinstalls its own bundled apps (pyroscope, exploretraces, + # metricsdrilldown, lokiexplore, elasticsearch). Those ship inside the + # image and need no network, which is what makes an offline start work. + # + # If a plugin is ever genuinely needed, add GF_PLUGINS_PREINSTALL — + # GF_INSTALL_PLUGINS is deprecated — and re-test the offline start before + # committing it. GF_PATHS_PROVISIONING: /etc/grafana/provisioning PROMETHEUS_URL: http://prometheus:9090 LOKI_URL: http://loki:3100 @@ -345,6 +366,42 @@ services: # an error. shm_size: "1gb" + # --------------------------------------------------------------------------- + # tar and gzip for scripts/backup-volumes.sh. NOT part of the running stack: + # it sits behind the `backup` profile, so `make up` neither starts it nor + # pulls it, and `docker compose ps` shows six services as before. + # + # This service is never actually started. It exists so that the image the + # backup script runs is pinned in the one place this repository pins images. + # `make backup` used to run a bare `alpine` — no tag at all, so it resolved to + # :latest and slipped past both the floating-tag check (which looks for a + # literal :latest) and the digest check (which reads image: lines out of this + # file). Writing `alpine:3.22` into the script instead would only move the + # problem: Dependabot watches this file, `make pin-digests` reads this file, + # and a pin anywhere else is one nothing bumps and nothing digests — which is + # the whole argument in the header of scripts/image-for.sh. So the image lives + # here and the script asks for it with `image-for.sh archiver`. + # + # Debian rather than Alpine, and this is not incidental. Alpine's tar is + # BusyBox tar, which does not notice a file that changed while it was being + # read; GNU tar reports it and exits 1, and that exit status is the only + # mechanical evidence a `--hot` run has that it just wrote a torn archive. + # BusyBox exits 0 on the identical archive and it verifies clean. BusyBox tar + # also has no --numeric-owner, which the restore path needs to put back uids + # 65534, 10001 and 472 on volumes whose names it does not have. + # + # Run with `docker run`, not `docker compose run`: the tar stream IS the + # container's stdout and compose writes its own progress lines into it. + # + # It mounts nothing, deliberately. backup-volumes.sh derives the + # volume -> service map by finding the one service that mounts each named + # volume, and a second claimant would make that ambiguous. + # --------------------------------------------------------------------------- + archiver: + profiles: ["backup"] + image: debian:13-slim@sha256:d7e12182ce18b85b93007c1dedf31f2d29e01ccf3182cc4017c709b6259bc132 + command: ["true"] + networks: observability: driver: bridge