From 535dbe9548c745cc620f8b454eef122bac2bc4d1 Mon Sep 17 00:00:00 2001 From: Josh Vaughen Date: Thu, 8 Oct 2026 14:16:10 -0700 Subject: [PATCH 1/7] feat(image): ship aws.config in the rsync image The rsync image carries docker/aws.config at /etc/aws.config and sets AWS_CONFIG_FILE to it: a 4 GiB multipart threshold, 512 MiB parts, and the standard retry mode with ten attempts. The four mirror copies differ (tlnet's threshold is 200MB); the image's value wins over a Taskfile env:, so they go dead once this image is live, and each mirror deletes its copy in its own PR. README and the publish comment no longer say the mirror's Taskfile names the file. --- README.md | 7 ++++--- docker/aws.config | 18 ++++++++++++++++++ docker/rsync.Dockerfile | 7 +++++-- engines/rsync.yml | 6 +++--- 4 files changed, 30 insertions(+), 8 deletions(-) create mode 100644 docker/aws.config diff --git a/README.md b/README.md index 11c4491..5b33bf2 100644 --- a/README.md +++ b/README.md @@ -641,9 +641,10 @@ default: a full CTAN mirror is under $2 a month. Things the engines know about i `when_required`. - R2 does not list keys in byte order, so every bucket listing is re-sorted before `join` or `comm`. -- The single-part upload limit is 4.995 GiB; a mirror with a larger file sets - `multipart_threshold` and `multipart_chunksize` in an `aws.config` the Taskfile names - through `AWS_CONFIG_FILE`. A multipart upload costs one Class A operation per part. +- The single-part upload limit is 4.995 GiB. The `rsync` image sets `AWS_CONFIG_FILE` to + its `/etc/aws.config`. There, `multipart_threshold` and `multipart_chunksize` make the + CLI send a file over 4 GiB in 512 MiB parts. A multipart upload costs one Class A + operation per part. - `DeleteObjects` takes 1,000 keys per call and is free. ## Monitoring diff --git a/docker/aws.config b/docker/aws.config new file mode 100644 index 0000000..e518e18 --- /dev/null +++ b/docker/aws.config @@ -0,0 +1,18 @@ +# This file configures the AWS CLI in the rsync image. The image copies it to +# /etc/aws.config and sets AWS_CONFIG_FILE to that path, so every aws call reads it. +# +# The CLI's default multipart threshold is 8 MB. Here it is 4 GiB, so a file under 4 GiB +# goes up as one PutObject. R2 refuses a single part over 4.995 GiB. So a file over +# 4 GiB goes up in 512 MiB parts. Five CTAN files are over 4.995 GiB, and each goes up in +# 13 parts, for 15 Class A operations. The largest gnu file is 1.51 GB, the largest +# nongnu file is 0.72 GB, and the largest tlnet file is about 145 MB. +# +# R2 answers a burst of writes with a 503 at times. The standard retry mode waits and +# tries again, for up to ten attempts. +[default] +retry_mode = standard +max_attempts = 10 +s3 = + multipart_threshold = 4GB + multipart_chunksize = 512MB + max_concurrent_requests = 32 diff --git a/docker/rsync.Dockerfile b/docker/rsync.Dockerfile index bbbe34f..68d07ef 100644 --- a/docker/rsync.Dockerfile +++ b/docker/rsync.Dockerfile @@ -56,7 +56,10 @@ RUN set -eu; \ rsync --version | head -1; gpgv --version | head -1; xz --version | head -1; shasum --version # Inside a run there is no network for Taskfiles: the mirror's .task/remote cache rides -# in with the bind mount. R2 has one region; the value is a literal, not a secret. +# in with the bind mount. R2 has one region; the value is a literal, not a secret. Every +# aws call reads its multipart and retry settings from /etc/aws.config. +COPY docker/aws.config /etc/aws.config ENV TASK_REMOTE_OFFLINE=1 \ - AWS_REGION=auto + AWS_REGION=auto \ + AWS_CONFIG_FILE=/etc/aws.config WORKDIR /work diff --git a/engines/rsync.yml b/engines/rsync.yml index 6c4f44b..ad5a769 100644 --- a/engines/rsync.yml +++ b/engines/rsync.yml @@ -390,9 +390,9 @@ tasks: - cat {{.RUN}}/labels.txt >> {{.RUN}}/labeled.txt # The CLI takes one --content-type a call and guesses from the name without one, so # each type's files go up from a tree of their own, RUN/up/N/, and come back for - # merge. cp --recursive never lists the destination: one PutObject per file, - # multipart only above the mirror's aws.config threshold. Keys are the paths under - # the tree, which are upstream's own. + # merge. cp --recursive never lists the destination. It sends each file as one + # PutObject, or in parts above the threshold in the image's aws.config. Keys are the + # paths under the tree, which are upstream's own. - {task: label-trees, vars: {TO: up}} - >- for d in {{.RUN}}/up/*/; do test -d "$d" || continue; From 18ac7dcceb4f6ed50840ef693f57c5a7c5be395f Mon Sep 17 00:00:00 2001 From: Josh Vaughen Date: Thu, 8 Oct 2026 14:16:16 -0700 Subject: [PATCH 2/7] feat(image): set the Proton CLI's plain-file store in the proton image The proton image has no keyring, so it sets PROTON_DRIVE_CREDENTIALS_STORE=unsafe_file as an ENV line beside AWS_REGION. The engine's PD drops the setting and keeps the cache dir, which stays with each caller. Every [pd] line of a render changes. --- docker/proton.Dockerfile | 6 ++++-- engines/proton.yml | 5 +++-- examples/proton/render.txt | 4 ++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/docker/proton.Dockerfile b/docker/proton.Dockerfile index 8499c15..e7ff2b7 100644 --- a/docker/proton.Dockerfile +++ b/docker/proton.Dockerfile @@ -68,11 +68,13 @@ PY # Inside a run there is no network for Taskfiles: the mirror's .task/remote cache rides # in with the bind mount. R2 has one region and rejects the SDK's default checksum -# headers; both values are literals, not secrets. +# headers; both values are literals, not secrets. The image has no keyring, so the +# Proton CLI keeps its session as plain files. ENV PYTHONUNBUFFERED=1 \ PYTHONPATH=/work/src \ TASK_REMOTE_OFFLINE=1 \ AWS_REGION=auto \ AWS_REQUEST_CHECKSUM_CALCULATION=when_required \ - AWS_RESPONSE_CHECKSUM_VALIDATION=when_required + AWS_RESPONSE_CHECKSUM_VALIDATION=when_required \ + PROTON_DRIVE_CREDENTIALS_STORE=unsafe_file WORKDIR /work diff --git a/engines/proton.yml b/engines/proton.yml index 16de42a..e8e4282 100644 --- a/engines/proton.yml +++ b/engines/proton.yml @@ -29,8 +29,9 @@ vars: SESSION: '{{.ROOT_DIR}}/.run/session' SESSION_KEY: .state/session.tar.age UPLOAD: '{{.ROOT_DIR}}/.run/upload.json' - # The CLI with its session as plain files under RUN/session: the image has no keyring. - PD: PROTON_DRIVE_CACHE_DIR={{.SESSION}} PROTON_DRIVE_CREDENTIALS_STORE=unsafe_file proton-drive + # The CLI keeps its session as plain files under RUN/session. The image sets the plain + # file store, because it has no keyring. + PD: PROTON_DRIVE_CACHE_DIR={{.SESSION}} proton-drive tasks: pipeline: diff --git a/examples/proton/render.txt b/examples/proton/render.txt index f3b5003..a07388c 100644 --- a/examples/proton/render.txt +++ b/examples/proton/render.txt @@ -16,7 +16,7 @@ rm -f "$k"; exit $rc task: [session] tar -xf /work/examples/proton/.run/session.tar -C /work/examples/proton/.run/session auth-session.json clientUid.json && chmod 600 /work/examples/proton/.run/session/*.json task: [session] rm -f /work/examples/proton/.run/session.tar /work/examples/proton/.run/session.tar.age task: [session] sha256sum /work/examples/proton/.run/session/auth-session.json > /work/examples/proton/.run/session.sha -task: [pd] rc=0; PROTON_DRIVE_CACHE_DIR=/work/examples/proton/.run/session PROTON_DRIVE_CREDENTIALS_STORE=unsafe_file proton-drive filesystem list -j "$(dirname "$MIRROR_PROTON_DESTINATION")" > /work/examples/proton/.run/parent.json 2> /work/examples/proton/.run/pd.err || rc=$?; echo $rc > /work/examples/proton/.run/pd.rc +task: [pd] rc=0; PROTON_DRIVE_CACHE_DIR=/work/examples/proton/.run/session proton-drive filesystem list -j "$(dirname "$MIRROR_PROTON_DESTINATION")" > /work/examples/proton/.run/parent.json 2> /work/examples/proton/.run/pd.err || rc=$?; echo $rc > /work/examples/proton/.run/pd.rc task: [session-push] echo "session: token rotated, sealing it back to the bucket" task: [seal] mkdir -p /work/examples/proton/.run && tar -cf /work/examples/proton/.run/session.tar -C /work/examples/proton/.run/session auth-session.json clientUid.json task: [age] test -n "$MIRROR_AGE_IDENTITY" || { echo "MIRROR_AGE_IDENTITY is unset" >&2; exit 1; } @@ -52,7 +52,7 @@ PY task: [destination] rm -f /work/examples/proton/.run/parent.json task: [stage] rm -rf /work/examples/proton/staging && mkdir -p /work/examples/proton/staging/example && printf 'hello\n' > /work/examples/proton/staging/example/hello.txt task: [upload] test -n "$(ls -A /work/examples/proton/staging 2>/dev/null)" || { echo "upload: staging is empty; stage put nothing there" >&2; exit 1; } -task: [pd] rc=0; PROTON_DRIVE_CACHE_DIR=/work/examples/proton/.run/session PROTON_DRIVE_CREDENTIALS_STORE=unsafe_file proton-drive filesystem upload -f create-new-revision -d merge -t --json /work/examples/proton/staging/* "$MIRROR_PROTON_DESTINATION" > /work/examples/proton/.run/upload.json 2> /work/examples/proton/.run/pd.err || rc=$?; echo $rc > /work/examples/proton/.run/pd.rc +task: [pd] rc=0; PROTON_DRIVE_CACHE_DIR=/work/examples/proton/.run/session proton-drive filesystem upload -f create-new-revision -d merge -t --json /work/examples/proton/staging/* "$MIRROR_PROTON_DESTINATION" > /work/examples/proton/.run/upload.json 2> /work/examples/proton/.run/pd.err || rc=$?; echo $rc > /work/examples/proton/.run/pd.rc task: [session-push] echo "session: token rotated, sealing it back to the bucket" task: [seal] mkdir -p /work/examples/proton/.run && tar -cf /work/examples/proton/.run/session.tar -C /work/examples/proton/.run/session auth-session.json clientUid.json task: [age] test -n "$MIRROR_AGE_IDENTITY" || { echo "MIRROR_AGE_IDENTITY is unset" >&2; exit 1; } From 7cbea1d9384f4119e467d2149d92dac34ed0b587 Mon Sep 17 00:00:00 2001 From: Josh Vaughen Date: Thu, 8 Oct 2026 14:16:25 -0700 Subject: [PATCH 3/7] refactor(toolbox): engines name IMAGE and report-engine; one STAGING Each engine names its image in vars: as {{.IMAGE | default ...}}, so the IMAGE on the toolbox include, the examples' -dev say, still wins; a literal there would beat the include. A mirror passes IMAGE only when its engine does not name it. The toolbox drops its report-engine no-op and the rule that every engine consumer excludes it: an engine defines report-engine, and a mirror with no engine defines its own (dropbox does). An excludes: entry that names no task is accepted, so every mirror's Taskfile works as it is. Both examples drop the exclude, and the README's examples follow. STAGING is defined once, beside RUN in the toolbox, and clean uses it. The proton engine's SESSION and UPLOAD keep ROOT_DIR/.run: read from a root task, a global that reads another include's RUN can render empty, because go-task orders sibling includes' vars differently from run to run. --- README.md | 20 ++++++++++---------- engines/proton.yml | 9 ++++++++- engines/rsync.yml | 5 ++++- examples/proton/Taskfile.yml | 1 - examples/rsync/Taskfile.yml | 1 - toolbox.yml | 19 +++++++++---------- 6 files changed, 31 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 5b33bf2..19c1285 100644 --- a/README.md +++ b/README.md @@ -84,7 +84,7 @@ flowchart TB |---|---|---|---| | Toolbox | `toolbox.yml` | How a run starts, is contained, resolves secrets, is rendered, checked and reported | Never; it is included as is | | Engine | `engines/.yml` | How bytes move for one transport, and the pipeline's order | It lists a hook under `excludes:` and defines its own | -| Image | `docker/`, `toolchain.lock.toml` | Every tool a run needs, pinned by checksum | Never; it names one in `IMAGE` | +| Image | `docker/`, `toolchain.lock.toml` | Every tool a run needs, pinned by checksum | Never; its engine names one, or it names one in `IMAGE` | | Workflows | `.github/workflows`, `.github/actions/toolbox` | How Actions installs the tools and runs `task sync` and `task check` | Never; its callers are a few lines each | | Mirror | The mirror's own repository | Identity, the hooks, the exceptions | Always; this is all it holds | @@ -189,7 +189,7 @@ keeps its own. | `due` | Decide whether this run reconciles, by the age of the last reconcile at `.state/reconciled`; leave `.run/reconcile` when it does. `due-rule` is the rule alone | | `reconciled` | Record this run's start at `.state/reconciled`; an engine runs it once a reconcile completes | | `report` | Append the run summary to the Actions job page (stdout elsewhere): its own rows, then `report-engine`'s, then `report-mirror`'s | -| `report-engine`, `report-mirror` | Hooks: no-ops here; the engine's and the mirror's rows | +| `report-mirror` | Hook: a no-op here; the mirror's rows | | `ping` | GET `HEALTHCHECK_URL`; skipped when unset | | `ping-fail` | GET `HEALTHCHECK_URL/fail`; skipped when unset | | `failed` | The failure path: `report STATUS=failed`, then `ping-fail` | @@ -225,8 +225,10 @@ reconciles defines `pull` and `push` itself. | Name | Kind | Meaning | |---|---|---| -| `NAME`, `DESC`, `IMAGE` | include vars, required | The menu's title and line, and the image the run happens inside | +| `NAME`, `DESC` | include vars, required | The menu's title and line | +| `IMAGE` | include var, required unless the engine names it | The image the run happens inside. The include's value wins over the engine's | | `pipeline`, `plan-pipeline` | tasks, required | The full run and its read-only half, inside the image. An engine supplies both; a mirror with no engine defines them | +| `report-engine` | task, required | The engine's rows of the run summary, which `report` calls. An engine supplies it; a mirror with no engine defines it, with no commands when it has no rows | | `op.env` | file | `op://` references, one per secret. Absent means the environment is already resolved: on a laptop, whatever is exported; in Actions, the repository's secrets, crossing by the names in `PASS` | | `PASS` | include var | Host environment names that cross into the container beside the ones in `op.env`. Task vars are not environment: they go after `--` | | `MENU` | include var | Extra lines for the menu, one per mirror-specific verb | @@ -706,7 +708,7 @@ flowchart LR rel --> tag["the git tag vX, moved"] tag --> gr["the GitHub release vX.Y.Z"] tag --> inc["mirrors include toolbox.yml and an engine at v2"] - ghcr --> img["mirrors name IMAGE at -v2"] + ghcr --> img["engines, and mirrors with no engine, name IMAGE at -v2"] ``` Two pins, one policy. The include and the image float at `v2`, on purpose: moving that @@ -793,7 +795,6 @@ includes: toolbox: taskfile: https://raw.githubusercontent.com/katoptra/lib/v2/toolbox.yml flatten: true - excludes: [report-engine] vars: NAME: ctan DESC: an hourly mirror of CTAN at https://ctan.katoptra.org/ @@ -811,8 +812,8 @@ AWS_ENDPOINT_URL=op:///ctan/r2/endpoint HEALTHCHECK_URL=op:///ctan/healthcheck/url ``` -That is a whole mirror: the engine supplies `pipeline` and `plan-pipeline`, and -`report-engine` is excluded on the toolbox include because the engine defines it too. +That is a whole mirror. The engine supplies `pipeline`, `plan-pipeline` and +`report-engine`. A mirror of one signed subtree, shaped like tlnet, adds `TL`, `TL_KEY`, `CEILING_GB`, `OWN` and a `FILTER` to the root vars, excludes `index` on the engine include, and @@ -832,7 +833,7 @@ includes: toolbox: taskfile: https://raw.githubusercontent.com/katoptra/lib/v2/toolbox.yml flatten: true - excludes: [report-engine, report-mirror] + excludes: [report-mirror] vars: {NAME: github, DESC: a nightly mirror of every repository under OWNERS into Proton Drive, IMAGE: ghcr.io/katoptra/toolbox:proton-v2} proton: taskfile: https://raw.githubusercontent.com/katoptra/lib/v2/engines/proton.yml @@ -862,6 +863,7 @@ tasks: cmds: [{task: clock}, {task: session}, {task: state}, '# the mirror's own steps', {task: report}, {task: ping}] plan-pipeline: cmds: [{task: clock}, {task: session}, {task: state}, '# the read-only steps'] + report-engine: {cmds: []} # no engine, so no engine rows report-mirror: {cmds: ['# cat the mirror's own report onto the job page']} ``` @@ -957,8 +959,6 @@ The rules that keep this honest: - The replacement reads the mirror's root vars, the same as the original did. - Override engine verbs and the in-container toolbox verbs. Do not override the host side: `sync`, `plan`, `check` and `run` are what the workflows and the menu promise. -- `report-engine` is defined in the toolbox as a no-op and in every engine, so an engine - consumer's toolbox include excludes it. ### Adding a hook or a verb, in an engine diff --git a/engines/proton.yml b/engines/proton.yml index e8e4282..7c1f9fb 100644 --- a/engines/proton.yml +++ b/engines/proton.yml @@ -25,7 +25,14 @@ version: '3' vars: - STAGING: '{{.ROOT_DIR}}/staging' + # The engine names its image. The IMAGE on the toolbox include, for example the -dev + # image of a lib example, must win. So this value is a default: a literal value here + # beats the include. + IMAGE: '{{.IMAGE | default "ghcr.io/katoptra/toolbox:proton-v2"}}' + # SESSION and UPLOAD spell out ROOT_DIR/.run and do not read RUN, the toolbox's var. + # For a root task, go-task evaluates the vars of sibling includes in an order that + # changes from run to run (verified, 3.53.1 and 3.54.0). So RUN can be unset when it + # evaluates these two. SESSION: '{{.ROOT_DIR}}/.run/session' SESSION_KEY: .state/session.tar.age UPLOAD: '{{.ROOT_DIR}}/.run/upload.json' diff --git a/engines/rsync.yml b/engines/rsync.yml index ad5a769..322872b 100644 --- a/engines/rsync.yml +++ b/engines/rsync.yml @@ -38,11 +38,14 @@ version: '3' vars: + # The engine names its image. The IMAGE on the toolbox include, for example the -dev + # image of a lib example, must win. So this value is a default: a literal value here + # beats the include. + IMAGE: '{{.IMAGE | default "ghcr.io/katoptra/toolbox:rsync-v2"}}' S3: s3://{{.BUCKET}} URL: https://{{.HOST}} STATE: .state/applied.txt.xz # the listing of what the bucket holds; never in upstream INDEXED: .state/indexed.txt.xz # the state as the directory pages last showed it - STAGING: '{{.ROOT_DIR}}/staging' BATCHES_MAX: '{{.MAX_BATCHES | default 4}}' # batches per run; the rest chain the next run RETRY_S: '{{.RETRY_BASE | default 15}}' # seconds; a retry sleeps RETRY_S * 2^i plus jitter FRESH_H: '{{.FRESH_HOURS | default 24}}' # hours; GNU's and CTAN's monitors call a mirror old past 28 diff --git a/examples/proton/Taskfile.yml b/examples/proton/Taskfile.yml index 02341b8..1b73ee3 100644 --- a/examples/proton/Taskfile.yml +++ b/examples/proton/Taskfile.yml @@ -9,7 +9,6 @@ includes: toolbox: taskfile: ../../toolbox.yml flatten: true - excludes: [report-engine] # the engine's vars: NAME: example-proton DESC: the proton toolbox and engine, exercised diff --git a/examples/rsync/Taskfile.yml b/examples/rsync/Taskfile.yml index b117fba..1451238 100644 --- a/examples/rsync/Taskfile.yml +++ b/examples/rsync/Taskfile.yml @@ -17,7 +17,6 @@ includes: toolbox: taskfile: ../../toolbox.yml flatten: true - excludes: [report-engine] # the engine's vars: NAME: example-rsync DESC: the rsync toolbox and engine, exercised diff --git a/toolbox.yml b/toolbox.yml index 8b06382..b8f6519 100644 --- a/toolbox.yml +++ b/toolbox.yml @@ -1,9 +1,10 @@ # The toolbox: every verb a katoptra mirror has whatever moves its bytes. A mirror -# includes this file flattened, passes NAME, DESC and IMAGE on the include, keeps its -# identity in its own root vars, and has two verbs, pipeline and plan-pipeline, both of -# which run inside IMAGE: an engine include supplies them, and a mirror with no engine -# defines them. A verb listed under the include's excludes: and defined in the mirror -# replaces the one here. +# includes this file flattened and passes NAME and DESC on the include. It passes IMAGE +# there too, unless the engine names the image. The mirror keeps its identity in its own +# root vars. Three verbs come from outside this file and run inside IMAGE: pipeline, +# plan-pipeline and report-engine. An engine include supplies them, and a mirror with no +# engine defines them. A verb listed under the include's excludes: and defined in the +# mirror replaces the one here. # # Rules this file keeps: no vars: default for anything a mirror owns (it would shadow # the mirror's value; defaults go inline), and host-side and in-container verbs share @@ -13,13 +14,12 @@ # label, label-trees, merge, checkpoint, remove, delete, reconcile, retry, pages, fresh, # and the hooks prepare, verify, index, smoke and smoke-mirror. The proton engine's: session, session-push, seal, # session-seal, age, pull, push, pd, destination, upload, confirm, list-folder, trash, -# empty-trash, empty-trash-pipeline, and the hooks stage and prune. report-engine lives -# here as a no-op and in every engine, so an engine consumer's toolbox include lists it -# under excludes:. +# empty-trash, empty-trash-pipeline, and the hooks stage and prune. version: '3' vars: RUN: '{{.ROOT_DIR}}/.run' + STAGING: '{{.ROOT_DIR}}/staging' # The container runtime, not the sync engine: Apple container when its daemon is up, # else Docker. ENGINE=docker forces Docker. ENGINE: @@ -126,7 +126,7 @@ tasks: clean: desc: Delete .run, the engine's staging tree and the Taskfile cache; nothing else, so a fixture kept in .git/info/exclude survives cmds: - - rm -rf {{.RUN}} {{.ROOT_DIR}}/staging {{.ROOT_DIR}}/.task + - rm -rf {{.RUN}} {{.STAGING}} {{.ROOT_DIR}}/.task # ---- pipeline ---- # The words after -- are task vars for the pipeline inside the image: task sync -- @@ -265,5 +265,4 @@ tasks: EOF - {task: report-engine} - {task: report-mirror} - report-engine: {cmds: []} # the engine's rows; an engine include replaces it report-mirror: {cmds: []} # the mirror's rows; a mirror replaces it From d5fba49de6bd6b1fbd69f42169b97be17d6e2f99 Mon Sep 17 00:00:00 2001 From: Josh Vaughen Date: Thu, 8 Oct 2026 14:16:31 -0700 Subject: [PATCH 4/7] refactor(rsync): read .run/chain where a step asks if batches remain batches counts the batch files once and writes .run/chain when more than MAX_BATCHES remain. delete's and reconcile's status, smoke's tlpdb read-back and smoke's never-landed check counted again; each runs after batches in the same pipeline run, so each now tests the chain file. The report's Delta row keeps its count, because it prints it. --- engines/rsync.yml | 8 ++++---- examples/rsync/render.txt | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/engines/rsync.yml b/engines/rsync.yml index 322872b..ab2b5ca 100644 --- a/engines/rsync.yml +++ b/engines/rsync.yml @@ -552,7 +552,7 @@ tasks: D: '{{if .D}}{{.D}}{{else}}{{.RUN}}/deleted.txt{{end}}' # Nothing to delete, or a batch is still waiting for a later hour: the live tlpdb may # still name these containers, so they go only in the hour every batch has landed. - status: ['test ! -s {{.D}} || test "$(ls {{.RUN}}/batch-*.txt 2>/dev/null | wc -l)" -gt {{.BATCHES_MAX}}'] + status: ['test ! -s {{.D}} || test -f {{.RUN}}/chain'] cmds: - {task: remove, vars: {K: '{{.D}}'}} - {task: checkpoint, vars: {B: '{{.D}}'}} @@ -562,7 +562,7 @@ tasks: # Not while a batch waits for a later run: delete would hold the orphans back, so the run # stays due and the chained run that lands the last batch reconciles. A run that fails # before the last line records nothing either, so the next run is due too. - status: ['test ! -f {{.RUN}}/reconcile || test "$(ls {{.RUN}}/batch-*.txt 2>/dev/null | wc -l)" -gt {{.BATCHES_MAX}}'] + status: ['test ! -f {{.RUN}}/reconcile || test -f {{.RUN}}/chain'] cmds: - touch {{.RUN}}/rebuild-now - {task: rebuild} @@ -712,7 +712,7 @@ tasks: # MAX_BATCHES has not published this hour's copy yet and has nothing to compare. - >- {{if .TL}}test ! -f {{.RUN}}/tl/tlpkg/texlive.tlpdb.sha512 || - test "$(ls {{.RUN}}/batch-*.txt 2>/dev/null | wc -l)" -gt {{.BATCHES_MAX}} || + test -f {{.RUN}}/chain || {{.CURL}} {{.URL}}/{{.TL}}/tlpkg/texlive.tlpdb.sha512 | cmp - {{.RUN}}/tl/tlpkg/texlive.tlpdb.sha512{{else}}echo "no TL subtree: no tlpdb to read back"{{end}} # One page this run redrew, read back under both of its keys by key (no zone rule # needed) and sized against the copy still in staging, the same one-byte ranged read as @@ -764,7 +764,7 @@ tasks: # been worked, the delta lines the state still lacks are those: report counts them, # and each of the first 20 is a warning on the Actions run. - >- - if test -f {{.RUN}}/changed.txt && test "$(ls {{.RUN}}/batch-*.txt 2>/dev/null | wc -l)" -le {{.BATCHES_MAX}}; + if test -f {{.RUN}}/changed.txt && test ! -f {{.RUN}}/chain; then LC_ALL=C comm -23 {{.RUN}}/changed.txt {{.RUN}}/applied.txt > {{.RUN}}/unlanded.txt; else : > {{.RUN}}/unlanded.txt; fi; cut -f1 {{.RUN}}/unlanded.txt | head -20 | while IFS= read -r p; do echo "::warning::listed upstream, never fetched: $p"; done # A called task sees none of its caller's call vars, so the three a check reads are diff --git a/examples/rsync/render.txt b/examples/rsync/render.txt index 1eeaeab..13216a5 100644 --- a/examples/rsync/render.txt +++ b/examples/rsync/render.txt @@ -319,6 +319,6 @@ task: [smoke] h=pub/index.html; n=$(awk -F'\t' -v k="$h" '$1 == k { print $2 }' task: [smoke] g=$(awk -F'\t' '$1 ~ /\.tar\.gz$/ && $2 > 0 { print $1; exit }' /work/examples/rsync/.run/applied.txt 2>/dev/null); case "https://example.katoptra.org" in http*) ;; *) g= ;; esac; if test -z "$g"; then echo "content-encoding check skipped (no non-empty .tar.gz in the state, or the URL is not http)"; else u=$(printf '%s\n' "$g" | awk 'BEGIN { for (i = 0; i < 256; i++) ord[sprintf("%c", i)] = i } function enc(s, out, j, c) { out = ""; for (j = 1; j <= length(s); j++) { c = substr(s, j, 1) out = out ((c ~ /[A-Za-z0-9\/._~-]/) ? c : sprintf("%%%02X", ord[c])) } return out } { print $0 "\t" enc($0) }' | cut -f2) && h=$(curl -fsS --connect-timeout 15 --max-time 60 --retry 6 --retry-connrefused --retry-max-time 600 -r 0-0 -D - -o /dev/null "https://example.katoptra.org/$u") && if printf '%s' "$h" | grep -qi '^content-encoding:'; then echo "$g: served with a content-encoding" >&2; exit 1; fi && echo "$g: served with no content-encoding"; fi -task: [smoke] if test -f /work/examples/rsync/.run/changed.txt && test "$(ls /work/examples/rsync/.run/batch-*.txt 2>/dev/null | wc -l)" -le 4; then LC_ALL=C comm -23 /work/examples/rsync/.run/changed.txt /work/examples/rsync/.run/applied.txt > /work/examples/rsync/.run/unlanded.txt; else : > /work/examples/rsync/.run/unlanded.txt; fi; cut -f1 /work/examples/rsync/.run/unlanded.txt | head -20 | while IFS= read -r p; do echo "::warning::listed upstream, never fetched: $p"; done +task: [smoke] if test -f /work/examples/rsync/.run/changed.txt && test ! -f /work/examples/rsync/.run/chain; then LC_ALL=C comm -23 /work/examples/rsync/.run/changed.txt /work/examples/rsync/.run/applied.txt > /work/examples/rsync/.run/unlanded.txt; else : > /work/examples/rsync/.run/unlanded.txt; fi; cut -f1 /work/examples/rsync/.run/unlanded.txt | head -20 | while IFS= read -r p; do echo "::warning::listed upstream, never fetched: $p"; done task: [fresh] echo "no FRESH_KEY: a source that stops moving goes unnoticed" task: [ping] test -z "$HEALTHCHECK_URL" || curl -fsS -m 10 --retry 3 -o /dev/null "$HEALTHCHECK_URL" From fda14acd8a7ce80637087b88b5e8813562d21b5f Mon Sep 17 00:00:00 2001 From: Josh Vaughen Date: Thu, 8 Oct 2026 14:16:40 -0700 Subject: [PATCH 5/7] fix(rsync): send the FRESH_KEY file last and sample it in smoke publish uploaded a bucket-root timestamp last, label-trees held it back, and smoke sampled it, all by that literal name. gnu's mirror-updated-timestamp.txt and nongnu's 00_TIME.txt could land before the tree they date. All four places now key on FRESH_KEY, under {{if .FRESH_KEY}}: ctan's render is unchanged, gnu's and nongnu's name their own file, and tlnet, which sets none, drops the upload-last step. The rsync example sets FRESH_KEY: timestamp, so smoke's offline check now runs fresh over run-smoke. offline writes a current Unix clock into that copy first, and the fixture lists timestamp at its 11 bytes. --- README.md | 2 +- engines/rsync.yml | 37 ++++++++++++------- examples/rsync/Taskfile.yml | 6 ++- examples/rsync/fixtures/run-smoke/applied.txt | 2 +- .../example.katoptra.org.directory.index.html | 2 +- .../fixtures/run-smoke/staging/timestamp | 2 +- .../rsync/fixtures/run-smoke/upstream.txt | 2 +- examples/rsync/render.txt | 7 +++- 8 files changed, 39 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 19c1285..573f0ac 100644 --- a/README.md +++ b/README.md @@ -288,7 +288,7 @@ until `INDEX` is set, and `smoke-mirror` until a mirror fills it. | `batches` | Work the first `MAX_BATCHES`, each `fetch`, `verify`, `publish`, `checkpoint`; touch `.run/chain` when batches remain | | `fetch` | rsync the batch's files into `staging/`, dereferencing symlinks; a path that vanished since the listing is skipped | | `verify` | Hook. With `TL_KEY` set: every signed file and every container in the batch against the tlpdb | -| `publish` | `label` the batch, then `aws s3 cp --recursive` once per type, each type's files in a tree of their own, one PutObject per file, never a destination listing; `timestamp` last | +| `publish` | `label` the batch, then `aws s3 cp --recursive` once per type, each type's files in a tree of their own, one PutObject per file, never a destination listing; the `FRESH_KEY` file last | | `label` | `.run/labels.txt`: every staged file's Content-Type, from its name and first KiB; see [Content types](#content-types) | | `checkpoint` | `merge` what landed into the state and push it as one PutObject; empty staging | | `delete` | Remove the keys upstream dropped, 1,000 per call, once every batch has landed, and drop them from the state | diff --git a/engines/rsync.yml b/engines/rsync.yml index ab2b5ca..b47e745 100644 --- a/engines/rsync.yml +++ b/engines/rsync.yml @@ -250,9 +250,10 @@ tasks: {{.RUN}}/changed.txt - rm -f {{.RUN}}/batch-*.txt # Cumulative size from the listing; nothing is touched to split. A file over the cap is - # a batch by itself and leaves the running batch alone. The decision batch, TL's - # tlpkg/ (the tlpdb) and the bucket-root files (timestamp among them), is last, so - # nothing a client reads to decide what to fetch can name a key that has not landed. + # a batch by itself and leaves the running batch alone. The decision batch goes last. + # It holds TL's tlpkg/ (the tlpdb) and the files at the bucket root, the FRESH_KEY + # file among them. So nothing that a client reads to decide what to fetch can name a + # key that has not landed. # A path that extends the previous one (x, x.sha512, x.sha512.asc) never starts a new batch, and diff puts x in the delta whenever x.sha512 is, so verify always finds the file its signature names. # ponytail: BATCH_GB is spliced bare because it lands inside $(( )), where a quoted # operand is a syntax error. What keeps it safe is the workflow refusing a vars input @@ -401,11 +402,16 @@ tasks: for d in {{.RUN}}/up/*/; do test -d "$d" || continue; aws s3 cp {{.AWS_FLAGS}} --recursive --content-type "$(cat "${d%/}.type")" "$d" {{.S3}}/ | tee -a {{.RUN}}/publish.txt; done - {task: label-trees, vars: {TO: staging}} - # timestamp is what mirmon reads, so it goes up alone, after everything else. + # The FRESH_KEY file goes up alone, after all other files. It dates the tree, and + # monitors such as mirmon read it. + # ponytail: this command and label-trees splice FRESH_KEY bare into an awk string and + # a Python string. Ceiling: a key with a quote, a backslash or a non-ASCII character + # breaks them. Upgrade path: give it to awk with -v and to Python as an argument. + # Then regenerate render.txt in lib and in each rsync mirror. - >- - test ! -f {{.STAGING}}/timestamp || - aws s3 cp {{.AWS_FLAGS}} --content-type "$(awk -F'\t' '$1 == "timestamp" { print $2 }' {{.RUN}}/labels.txt)" - {{.STAGING}}/timestamp {{.S3}}/timestamp | tee -a {{.RUN}}/publish.txt + {{if .FRESH_KEY}}test ! -f {{.STAGING}}/{{shellQuote .FRESH_KEY}} || + aws s3 cp {{.AWS_FLAGS}} --content-type "$(awk -F'\t' '$1 == "{{.FRESH_KEY}}" { print $2 }' {{.RUN}}/labels.txt)" + {{.STAGING}}/{{shellQuote .FRESH_KEY}} {{.S3}}/{{shellQuote .FRESH_KEY}} | tee -a {{.RUN}}/publish.txt{{end}} label: desc: RUN/labels.txt, "path TAB type" for every file under DIR (STAGING), byte-sorted; the type is decided from the name and the first KiB, never left to the CLI @@ -466,13 +472,15 @@ tasks: - LC_ALL=C sort {{.RUN}}/labels.raw > {{.RUN}}/labels.txt && rm {{.RUN}}/labels.raw label-trees: - desc: 'TO=up: rename every file of RUN/labels.txt but the root timestamp out of STAGING into RUN/up/N/, one tree per type, its type in RUN/up/N.type. TO=staging: rename them back' + desc: 'TO=up: rename every file of RUN/labels.txt but the FRESH_KEY file out of STAGING into RUN/up/N/, one tree per type, its type in RUN/up/N.type. TO=staging: rename them back' internal: true requires: {vars: [TO]} cmds: # Renames on one filesystem, so no byte is copied, and staging is whole again for # merge. A run that dies between the two leaves files under up/; the next run's fetch # starts staging from nothing and this starts up/ from nothing. + # The FRESH_KEY file stays in staging, and publish sends it last. With no FRESH_KEY, + # the empty name matches no file, so every file moves. - | python3 -I - {{.STAGING}} {{.RUN}}/up {{.RUN}}/labels.txt {{.TO}} <<'PY' import os, shutil, sys @@ -489,7 +497,7 @@ tasks: with open(labels, "rb") as f: for line in f: path, t = line.rstrip(b"\n").split(b"\t") - if path != b"timestamp": + if path != b"{{.FRESH_KEY}}": n = trees.setdefault(t, str(len(trees) + 1).encode()) move(os.path.join(staging, path), os.path.join(up, n, path)) for t, n in trees.items(): @@ -684,21 +692,22 @@ tasks: smoke: desc: Read a key of every type this run published back through the domain, size against the listing and type against the label; with TL set the tlpdb sha512 against the verified copy; with INDEX set one redrawn page under both keys; with CANARY set the canary, as a Perl client, against the bucket's copy; no Content-Encoding on a .tar.gz; warn of paths that never land; fresh; then the mirror's own checks, smoke-mirror vars: - # The check below that names /timestamp asks for it only once the state says it landed. + # The sample below reads the FRESH_KEY file only once the state says that it landed. # It rides the decision batch, the last one, so a first fill capped at MAX_BATCHES has # not reached it and the bucket answers 404 to a key it has never held; the run it # lands in and every run after name it. Empty is the only falsy value, so the path is # both the flag and the key. TS: - sh: awk -F'\t' '$1 == "timestamp" { print $1 }' {{.RUN}}/applied.txt 2>/dev/null || true + sh: |- + {{if .FRESH_KEY}}awk -F'\t' -v k={{shellQuote .FRESH_KEY}} '$1 == k { print $1 }' {{.RUN}}/applied.txt 2>/dev/null || true{{end}} cmds: # The sample is the first key of every type label gave this run, read back by its # encoded URL and looked up by its raw path: its size against the listing, and over # http its Content-Type against the label, which is what finds the CLI, R2 or the zone - # serving a type other than the one decided. timestamp joins for its size alone. A - # sampled key is as likely as any other to carry a space. + # serving a type other than the one decided. The FRESH_KEY file joins for its size + # alone. A sampled key is as likely as any other to carry a space. - >- - { {{if .TS}}printf '%s\t\n' {{.TS}};{{end}} + { {{if .TS}}printf '%s\t\n' {{shellQuote .TS}};{{end}} awk -F'\t' '!seen[$2]++' {{.RUN}}/labeled.txt 2>/dev/null; } | while IFS="$(printf '\t')" read -r k t; do u=$(printf '%s\n' "$k" | {{.ENCODE}} | cut -f2); diff --git a/examples/rsync/Taskfile.yml b/examples/rsync/Taskfile.yml index 1451238..de821fc 100644 --- a/examples/rsync/Taskfile.yml +++ b/examples/rsync/Taskfile.yml @@ -13,6 +13,7 @@ vars: INDEX: '{{.HOST}}.directory.index.html' PAGE_FOOT: 'A mirror of rsync.example.org/pub/. Report a problem.' CANARY: pub/index.html + FRESH_KEY: timestamp includes: toolbox: taskfile: ../../toolbox.yml @@ -124,7 +125,10 @@ tasks: # run-smoke: an hour whose only change is timestamp, whose one redrawn page is the root. # smoke reads timestamp back and sizes it against the listing, then the root page under # its INDEX key (a filesystem cannot hold the slashless key beside it). Over file:// the - # canary and the Content-Encoding check say they skipped. + # canary and the Content-Encoding check say they skipped. Last, fresh reads the clock + # in timestamp, the FRESH_KEY file. The first command below writes the current time + # into it, at the 11 bytes that the listing records. + - date -u +%s > {{.F}}/run-smoke/staging/timestamp - {task: smoke, vars: {RUN: '{{.F}}/run-smoke', STAGING: '{{.F}}/run-smoke/staging', URL: 'file://{{.F}}/run-smoke/staging'}} # A page index drew that is not in staging cannot pass. - '! task smoke-refuses 2>/dev/null' diff --git a/examples/rsync/fixtures/run-smoke/applied.txt b/examples/rsync/fixtures/run-smoke/applied.txt index d25ed5c..c8856ac 100644 --- a/examples/rsync/fixtures/run-smoke/applied.txt +++ b/examples/rsync/fixtures/run-smoke/applied.txt @@ -1,2 +1,2 @@ a/b/f.txt 0 2026/09/08 00:00:00 -timestamp 12 2026/09/08 00:00:00 +timestamp 11 2026/09/08 00:00:00 diff --git a/examples/rsync/fixtures/run-smoke/staging/example.katoptra.org.directory.index.html b/examples/rsync/fixtures/run-smoke/staging/example.katoptra.org.directory.index.html index 9d250ea..d4549c3 100644 --- a/examples/rsync/fixtures/run-smoke/staging/example.katoptra.org.directory.index.html +++ b/examples/rsync/fixtures/run-smoke/staging/example.katoptra.org.directory.index.html @@ -3,5 +3,5 @@

Index of /

- +
NameLast modifiedSize
a/
timestamp2026/09/08 00:00:0012
timestamp2026/09/08 00:00:0011

This directory on ctan.org. Sizes in bytes, times as CTAN lists them.

diff --git a/examples/rsync/fixtures/run-smoke/staging/timestamp b/examples/rsync/fixtures/run-smoke/staging/timestamp index 53e8aaa..70c5980 100644 --- a/examples/rsync/fixtures/run-smoke/staging/timestamp +++ b/examples/rsync/fixtures/run-smoke/staging/timestamp @@ -1 +1 @@ -20260908T00 +1788825600 diff --git a/examples/rsync/fixtures/run-smoke/upstream.txt b/examples/rsync/fixtures/run-smoke/upstream.txt index d25ed5c..c8856ac 100644 --- a/examples/rsync/fixtures/run-smoke/upstream.txt +++ b/examples/rsync/fixtures/run-smoke/upstream.txt @@ -1,2 +1,2 @@ a/b/f.txt 0 2026/09/08 00:00:00 -timestamp 12 2026/09/08 00:00:00 +timestamp 11 2026/09/08 00:00:00 diff --git a/examples/rsync/render.txt b/examples/rsync/render.txt index 13216a5..fa9de98 100644 --- a/examples/rsync/render.txt +++ b/examples/rsync/render.txt @@ -320,5 +320,10 @@ task: [smoke] g=$(awk -F'\t' '$1 ~ /\.tar\.gz$/ && $2 > 0 { print $1; exit }' /w function enc(s, out, j, c) { out = ""; for (j = 1; j <= length(s); j++) { c = substr(s, j, 1) out = out ((c ~ /[A-Za-z0-9\/._~-]/) ? c : sprintf("%%%02X", ord[c])) } return out } { print $0 "\t" enc($0) }' | cut -f2) && h=$(curl -fsS --connect-timeout 15 --max-time 60 --retry 6 --retry-connrefused --retry-max-time 600 -r 0-0 -D - -o /dev/null "https://example.katoptra.org/$u") && if printf '%s' "$h" | grep -qi '^content-encoding:'; then echo "$g: served with a content-encoding" >&2; exit 1; fi && echo "$g: served with no content-encoding"; fi task: [smoke] if test -f /work/examples/rsync/.run/changed.txt && test ! -f /work/examples/rsync/.run/chain; then LC_ALL=C comm -23 /work/examples/rsync/.run/changed.txt /work/examples/rsync/.run/applied.txt > /work/examples/rsync/.run/unlanded.txt; else : > /work/examples/rsync/.run/unlanded.txt; fi; cut -f1 /work/examples/rsync/.run/unlanded.txt | head -20 | while IFS= read -r p; do echo "::warning::listed upstream, never fetched: $p"; done -task: [fresh] echo "no FRESH_KEY: a source that stops moving goes unnoticed" +task: [fresh] k=timestamp; if ! awk -F'\t' -v k="$k" '$1 == k { f = 1 } END { exit !f }' /work/examples/rsync/.run/applied.txt 2>/dev/null; then echo "freshness skipped ($k is not in the state)"; else case "24" in ''|*[!0-9]*) echo "FRESH_HOURS is a whole number" >&2; exit 1 ;; esac; u=$(printf '%s\n' "$k" | awk 'BEGIN { for (i = 0; i < 256; i++) ord[sprintf("%c", i)] = i } +function enc(s, out, j, c) { out = ""; for (j = 1; j <= length(s); j++) { c = substr(s, j, 1) + out = out ((c ~ /[A-Za-z0-9\/._~-]/) ? c : sprintf("%%%02X", ord[c])) } return out } { print $0 "\t" enc($0) }' | cut -f2); t=$(curl -fsS --connect-timeout 15 --max-time 60 --retry 6 --retry-connrefused --retry-max-time 600 "https://example.katoptra.org/$u" | awk '!/^#/ { for (i = 1; i <= NF; i++) { + if ($i ~ /^[0-9]+$/ && length($i) >= 9) { print $i; exit } + if ($i ~ /^[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]-[0-9][0-9]-[0-9][0-9]$/) { + split($i, d, "-"); print d[1] "-" d[2] "-" d[3] " " d[4] ":" d[5]; exit } } }'); case "$t" in *-*) t=$(date -u -d "$t" +%s) ;; esac; test -n "$t"; a=$(( $(date -u +%s) - t )); echo "$a" > /work/examples/rsync/.run/fresh.txt; echo "$k: upstream's clock $((a / 3600)) h $((a % 3600 / 60)) min ago, limit 24 h"; test "$a" -le $(( 24 * 3600 )) || { echo "::error::$k is $((a / 3600)) h old, past FRESH_HOURS 24: upstream or SOURCE has stopped moving"; exit 1; }; fi task: [ping] test -z "$HEALTHCHECK_URL" || curl -fsS -m 10 --retry 3 -o /dev/null "$HEALTHCHECK_URL" From d3db189aec98ce1a7424918f1e007ab47298b4dc Mon Sep 17 00:00:00 2001 From: Josh Vaughen Date: Thu, 8 Oct 2026 14:16:47 -0700 Subject: [PATCH 6/7] docs(rsync): LIST_FLOOR is about 90% of the usual line count Exit 23 can hide a whole directory, and LIST_FLOOR is the only cap on a deletion list, so a floor at half the usual size lets one run delete half a mirror. The engine's comment and the README's vars table say about 90%. --- README.md | 2 +- engines/rsync.yml | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 573f0ac..e1e0185 100644 --- a/README.md +++ b/README.md @@ -421,7 +421,7 @@ has an inline default, and a mirror sets only what differs: | `CEILING_GB` | 0, no ceiling | `split` refuses a tree larger than this many decimal GB | | `BATCH_GB` | 4 | Decimal GB per batch; a larger file is a batch by itself | | `MAX_BATCHES` | 4 | Batches per run; the rest chain the next run | -| `LIST_FLOOR` | 0, no guard | A listing under this many lines is a truncated one, never a deletion list | +| `LIST_FLOOR` | 0, no guard | A listing under this many lines is a truncated one, never a deletion list. Set it to about 90% of the usual line count | | `RECONCILE`, `RECONCILE_HOURS` | `auto`, 24 | The toolbox's: see [Reconcile, by age](#reconcile-by-age) | | `RETRY_BASE` | 15 | Seconds; the retry sleeps are `RETRY_BASE * 2^i` plus jitter | | `TL`, `TL_KEY` | empty | A signed TeX Live subtree and the fingerprint that signs it; empty, no signature checks | diff --git a/engines/rsync.yml b/engines/rsync.yml index b47e745..6f6a744 100644 --- a/engines/rsync.yml +++ b/engines/rsync.yml @@ -165,7 +165,8 @@ tasks: print path "\t" size + 0 "\t" mtime }' {{.RUN}}/listing.txt \ | LC_ALL=C sort > {{.RUN}}/upstream.txt # A truncated listing must never become a deletion list. LIST_FLOOR is a line count - # the listing never legitimately falls to, about half its usual size; unset, no guard. + # that a correct listing never falls to, about 90% of its usual size. When it is + # unset, there is no guard. - '{{if .LIST_FLOOR}}test "$(wc -l < {{.RUN}}/upstream.txt)" -gt {{.LIST_FLOOR}}{{else}}echo "no LIST_FLOOR: a truncated listing is not caught here"{{end}}' state: From 7e6878232dbfbee05187bca20cedea8ad3698702 Mon Sep 17 00:00:00 2001 From: Josh Vaughen Date: Thu, 8 Oct 2026 14:16:47 -0700 Subject: [PATCH 7/7] test(proton): render list-folder and trash in the proton example Only github's render covered the two verbs, so lib's CI could not catch a break in them. The example excludes the engine's prune and defines a two-command one: list the folder stage fills, then trash nothing, since stage always writes the same file. --- examples/proton/Taskfile.yml | 7 ++++++- examples/proton/render.txt | 40 ++++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/examples/proton/Taskfile.yml b/examples/proton/Taskfile.yml index 1b73ee3..a8b8266 100644 --- a/examples/proton/Taskfile.yml +++ b/examples/proton/Taskfile.yml @@ -18,12 +18,17 @@ includes: proton: taskfile: ../../engines/proton.yml flatten: true - excludes: [stage] + excludes: [stage, prune] tasks: stage: desc: One file in one folder under STAGING, so the pipeline has something to upload cmds: - rm -rf {{.STAGING}} && mkdir -p {{.STAGING}}/example && printf 'hello\n' > {{.STAGING}}/example/hello.txt + prune: + desc: List the folder stage fills and trash nothing, since stage always writes the same file; two commands, so the render covers list-folder and trash + cmds: + - {task: list-folder, vars: {FOLDER: '$MIRROR_PROTON_DESTINATION/example', OUT: '{{.RUN}}/example.json'}} + - {task: trash, vars: {PATHS: ''}} tools: desc: Every tool the proton image promises, by version cmds: diff --git a/examples/proton/render.txt b/examples/proton/render.txt index a07388c..291c29d 100644 --- a/examples/proton/render.txt +++ b/examples/proton/render.txt @@ -96,4 +96,44 @@ print("confirm: " + verdict) sys.exit(0 if ok else 1) PY +task: [pd] rc=0; PROTON_DRIVE_CACHE_DIR=/work/examples/proton/.run/session proton-drive filesystem list -j "$MIRROR_PROTON_DESTINATION/example" > /work/examples/proton/.run/example.json || echo "[]" > /work/examples/proton/.run/example.json 2> /work/examples/proton/.run/pd.err || rc=$?; echo $rc > /work/examples/proton/.run/pd.rc +task: [session-push] echo "session: token rotated, sealing it back to the bucket" +task: [seal] mkdir -p /work/examples/proton/.run && tar -cf /work/examples/proton/.run/session.tar -C /work/examples/proton/.run/session auth-session.json clientUid.json +task: [age] test -n "$MIRROR_AGE_IDENTITY" || { echo "MIRROR_AGE_IDENTITY is unset" >&2; exit 1; } +k=/work/examples/proton/.run/age.key; mkdir -p /work/examples/proton/.run; install -m 600 /dev/null "$k" && printf '%s\n' "$MIRROR_AGE_IDENTITY" > "$k" +rc=0 +case encrypt in + encrypt) age -r "$(age-keygen -y "$k")" -o /work/examples/proton/.run/session.tar.age /work/examples/proton/.run/session.tar || rc=$? ;; + decrypt) age -d -i "$k" -o /work/examples/proton/.run/session.tar.age /work/examples/proton/.run/session.tar || rc=$? ;; + *) echo "age: MODE must be encrypt or decrypt" >&2; rc=2 ;; +esac +rm -f "$k"; exit $rc + +task: [push] s3 put /work/examples/proton/.run/session.tar.age .state/session.tar.age +task: [seal] rm -f /work/examples/proton/.run/session.tar /work/examples/proton/.run/session.tar.age +task: [session-push] sha256sum /work/examples/proton/.run/session/auth-session.json > /work/examples/proton/.run/session.sha +task: [pd] rc=$(cat /work/examples/proton/.run/pd.rc) +case " 0 " in *" $rc "*) exit 0 ;; esac +echo "proton-drive exited $rc; its stderr, $(wc -l < /work/examples/proton/.run/pd.err) lines, is in .run/pd.err" >&2; exit $rc + +task: [pd] rc=0; PROTON_DRIVE_CACHE_DIR=/work/examples/proton/.run/session proton-drive filesystem trash 2> /work/examples/proton/.run/pd.err || rc=$?; echo $rc > /work/examples/proton/.run/pd.rc +task: [session-push] echo "session: token rotated, sealing it back to the bucket" +task: [seal] mkdir -p /work/examples/proton/.run && tar -cf /work/examples/proton/.run/session.tar -C /work/examples/proton/.run/session auth-session.json clientUid.json +task: [age] test -n "$MIRROR_AGE_IDENTITY" || { echo "MIRROR_AGE_IDENTITY is unset" >&2; exit 1; } +k=/work/examples/proton/.run/age.key; mkdir -p /work/examples/proton/.run; install -m 600 /dev/null "$k" && printf '%s\n' "$MIRROR_AGE_IDENTITY" > "$k" +rc=0 +case encrypt in + encrypt) age -r "$(age-keygen -y "$k")" -o /work/examples/proton/.run/session.tar.age /work/examples/proton/.run/session.tar || rc=$? ;; + decrypt) age -d -i "$k" -o /work/examples/proton/.run/session.tar.age /work/examples/proton/.run/session.tar || rc=$? ;; + *) echo "age: MODE must be encrypt or decrypt" >&2; rc=2 ;; +esac +rm -f "$k"; exit $rc + +task: [push] s3 put /work/examples/proton/.run/session.tar.age .state/session.tar.age +task: [seal] rm -f /work/examples/proton/.run/session.tar /work/examples/proton/.run/session.tar.age +task: [session-push] sha256sum /work/examples/proton/.run/session/auth-session.json > /work/examples/proton/.run/session.sha +task: [pd] rc=$(cat /work/examples/proton/.run/pd.rc) +case " 0 " in *" $rc "*) exit 0 ;; esac +echo "proton-drive exited $rc; its stderr, $(wc -l < /work/examples/proton/.run/pd.err) lines, is in .run/pd.err" >&2; exit $rc + task: [ping] test -z "$HEALTHCHECK_URL" || curl -fsS -m 10 --retry 3 -o /dev/null "$HEALTHCHECK_URL"