From 30c6303bd33ea365fa35bf2d0c43023ec7d5061c Mon Sep 17 00:00:00 2001 From: Josh Vaughen Date: Fri, 16 Oct 2026 08:10:58 -0700 Subject: [PATCH] fix(toolbox): retry op run and the toolbox action downloads ctan run 38917887841 (2026-10-16 14:42 UTC) failed one second in: op run got a 502 from 1Password before the pipeline started. op was the one setup call with no retry, so one blip cost the hourly slot. The failure path runs inside op run, so the run also sent no /fail ping. - toolbox.yml op: up to five tries, waits 5, 10, 20, 40 s. The command first touches a marker file; once it exists, op started the command, so its exit stands and the pipeline is never started twice. - toolbox action: the go-task and op downloads use curl --retry 4 --retry-all-errors (five tries, waits 1, 2, 4, 8 s). - .github/op-retry-check.sh: stub op and sleep on PATH, run on the host. Three cases: op fails twice then starts, the command fails (no retry), op always fails (five tries). Runs in ci.yml. --- .github/actions/toolbox/action.yml | 7 ++-- .github/op-retry-check.sh | 56 ++++++++++++++++++++++++++++++ .github/workflows/ci.yml | 3 ++ CLAUDE.md | 1 + README.md | 10 +++--- toolbox.yml | 18 ++++++++-- 6 files changed, 87 insertions(+), 8 deletions(-) create mode 100644 .github/op-retry-check.sh diff --git a/.github/actions/toolbox/action.yml b/.github/actions/toolbox/action.yml index efe3a2b..ecfdefd 100644 --- a/.github/actions/toolbox/action.yml +++ b/.github/actions/toolbox/action.yml @@ -17,13 +17,16 @@ runs: run: | set -eu lock() { python3 "$LIB/docker/lock.py" "$1" "$LIB/toolchain.lock.toml"; } + # curl tries each download a maximum of five times. After each failure, it waits + # two times longer than the wait before, from 1 second. + get() { curl -fsSL --retry 4 --retry-all-errors "$@"; } arch="$(dpkg --print-architecture)" - curl -fsSL "$(lock task.base_url)/$(lock task.linux_${arch}.archive)" -o /tmp/task.tgz + get "$(lock task.base_url)/$(lock task.linux_${arch}.archive)" -o /tmp/task.tgz echo "$(lock task.linux_${arch}.sha256) /tmp/task.tgz" | sha256sum -c - sudo tar -xzf /tmp/task.tgz -C /usr/local/bin task task --version test "$OP" = true || exit 0 - curl -fsSL "$(lock op.base_url)/$(lock op.linux_${arch}.archive)" -o /tmp/op.zip + get "$(lock op.base_url)/$(lock op.linux_${arch}.archive)" -o /tmp/op.zip echo "$(lock op.linux_${arch}.sha256) /tmp/op.zip" | sha256sum -c - sudo unzip -q -o /tmp/op.zip op -d /usr/local/bin op --version diff --git a/.github/op-retry-check.sh b/.github/op-retry-check.sh new file mode 100644 index 0000000..4201a38 --- /dev/null +++ b/.github/op-retry-check.sh @@ -0,0 +1,56 @@ +#!/bin/sh +# The check of the op verb in toolbox.yml. A stub op and a stub sleep go first in PATH. A +# test Taskfile includes toolbox.yml and runs `run` on the host. Thus, the check needs only +# task. It needs no image and no vault. +# +# The three cases: +# - op fails two times before the command starts: the verb tries again, after 5 and 10 s +# - op starts the command, and the command fails: the verb does not start it again +# - op always fails: the verb stops after five tries. +set -eu + +lib=$(cd "$(dirname "$0")/.." && pwd) +dir=$(mktemp -d) +trap 'rm -rf "$dir"' EXIT +mkdir "$dir/bin" +: > "$dir/op.env" +cat > "$dir/Taskfile.yml" < "$dir/bin/op" <<'EOF' +#!/bin/sh +echo call >> "$CHECK_DIR/calls" +test "$(wc -l < "$CHECK_DIR/calls")" -gt "$OP_FAILS" || { echo "[ERROR] (502) Unknown" >&2; exit 1; } +shift 3 +exec "$@" +EOF +cat > "$dir/bin/sleep" <<'EOF' +#!/bin/sh +echo "$1" >> "$CHECK_DIR/sleeps" +EOF +chmod +x "$dir/bin/op" "$dir/bin/sleep" + +# try OP_FAILS CMD prints " ". +try() { + rm -f "$dir/calls" "$dir/sleeps" + rc=0 + (cd "$dir" && CHECK_DIR="$dir" OP_FAILS="$1" PATH="$dir/bin:$PATH" task op -- sh -c "$2" > /dev/null 2>&1) || rc=$? + printf '%s %s %s\n' "$(test "$rc" = 0 && echo ok || echo fail)" "$(wc -l < "$dir/calls" | tr -d ' ')" "$(cat "$dir/sleeps" 2> /dev/null | xargs)" +} + +fails=0 +expect() { + got=$(try "$1" "$2") + test "$got" = "$3" || { echo "op-retry: OP_FAILS=$1 '$2' gave '$got', not '$3'" >&2; fails=1; } +} +expect 2 true 'ok 3 5 10' +expect 0 'exit 7' 'fail 1 ' +expect 9 true 'fail 5 5 10 20 40' +test "$fails" = 0 +echo "op-retry: PASS" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dfe0310..90eb9cc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,6 +23,9 @@ jobs: sh .github/validate-vars.sh --check sh .github/chain-file.sh --check sh docker/gpg-gate-check.sh + - name: The op verb tries again only when the command did not start + if: matrix.variant == 'rsync' + run: sh .github/op-retry-check.sh - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 # Actions keeps the layers in its cache, with one scope for each variant. release # reads the same scope. Thus, for a tag, the build makes new layers only for the diff --git a/CLAUDE.md b/CLAUDE.md index 4cfe134..26a9ac4 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -110,6 +110,7 @@ Read that section before you write. cd examples/rsync && task image-build && task run -- task tools && task check && task run -- task offline cd examples/proton && task image-build && task run -- task tools && task check && task run -- task offline sh .github/validate-vars.sh --check && sh .github/chain-file.sh --check && sh docker/gpg-gate-check.sh +sh .github/op-retry-check.sh ``` After a change to a verb, run `task render-update` to update the `render.txt` files. diff --git a/README.md b/README.md index 5c937ca..f54b540 100644 --- a/README.md +++ b/README.md @@ -147,7 +147,9 @@ The diagram does not show these four items: not read the vault a second time. If the runner stops a run, at the timeout or because a person cancels it, the run does not start its failure path. Then the workflow runs `failed` again from the host, and this reads the vault one more time. Such a run does - not occur frequently. + not occur frequently. If `op` fails before it starts the pipeline, for example with a + 502 from 1Password, the `op` verb reads the vault again. It tries a maximum of five + times. - **In the container, task does not use the network for its Taskfiles.** The image sets `TASK_REMOTE_OFFLINE=1`. The `.task/remote` cache of the mirror goes into the container with the repository. @@ -190,7 +192,7 @@ not override them. | `render-update` | `render`, then it copies the render to `render.txt` | | `render` | `task --dry --force pipeline` in the image, with an empty `.run`. It writes the result to `.run/render.txt`. No secret goes into the container | | `run -- ` | Runs a command in the image, with the repository at `/work` | -| `op -- ` | `run`, in `op run --env-file=op.env` if the repository has `op.env` | +| `op -- ` | `run`, in `op run --env-file=op.env` if the repository has `op.env`. If `op` fails before it starts the command, the verb tries again, a maximum of five times. If the command starts, the verb does not start it again | | `image` | Downloads `IMAGE`. While the host has the image, it does not download it again | | `image-build` | Builds `IMAGE` from `LIB_DIR/docker/.Dockerfile`. The default `LIB_DIR` is `../lib` | | `image-clean` | Removes `IMAGE` | @@ -1117,8 +1119,8 @@ verb sets `offline: true`. The job then also runs `task run -- task offline` in `.github/actions/toolbox` installs go-task on the runner. Unless `op: 'false'`, it also installs the 1Password CLI. It uses the versions and the checksums in -`toolchain.lock.toml`. With this action, a runner can run `task sync`. All other tools run -in the image. +`toolchain.lock.toml`. curl tries each download a maximum of five times. With this action, +a runner can run `task sync`. All other tools run in the image. ## Using it in a mirror diff --git a/toolbox.yml b/toolbox.yml index e1cc16c..8da81ee 100644 --- a/toolbox.yml +++ b/toolbox.yml @@ -139,9 +139,23 @@ tasks: cmds: # With no op.env, the secrets are in the environment before this command starts (for # example, GitHub secrets). + # + # op can fail before it starts the command, for example with a 502 from 1Password. + # Then the verb tries again. It tries a maximum of five times, and it waits 5, 10, 20 + # and 40 seconds between the tries. The command first makes the file $m. If $m + # exists, op started the command. Then the exit of the verb is the exit of the + # command, and the verb does not start the command again. + # .github/op-retry-check.sh is the check. - | - if test -f op.env; then op run --env-file=op.env -- task run -- {{.CMD}} - else task run -- {{.CMD}}; fi + if test ! -f op.env; then task run -- {{.CMD}}; else + m=$(mktemp -u) + for i in 1 2 3 4 5; do + rc=0; op run --env-file=op.env -- sh -c ': > "$1"; shift; exec "$@"' sh "$m" task run -- {{.CMD}} || rc=$? + if test $rc = 0 || test -e "$m" || test $i = 5; then break; fi + s=$((5 * 2 ** (i - 1))); echo "op run exit $rc, the command did not start, retry $i in ${s}s" >&2; sleep $s + done + rm -f "$m"; exit $rc + fi clean: desc: Delete .run, the engine's staging tree and the Taskfile cache; nothing else, so a fixture kept in .git/info/exclude survives