diff --git a/.github/actions/toolbox/action.yml b/.github/actions/toolbox/action.yml index efe3a2b..ecfdefd 100644 --- a/.github/actions/toolbox/action.yml +++ b/.github/actions/toolbox/action.yml @@ -17,13 +17,16 @@ runs: run: | set -eu lock() { python3 "$LIB/docker/lock.py" "$1" "$LIB/toolchain.lock.toml"; } + # curl tries each download a maximum of five times. After each failure, it waits + # two times longer than the wait before, from 1 second. + get() { curl -fsSL --retry 4 --retry-all-errors "$@"; } arch="$(dpkg --print-architecture)" - curl -fsSL "$(lock task.base_url)/$(lock task.linux_${arch}.archive)" -o /tmp/task.tgz + get "$(lock task.base_url)/$(lock task.linux_${arch}.archive)" -o /tmp/task.tgz echo "$(lock task.linux_${arch}.sha256) /tmp/task.tgz" | sha256sum -c - sudo tar -xzf /tmp/task.tgz -C /usr/local/bin task task --version test "$OP" = true || exit 0 - curl -fsSL "$(lock op.base_url)/$(lock op.linux_${arch}.archive)" -o /tmp/op.zip + get "$(lock op.base_url)/$(lock op.linux_${arch}.archive)" -o /tmp/op.zip echo "$(lock op.linux_${arch}.sha256) /tmp/op.zip" | sha256sum -c - sudo unzip -q -o /tmp/op.zip op -d /usr/local/bin op --version diff --git a/.github/op-retry-check.sh b/.github/op-retry-check.sh new file mode 100644 index 0000000..4201a38 --- /dev/null +++ b/.github/op-retry-check.sh @@ -0,0 +1,56 @@ +#!/bin/sh +# The check of the op verb in toolbox.yml. A stub op and a stub sleep go first in PATH. A +# test Taskfile includes toolbox.yml and runs `run` on the host. Thus, the check needs only +# task. It needs no image and no vault. +# +# The three cases: +# - op fails two times before the command starts: the verb tries again, after 5 and 10 s +# - op starts the command, and the command fails: the verb does not start it again +# - op always fails: the verb stops after five tries. +set -eu + +lib=$(cd "$(dirname "$0")/.." && pwd) +dir=$(mktemp -d) +trap 'rm -rf "$dir"' EXIT +mkdir "$dir/bin" +: > "$dir/op.env" +cat > "$dir/Taskfile.yml" < "$dir/bin/op" <<'EOF' +#!/bin/sh +echo call >> "$CHECK_DIR/calls" +test "$(wc -l < "$CHECK_DIR/calls")" -gt "$OP_FAILS" || { echo "[ERROR] (502) Unknown" >&2; exit 1; } +shift 3 +exec "$@" +EOF +cat > "$dir/bin/sleep" <<'EOF' +#!/bin/sh +echo "$1" >> "$CHECK_DIR/sleeps" +EOF +chmod +x "$dir/bin/op" "$dir/bin/sleep" + +# try OP_FAILS CMD prints " ". +try() { + rm -f "$dir/calls" "$dir/sleeps" + rc=0 + (cd "$dir" && CHECK_DIR="$dir" OP_FAILS="$1" PATH="$dir/bin:$PATH" task op -- sh -c "$2" > /dev/null 2>&1) || rc=$? + printf '%s %s %s\n' "$(test "$rc" = 0 && echo ok || echo fail)" "$(wc -l < "$dir/calls" | tr -d ' ')" "$(cat "$dir/sleeps" 2> /dev/null | xargs)" +} + +fails=0 +expect() { + got=$(try "$1" "$2") + test "$got" = "$3" || { echo "op-retry: OP_FAILS=$1 '$2' gave '$got', not '$3'" >&2; fails=1; } +} +expect 2 true 'ok 3 5 10' +expect 0 'exit 7' 'fail 1 ' +expect 9 true 'fail 5 5 10 20 40' +test "$fails" = 0 +echo "op-retry: PASS" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dfe0310..90eb9cc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,6 +23,9 @@ jobs: sh .github/validate-vars.sh --check sh .github/chain-file.sh --check sh docker/gpg-gate-check.sh + - name: The op verb tries again only when the command did not start + if: matrix.variant == 'rsync' + run: sh .github/op-retry-check.sh - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 # Actions keeps the layers in its cache, with one scope for each variant. release # reads the same scope. Thus, for a tag, the build makes new layers only for the diff --git a/CLAUDE.md b/CLAUDE.md index 4cfe134..26a9ac4 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -110,6 +110,7 @@ Read that section before you write. cd examples/rsync && task image-build && task run -- task tools && task check && task run -- task offline cd examples/proton && task image-build && task run -- task tools && task check && task run -- task offline sh .github/validate-vars.sh --check && sh .github/chain-file.sh --check && sh docker/gpg-gate-check.sh +sh .github/op-retry-check.sh ``` After a change to a verb, run `task render-update` to update the `render.txt` files. diff --git a/README.md b/README.md index 5c937ca..f54b540 100644 --- a/README.md +++ b/README.md @@ -147,7 +147,9 @@ The diagram does not show these four items: not read the vault a second time. If the runner stops a run, at the timeout or because a person cancels it, the run does not start its failure path. Then the workflow runs `failed` again from the host, and this reads the vault one more time. Such a run does - not occur frequently. + not occur frequently. If `op` fails before it starts the pipeline, for example with a + 502 from 1Password, the `op` verb reads the vault again. It tries a maximum of five + times. - **In the container, task does not use the network for its Taskfiles.** The image sets `TASK_REMOTE_OFFLINE=1`. The `.task/remote` cache of the mirror goes into the container with the repository. @@ -190,7 +192,7 @@ not override them. | `render-update` | `render`, then it copies the render to `render.txt` | | `render` | `task --dry --force pipeline` in the image, with an empty `.run`. It writes the result to `.run/render.txt`. No secret goes into the container | | `run -- ` | Runs a command in the image, with the repository at `/work` | -| `op -- ` | `run`, in `op run --env-file=op.env` if the repository has `op.env` | +| `op -- ` | `run`, in `op run --env-file=op.env` if the repository has `op.env`. If `op` fails before it starts the command, the verb tries again, a maximum of five times. If the command starts, the verb does not start it again | | `image` | Downloads `IMAGE`. While the host has the image, it does not download it again | | `image-build` | Builds `IMAGE` from `LIB_DIR/docker/.Dockerfile`. The default `LIB_DIR` is `../lib` | | `image-clean` | Removes `IMAGE` | @@ -1117,8 +1119,8 @@ verb sets `offline: true`. The job then also runs `task run -- task offline` in `.github/actions/toolbox` installs go-task on the runner. Unless `op: 'false'`, it also installs the 1Password CLI. It uses the versions and the checksums in -`toolchain.lock.toml`. With this action, a runner can run `task sync`. All other tools run -in the image. +`toolchain.lock.toml`. curl tries each download a maximum of five times. With this action, +a runner can run `task sync`. All other tools run in the image. ## Using it in a mirror diff --git a/toolbox.yml b/toolbox.yml index e1cc16c..8da81ee 100644 --- a/toolbox.yml +++ b/toolbox.yml @@ -139,9 +139,23 @@ tasks: cmds: # With no op.env, the secrets are in the environment before this command starts (for # example, GitHub secrets). + # + # op can fail before it starts the command, for example with a 502 from 1Password. + # Then the verb tries again. It tries a maximum of five times, and it waits 5, 10, 20 + # and 40 seconds between the tries. The command first makes the file $m. If $m + # exists, op started the command. Then the exit of the verb is the exit of the + # command, and the verb does not start the command again. + # .github/op-retry-check.sh is the check. - | - if test -f op.env; then op run --env-file=op.env -- task run -- {{.CMD}} - else task run -- {{.CMD}}; fi + if test ! -f op.env; then task run -- {{.CMD}}; else + m=$(mktemp -u) + for i in 1 2 3 4 5; do + rc=0; op run --env-file=op.env -- sh -c ': > "$1"; shift; exec "$@"' sh "$m" task run -- {{.CMD}} || rc=$? + if test $rc = 0 || test -e "$m" || test $i = 5; then break; fi + s=$((5 * 2 ** (i - 1))); echo "op run exit $rc, the command did not start, retry $i in ${s}s" >&2; sleep $s + done + rm -f "$m"; exit $rc + fi clean: desc: Delete .run, the engine's staging tree and the Taskfile cache; nothing else, so a fixture kept in .git/info/exclude survives