fix(toolbox): retry op run and the toolbox action downloads

.github/actions/toolbox/action.yml

@@ -17,13 +17,16 @@ runs:
       run: |
         set -eu
         lock() { python3 "$LIB/docker/lock.py" "$1" "$LIB/toolchain.lock.toml"; }
+        # curl tries each download a maximum of five times. After each failure, it waits
+        # two times longer than the wait before, from 1 second.
+        get() { curl -fsSL --retry 4 --retry-all-errors "$@"; }
         arch="$(dpkg --print-architecture)"
-        curl -fsSL "$(lock task.base_url)/$(lock task.linux_${arch}.archive)" -o /tmp/task.tgz
+        get "$(lock task.base_url)/$(lock task.linux_${arch}.archive)" -o /tmp/task.tgz
         echo "$(lock task.linux_${arch}.sha256)  /tmp/task.tgz" | sha256sum -c -
         sudo tar -xzf /tmp/task.tgz -C /usr/local/bin task
         task --version
         test "$OP" = true || exit 0
-        curl -fsSL "$(lock op.base_url)/$(lock op.linux_${arch}.archive)" -o /tmp/op.zip
+        get "$(lock op.base_url)/$(lock op.linux_${arch}.archive)" -o /tmp/op.zip
         echo "$(lock op.linux_${arch}.sha256)  /tmp/op.zip" | sha256sum -c -
         sudo unzip -q -o /tmp/op.zip op -d /usr/local/bin
         op --version

.github/op-retry-check.sh

@@ -0,0 +1,56 @@
+#!/bin/sh
+# The check of the op verb in toolbox.yml. A stub op and a stub sleep go first in PATH. A
+# test Taskfile includes toolbox.yml and runs `run` on the host. Thus, the check needs only
+# task. It needs no image and no vault.
+#
+# The three cases:
+#   - op fails two times before the command starts: the verb tries again, after 5 and 10 s
+#   - op starts the command, and the command fails: the verb does not start it again
+#   - op always fails: the verb stops after five tries.
+set -eu
+
+lib=$(cd "$(dirname "$0")/.." && pwd)
+dir=$(mktemp -d)
+trap 'rm -rf "$dir"' EXIT
+mkdir "$dir/bin"
+: > "$dir/op.env"
+cat > "$dir/Taskfile.yml" <<EOF
+version: '3'
+includes:
+  toolbox: {taskfile: $lib/toolbox.yml, flatten: true, excludes: [run], vars: {NAME: check, DESC: check}}
+tasks:
+  run:
+    cmds: ['{{range .CLI_ARGS_LIST}}{{shellQuote .}} {{end}}']
+EOF
+# The stub op counts its calls. The first OP_FAILS calls fail before the command starts.
+cat > "$dir/bin/op" <<'EOF'
+#!/bin/sh
+echo call >> "$CHECK_DIR/calls"
+test "$(wc -l < "$CHECK_DIR/calls")" -gt "$OP_FAILS" || { echo "[ERROR] (502) Unknown" >&2; exit 1; }
+shift 3
+exec "$@"
+EOF
+cat > "$dir/bin/sleep" <<'EOF'
+#!/bin/sh
+echo "$1" >> "$CHECK_DIR/sleeps"
+EOF
+chmod +x "$dir/bin/op" "$dir/bin/sleep"
+
+# try OP_FAILS CMD prints "<ok|fail> <op calls> <waits>".
+try() {
+  rm -f "$dir/calls" "$dir/sleeps"
+  rc=0
+  (cd "$dir" && CHECK_DIR="$dir" OP_FAILS="$1" PATH="$dir/bin:$PATH" task op -- sh -c "$2" > /dev/null 2>&1) || rc=$?
+  printf '%s %s %s\n' "$(test "$rc" = 0 && echo ok || echo fail)" "$(wc -l < "$dir/calls" | tr -d ' ')" "$(cat "$dir/sleeps" 2> /dev/null | xargs)"
+}
+
+fails=0
+expect() {
+  got=$(try "$1" "$2")
+  test "$got" = "$3" || { echo "op-retry: OP_FAILS=$1 '$2' gave '$got', not '$3'" >&2; fails=1; }
+}
+expect 2 true 'ok 3 5 10'
+expect 0 'exit 7' 'fail 1 '
+expect 9 true 'fail 5 5 10 20 40'
+test "$fails" = 0
+echo "op-retry: PASS"

.github/workflows/ci.yml

@@ -23,6 +23,9 @@ jobs:
           sh .github/validate-vars.sh --check
           sh .github/chain-file.sh --check
           sh docker/gpg-gate-check.sh
+      - name: The op verb tries again only when the command did not start
+        if: matrix.variant == 'rsync'
+        run: sh .github/op-retry-check.sh
       - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1
       # Actions keeps the layers in its cache, with one scope for each variant. release
       # reads the same scope. Thus, for a tag, the build makes new layers only for the

CLAUDE.md

@@ -110,6 +110,7 @@ Read that section before you write.
 cd examples/rsync  && task image-build && task run -- task tools && task check && task run -- task offline
 cd examples/proton && task image-build && task run -- task tools && task check && task run -- task offline
 sh .github/validate-vars.sh --check && sh .github/chain-file.sh --check && sh docker/gpg-gate-check.sh
+sh .github/op-retry-check.sh
 ```
 
 After a change to a verb, run `task render-update` to update the `render.txt` files.

README.md

@@ -147,7 +147,9 @@ The diagram does not show these four items:
   not read the vault a second time. If the runner stops a run, at the timeout or because a
   person cancels it, the run does not start its failure path. Then the workflow runs
   `failed` again from the host, and this reads the vault one more time. Such a run does
-  not occur frequently.
+  not occur frequently. If `op` fails before it starts the pipeline, for example with a
+  502 from 1Password, the `op` verb reads the vault again. It tries a maximum of five
+  times.
 - **In the container, task does not use the network for its Taskfiles.** The image sets
   `TASK_REMOTE_OFFLINE=1`. The `.task/remote` cache of the mirror goes into the container
   with the repository.
@@ -190,7 +192,7 @@ not override them.
 | `render-update` | `render`, then it copies the render to `render.txt` |
 | `render` | `task --dry --force pipeline` in the image, with an empty `.run`. It writes the result to `.run/render.txt`. No secret goes into the container |
 | `run -- <cmd>` | Runs a command in the image, with the repository at `/work` |
-| `op -- <cmd>` | `run`, in `op run --env-file=op.env` if the repository has `op.env` |
+| `op -- <cmd>` | `run`, in `op run --env-file=op.env` if the repository has `op.env`. If `op` fails before it starts the command, the verb tries again, a maximum of five times. If the command starts, the verb does not start it again |
 | `image` | Downloads `IMAGE`. While the host has the image, it does not download it again |
 | `image-build` | Builds `IMAGE` from `LIB_DIR/docker/<variant>.Dockerfile`. The default `LIB_DIR` is `../lib` |
 | `image-clean` | Removes `IMAGE` |
@@ -1117,8 +1119,8 @@ verb sets `offline: true`. The job then also runs `task run -- task offline` in
 
 `.github/actions/toolbox` installs go-task on the runner. Unless `op: 'false'`, it also
 installs the 1Password CLI. It uses the versions and the checksums in
-`toolchain.lock.toml`. With this action, a runner can run `task sync`. All other tools run
-in the image.
+`toolchain.lock.toml`. curl tries each download a maximum of five times. With this action,
+a runner can run `task sync`. All other tools run in the image.
 
 ## Using it in a mirror
 

toolbox.yml

@@ -139,9 +139,23 @@ tasks:
     cmds:
       # With no op.env, the secrets are in the environment before this command starts (for
       # example, GitHub secrets).
+      #
+      # op can fail before it starts the command, for example with a 502 from 1Password.
+      # Then the verb tries again. It tries a maximum of five times, and it waits 5, 10, 20
+      # and 40 seconds between the tries. The command first makes the file $m. If $m
+      # exists, op started the command. Then the exit of the verb is the exit of the
+      # command, and the verb does not start the command again.
+      # .github/op-retry-check.sh is the check.
       - |
-        if test -f op.env; then op run --env-file=op.env -- task run -- {{.CMD}}
-        else task run -- {{.CMD}}; fi
+        if test ! -f op.env; then task run -- {{.CMD}}; else
+          m=$(mktemp -u)
+          for i in 1 2 3 4 5; do
+            rc=0; op run --env-file=op.env -- sh -c ': > "$1"; shift; exec "$@"' sh "$m" task run -- {{.CMD}} || rc=$?
+            if test $rc = 0 || test -e "$m" || test $i = 5; then break; fi
+            s=$((5 * 2 ** (i - 1))); echo "op run exit $rc, the command did not start, retry $i in ${s}s" >&2; sleep $s
+          done
+          rm -f "$m"; exit $rc
+        fi
 
   clean:
     desc: Delete .run, the engine's staging tree and the Taskfile cache; nothing else, so a fixture kept in .git/info/exclude survives