From 02f337470f1ed14d6195b974ac338a0b48041486 Mon Sep 17 00:00:00 2001 From: noctuum <25441068+noctuum@users.noreply.github.com> Date: Fri, 16 Oct 2026 18:24:51 +0500 Subject: [PATCH 1/3] fix(scripts): run no strategy in publish --- .github/workflows/check-updates.yml | 3 +-- docs/WORKFLOW.md | 13 ++++++------- scripts/check-all.sh | 12 +++++++----- scripts/publish-pkg.sh | 6 +++--- tests/publish.bats | 9 ++++++--- tests/workflows.bats | 8 ++++++++ 6 files changed, 31 insertions(+), 20 deletions(-) diff --git a/.github/workflows/check-updates.yml b/.github/workflows/check-updates.yml index 41a5fef..35daf61 100644 --- a/.github/workflows/check-updates.yml +++ b/.github/workflows/check-updates.yml @@ -325,9 +325,8 @@ jobs: env: AUR_GIT_NAME: ${{ secrets.AUR_GIT_NAME }} AUR_GIT_EMAIL: ${{ secrets.AUR_GIT_EMAIL }} - GITHUB_TOKEN: ${{ github.token }} run: | - sudo --preserve-env=GITHUB_TOKEN -u builder env \ + sudo -u builder env \ GPG_KEY_ID="${GPG_KEY_ID:-}" \ GIT_AUTHOR_NAME="${AUR_GIT_NAME:-aurtomator}" \ GIT_AUTHOR_EMAIL="${AUR_GIT_EMAIL:-aurtomator@users.noreply.github.com}" \ diff --git a/docs/WORKFLOW.md b/docs/WORKFLOW.md index 102c8e4..2874b38 100644 --- a/docs/WORKFLOW.md +++ b/docs/WORKFLOW.md @@ -47,9 +47,8 @@ flowchart TD V2 -->|failure| V3[take the update out, write update_failed] V2 --> Q[publish job: SSH key, GPG key, write token] V3 --> Q - Q --> R[check-all.sh --publish: statuses of the check job] - R --> R1[run strategy again] - R1 --> R2[publish-pkg.sh: check the prepared files against the AUR copy] + Q --> R[check-all.sh --publish: statuses and versions of the check job] + R --> R2[publish-pkg.sh: check the prepared files against the AUR copy] R2 --> R3[git commit + push to AUR] R3 --> K[update-readme.sh] K --> L[commit via GitHub API] @@ -63,7 +62,7 @@ flowchart TD - **One YAML per package.** Adding or removing a package is one file. - **Strategy-based version detection.** Each type of upstream source has its own script. A new source is a new script plus its entry in `scripts/validate-pkg.sh`. - **Patch, do not generate.** aurtomator trusts your PKGBUILD and changes only `pkgver`, `pkgrel`, checksums and `.SRCINFO`. Build steps, dependencies and install files stay untouched. -- **Four jobs, one with secrets.** A PKGBUILD is code that every step reading it executes. `check` runs the strategies with the read-only workflow token, executes no PKGBUILD and decides what the run does. The jobs after it take the status of every package from `check`. `prepare` runs the PKGBUILD steps without secrets and without a token, and hands over only prepared files and update failures, because code from one PKGBUILD can write the files of every package there. `verify` downloads the sources of each prepared update and checks its checksums, without secrets, so code from one PKGBUILD cannot forge the checksums of another package. `publish` holds the SSH key, the GPG key and the write token, never executes a PKGBUILD and never downloads a source, and checks what `verify` passes on before it pushes. See [4.4](#44-prepare-and-publish). +- **Four jobs, one with secrets.** A PKGBUILD is code that every step reading it executes. `check` runs the strategies with the read-only workflow token, executes no PKGBUILD and decides what the run does. The jobs after it take the status of every package from `check`. `prepare` runs the PKGBUILD steps without secrets and without a token, and hands over only prepared files and update failures, because code from one PKGBUILD can write the files of every package there. `verify` downloads the sources of each prepared update and checks its checksums, without secrets, so code from one PKGBUILD cannot forge the checksums of another package. `publish` holds the SSH key, the GPG key and the write token, never executes a PKGBUILD, runs no strategy and never downloads a source, and checks what `verify` passes on before it pushes. See [4.4](#44-prepare-and-publish). - **Failure isolation.** A broken package does not block the others. Each failure is logged for its package and opens a GitHub Issue. Each package has a time budget, so one that hangs does not hold up the run either. ## 2. Package YAML Format @@ -633,7 +632,7 @@ A PKGBUILD is a bash script, and every step that reads it executes it: the audit **Publish**, `publish-pkg.sh /`, never sources the PKGBUILD and never runs `makepkg`. It treats the prepared files as untrusted: 1. The prepared directory must not be a symlink, `PKGBUILD`, `.SRCINFO` and `base` must be regular files, and `base` a commit id. -2. When the prepared `pkgver` is not ``, upstream moved after the update was prepared: exit code 3. +2. The prepared `pkgver` must be ``. `check-all.sh --publish` passes the version the check run found, and the prepare run was given the same one, so another `pkgver` stops the publish with exit code 1. 3. Clone over SSH. When `HEAD` is not `base`, the AUR repo moved after the update was prepared: exit code 3. `PKGBUILD` and `.SRCINFO` in the clone must be regular files, not symlinks. 4. The prepared `PKGBUILD` may differ from the AUR copy only in `pkgver=`, which must be ``, in `pkgrel=`, which must be `1`, and in the values of the checksum arrays. The arrays keep their names and lengths, hold only quoted hex checksums or `SKIP`, and have a `SKIP` exactly where the AUR copy has one. Comments and line continuations may stand between the checksums, read as bash reads them: a `#` at the start of a line, right after the opening parenthesis, or after a space or a tab comments out the rest of the line, and a backslash that ends a line without a comment, as a word of its own, continues the array on the next line. A backslash on the line that closes the array stops the publish, because bash joins the next line to the assignment. 5. Each `source` line of the prepared `.SRCINFO`, with or without an arch suffix, must be the line of the AUR copy with the old version replaced by ``. A source that is `SKIP` in every checksum array of its arch may also change the digits in the name it is saved under, the part before `::`, as `neovim-20261014.tar.gz` becomes `neovim-20261016.tar.gz` in a PKGBUILD that names a nightly download by `$(date -u +%Y%m%d)`. Its URL after `::` must still be the AUR URL with the version replaced. Verify never downloads a `SKIP` source, and makepkg takes the sources from the `PKGBUILD`. Any other line must either equal the line of the AUR copy or be that line with the version replaced in the same way, because a license, a dependency or a description can hold the old version without taking it from `pkgver`, as `GPL-2.0` does in a package at version `2.0`. The replacement also covers three forms a PKGBUILD often derives from `pkgver`: dots turned into underscores (`1_86_0`), the part before the first `+` (`0.13.0` of `0.13.0+dev+1853+g6edb38ca98`), and the version without its last component (`6.19` of `6.19.0`, as in `frameworks/${pkgver%.*}/` of KDE Frameworks). The last form counts only while it keeps a dot. Each part of a line is replaced at most once, so `5.1` inside an already replaced `5.10.0` stays. A line built from any other form of the version, a lone major version for example, stops the publish, and so does a line that has the version replaced in one place and kept in another. The package then needs a manual update. `pkgrel` and the checksums follow the rules of step 4. @@ -643,7 +642,7 @@ A PKGBUILD is a bash script, and every step that reads it executes it: the audit `check-all.sh --verify ` runs `verify-pkg.sh` for every package in `packages/` that has a directory in `/prepared/`, and runs no strategy. An update that fails is removed from `/prepared/`, its status in `/status/` becomes `update_failed`, and `/status/_error_count` grows by one. Exit code 3 counts as a failure as well: the prepare run writes `base` for every package, so a moved AUR repo looks the same as a wrong `base`. The next run prepares the package again. -`check-all.sh --publish ` takes the statuses of the check run from `.status/`, as `--prepare` does, and writes `.status/` anew. `up_to_date` and `check_failed` stay as they are. A missing status, and one the check run does not write, become `check_failed`. For a package with `new_version` or `available_downgrade` and a directory in `/prepared/`, it runs the strategy once more and publishes with the version the strategy reports now. Exit code 3 leaves the status at `new_version` or `available_downgrade` without an error, and the next run prepares the package again. A package with `new_version` or `available_downgrade` and no prepared directory gets the `update_failed` status that the prepare or the verify run wrote for it to `/status/`, or `update_failed: prepare and verify handed over no update`. Nothing else is taken from `/status/`, and only a regular file without a NUL byte: the prepare run executes PKGBUILDs, and code from one of them can write the files of every package. `_error_count` is the number of `check_failed` and `update_failed` statuses. +`check-all.sh --publish ` takes the statuses of the check run from `.status/`, as `--prepare` does, and writes `.status/` anew. `up_to_date` and `check_failed` stay as they are. A missing status, and one the check run does not write, become `check_failed`. For a package with `new_version` or `available_downgrade` and a directory in `/prepared/`, it publishes the version in that status and runs no strategy, so nothing contacts an upstream host while the AUR and GPG keys are in place. Exit code 3, an AUR repo that moved, leaves the status at `new_version` or `available_downgrade` without an error, and the next run prepares the package again. A package with `new_version` or `available_downgrade` and no prepared directory gets the `update_failed` status that the prepare or the verify run wrote for it to `/status/`, or `update_failed: prepare and verify handed over no update`. Nothing else is taken from `/status/`, and only a regular file without a NUL byte: the prepare run executes PKGBUILDs, and code from one of them can write the files of every package. `_error_count` is the number of `check_failed` and `update_failed` statuses. ## 5. CI Workflow: check-updates.yml @@ -704,7 +703,7 @@ Every job starts with **Checkout:** `actions/checkout`, pinned by commit SHA lik 4. **Setup workspace:** git identity for `builder` from the `AUR_GIT_NAME` and `AUR_GIT_EMAIL` secrets, default `aurtomator`. 5. **Setup AUR SSH:** writes `AUR_SSH_KEY` to `/home/builder/.ssh/aur`, writes the Ed25519 host key that publishes to `known_hosts`, and writes an SSH config with `StrictHostKeyChecking yes` and connection multiplexing. A host that answers with another key fails with `Host key verification failed`. When the AUR changes its host key, update the key in `check-updates.yml`. 6. **Setup GPG:** with `GPG_SIGNING_KEY` set, imports the key for `builder`, passes `GPG_KEY_ID` to the later steps and enables commit signing. Without it, the step is skipped. -7. **Publish updates:** runs `check-all.sh --publish handoff` as `builder`, with `GPG_KEY_ID`, `GIT_AUTHOR_NAME`, `GIT_AUTHOR_EMAIL` and `GITHUB_TOKEN` set to the workflow token. It keeps the statuses of `check` and checks the prepared files before it pushes, see [4.4](#44-prepare-and-publish). +7. **Publish updates:** runs `check-all.sh --publish handoff` as `builder`, with `GPG_KEY_ID`, `GIT_AUTHOR_NAME` and `GIT_AUTHOR_EMAIL`, and without `GITHUB_TOKEN`, as it runs no strategy. It keeps the statuses of `check` and checks the prepared files before it pushes, see [4.4](#44-prepare-and-publish). 8. **Update README status table:** runs `update-readme.sh` as `builder`. 9. **Commit changes via API:** see [5.5](#55-verified-commit). 10. **Create issues for failed packages:** see [5.6](#56-failure-issues). diff --git a/scripts/check-all.sh b/scripts/check-all.sh index baaea95..d7ec6b1 100755 --- a/scripts/check-all.sh +++ b/scripts/check-all.sh @@ -12,7 +12,8 @@ # With --verify : runs verify-pkg.sh for each package in /prepared # and takes out the ones that fail, without running the strategies # With --publish : keeps the statuses of the check run before it, and -# runs publish-pkg.sh for each package that run found outdated +# runs publish-pkg.sh for each package that run found outdated, with the +# version that run found and without running the strategies # With --dry-run: runs update-pkg.sh or publish-pkg.sh in dry-run mode # # Writes per-package status to .status/ directory for README generation @@ -140,8 +141,9 @@ fi STATUS_DIR="${AURTOMATOR_DIR}/.status" # --prepare and --publish take the statuses of the check run before them, -# read here before anything else runs. --prepare runs code from PKGBUILDs, -# so the workflow gives it no GITHUB_TOKEN and it runs no strategy. +# read here before anything else runs, and run no strategy: --prepare runs +# code from PKGBUILDs without GITHUB_TOKEN, and --publish holds the AUR and +# GPG keys. declare -A CHECKED=() if [[ "$MODE" == "prepare" || "$MODE" == "publish" ]] && [[ ! -L "$STATUS_DIR" ]]; then for status_file in "$STATUS_DIR"/*; do @@ -232,7 +234,7 @@ for pkg_file in $(printf '%s\n' "${AURTOMATOR_DIR}"/packages/*.yml | sort); do fi ((checked++)) || true - if [[ "$MODE" == "prepare" ]]; then + if [[ "$MODE" == "prepare" || "$MODE" == "publish" ]]; then check_cmd=(checked_version "$pkg_name") else check_cmd=(bounded "$CHECK_TIMEOUT" "${AURTOMATOR_DIR}/scripts/check-update.sh" "$pkg_name") @@ -316,7 +318,7 @@ for pkg_file in $(printf '%s\n' "${AURTOMATOR_DIR}"/packages/*.yml | sort); do echo "up_to_date" >"${STATUS_DIR}/${pkg_name}" ;; publish:3) - # The AUR or upstream moved after prepare; the next run prepares again + # The AUR moved after prepare; the next run prepares again echo "$available" >"${STATUS_DIR}/${pkg_name}" ;; *) diff --git a/scripts/publish-pkg.sh b/scripts/publish-pkg.sh index 90a8a5a..33e587f 100755 --- a/scripts/publish-pkg.sh +++ b/scripts/publish-pkg.sh @@ -9,7 +9,7 @@ # Usage: ./scripts/publish-pkg.sh [--dry-run] # # Exit codes: 0 pushed, 1 failed, 2 the AUR already has the version, -# 3 the AUR or upstream moved since the update was prepared +# 3 the AUR moved since the update was prepared set -euo pipefail # shellcheck source=lib.sh @@ -32,8 +32,8 @@ base=$(prepared_base "$PREPARED") || exit 1 prepared_ver=$(sed -n 's/^pkgver=//p' "${PREPARED}/PKGBUILD" | head -1) if [[ "$prepared_ver" != "$VERSION" ]]; then - log_warn "$PKG_NAME was prepared for $prepared_ver, upstream now has $VERSION. The next run prepares it again." - exit 3 + log_err "Prepared PKGBUILD has pkgver=${prepared_ver}, expected ${VERSION}" + exit 1 fi # ============================================================================= diff --git a/tests/publish.bats b/tests/publish.bats index 057939f..88c89d7 100644 --- a/tests/publish.bats +++ b/tests/publish.bats @@ -236,10 +236,11 @@ current: '1.0.0'" [ "$(git -C "$AUR_REMOTE" rev-parse HEAD)" = "$AUR_BASE" ] } -@test "publish-pkg: defers when upstream moved since the update was prepared" { +@test "publish-pkg: refuses an update prepared for another version" { setup_publish run "$TEST_TMPDIR/scripts/publish-pkg.sh" demo 1.2.0 "$PREPARED" - [ "$status" -eq 3 ] + [ "$status" -eq 1 ] + [[ "$output" == *"pkgver=1.1.0, expected 1.2.0"* ]] [ "$(git -C "$AUR_REMOTE" rev-parse HEAD)" = "$AUR_BASE" ] } @@ -683,7 +684,7 @@ checked_status() { printf '%s\n' "$2" >"$TEST_TMPDIR/.status/$1" } -@test "check-all --publish: publishes with the version upstream reports now and keeps the checked statuses" { +@test "check-all --publish: publishes the version the check run found, runs no strategy and keeps the checked statuses" { create_test_package "pkg" "github-release" "upstream: project: owner/repo current: '1.0.0'" @@ -693,9 +694,11 @@ current: '2.0.0'" checked_status pkg "new_version: 2.0.0" checked_status other "up_to_date" mkdir -p "$TEST_TMPDIR/out/prepared/pkg" + stub_script check-update.sh 1 "strategy ran" stub_script publish-pkg.sh 0 run "$TEST_TMPDIR/scripts/check-all.sh" --publish "$TEST_TMPDIR/out" [ "$status" -eq 0 ] + [ ! -e "$TEST_TMPDIR/check-update.sh.args" ] [ "$(cat "$TEST_TMPDIR/publish-pkg.sh.args")" = "pkg 2.0.0 $TEST_TMPDIR/out/prepared/pkg" ] [ "$(cat "$TEST_TMPDIR/.status/pkg")" = "updated: 2.0.0" ] [ "$(cat "$TEST_TMPDIR/.status/other")" = "up_to_date" ] diff --git a/tests/workflows.bats b/tests/workflows.bats index dd095db..e419b27 100644 --- a/tests/workflows.bats +++ b/tests/workflows.bats @@ -402,6 +402,14 @@ EOF [ "$output" = "0" ] } +@test "check-updates: the publish step gets no GitHub token" { + run step_run publish "Publish updates" + [ "$status" -eq 0 ] + [[ "$output" != *GITHUB_TOKEN* ]] + run yq '.jobs.publish.steps[] | select(.name == "Publish updates") | .env | has("GITHUB_TOKEN")' "$CHECK_UPDATES" + [ "$output" = "false" ] +} + @test "check-updates: publish takes the statuses from the check job" { run yq '.jobs.publish.steps[] | select(.with.name == "checked") | .with.path' "$CHECK_UPDATES" [ "$output" = ".status" ] From 80e5629fa7a98365133cebc77b7d978c05063222 Mon Sep 17 00:00:00 2001 From: noctuum <25441068+noctuum@users.noreply.github.com> Date: Fri, 16 Oct 2026 18:44:26 +0500 Subject: [PATCH 2/3] fix(scripts): read the AUR repo without a checkout --- docs/WORKFLOW.md | 8 ++--- scripts/lib.sh | 35 +++++++++++++++--- scripts/publish-pkg.sh | 17 ++++++--- scripts/verify-pkg.sh | 10 +++--- tests/publish.bats | 81 ++++++++++++++++++++++++++++++++++++++++-- 5 files changed, 132 insertions(+), 19 deletions(-) diff --git a/docs/WORKFLOW.md b/docs/WORKFLOW.md index 2874b38..6fab28b 100644 --- a/docs/WORKFLOW.md +++ b/docs/WORKFLOW.md @@ -623,8 +623,8 @@ A PKGBUILD is a bash script, and every step that reads it executes it: the audit **Verify**, `verify-pkg.sh /`, needs no secrets, never sources the PKGBUILD and never runs `makepkg`: 1. The prepared directory must pass step 1 of publish below. -2. Clone `https://aur.archlinux.org/.git` anonymously. When `HEAD` is not `base`, the AUR repo moved after the update was prepared: exit code 3. -3. `PKGBUILD` and `.SRCINFO` in the clone must be regular files. The prepared files must hold no NUL byte and pass steps 4 and 5 of publish below, with the prepared `pkgver` as ``. From here on, every source URL in the prepared `.SRCINFO` follows from the AUR copy. +2. Clone `https://aur.archlinux.org/.git` anonymously with `--no-checkout`. When `HEAD` is not `base`, the AUR repo moved after the update was prepared: exit code 3. The entries at the top of `HEAD` are written to a separate directory with `git ls-tree` and `git cat-file`, so git applies none of the links, attributes or filters of the repo: a regular file with its content, a symlink as a link to nowhere, anything else as an empty directory. The steps below read the AUR copy there. +3. `PKGBUILD` and `.SRCINFO` in the AUR copy must be regular files. The prepared files must hold no NUL byte and pass steps 4 and 5 of publish below, with the prepared `pkgver` as ``. From here on, every source URL in the prepared `.SRCINFO` follows from the AUR copy. 4. Each checksum array of the prepared `PKGBUILD` must hold the same values as the same key in the prepared `.SRCINFO`. 5. Every checksum other than `SKIP` must match its source, computed as makepkg does with `sum`. The file is the one makepkg uses. A file in the AUR repo with the name of the source comes first and must be a regular file. Otherwise a remote source is downloaded with curl, retried as makepkg does and limited to `http`, `https` and `ftp`, redirects included. A git source at `#tag=` or `#commit=` is cloned as a mirror over `https` or `http`, and its checksum is that of `git archive --format tar` of the tag or commit, with `export-subst` and `export-ignore` turned off, as makepkg computes it. A checksum for any other source, another VCS for example, cannot be checked and stops the verify. 6. Any mismatch stops the verify with exit code 1. Otherwise the exit code is 0. @@ -633,10 +633,10 @@ A PKGBUILD is a bash script, and every step that reads it executes it: the audit 1. The prepared directory must not be a symlink, `PKGBUILD`, `.SRCINFO` and `base` must be regular files, and `base` a commit id. 2. The prepared `pkgver` must be ``. `check-all.sh --publish` passes the version the check run found, and the prepare run was given the same one, so another `pkgver` stops the publish with exit code 1. -3. Clone over SSH. When `HEAD` is not `base`, the AUR repo moved after the update was prepared: exit code 3. `PKGBUILD` and `.SRCINFO` in the clone must be regular files, not symlinks. +3. Clone over SSH with `--no-checkout`, and write the AUR copy as step 2 of verify does. When `HEAD` is not `base`, the AUR repo moved after the update was prepared: exit code 3. `PKGBUILD` and `.SRCINFO` in the AUR copy must be regular files, not symlinks. 4. The prepared `PKGBUILD` may differ from the AUR copy only in `pkgver=`, which must be ``, in `pkgrel=`, which must be `1`, and in the values of the checksum arrays. The arrays keep their names and lengths, hold only quoted hex checksums or `SKIP`, and have a `SKIP` exactly where the AUR copy has one. Comments and line continuations may stand between the checksums, read as bash reads them: a `#` at the start of a line, right after the opening parenthesis, or after a space or a tab comments out the rest of the line, and a backslash that ends a line without a comment, as a word of its own, continues the array on the next line. A backslash on the line that closes the array stops the publish, because bash joins the next line to the assignment. 5. Each `source` line of the prepared `.SRCINFO`, with or without an arch suffix, must be the line of the AUR copy with the old version replaced by ``. A source that is `SKIP` in every checksum array of its arch may also change the digits in the name it is saved under, the part before `::`, as `neovim-20261014.tar.gz` becomes `neovim-20261016.tar.gz` in a PKGBUILD that names a nightly download by `$(date -u +%Y%m%d)`. Its URL after `::` must still be the AUR URL with the version replaced. Verify never downloads a `SKIP` source, and makepkg takes the sources from the `PKGBUILD`. Any other line must either equal the line of the AUR copy or be that line with the version replaced in the same way, because a license, a dependency or a description can hold the old version without taking it from `pkgver`, as `GPL-2.0` does in a package at version `2.0`. The replacement also covers three forms a PKGBUILD often derives from `pkgver`: dots turned into underscores (`1_86_0`), the part before the first `+` (`0.13.0` of `0.13.0+dev+1853+g6edb38ca98`), and the version without its last component (`6.19` of `6.19.0`, as in `frameworks/${pkgver%.*}/` of KDE Frameworks). The last form counts only while it keeps a dot. Each part of a line is replaced at most once, so `5.1` inside an already replaced `5.10.0` stays. A line built from any other form of the version, a lone major version for example, stops the publish, and so does a line that has the version replaced in one place and kept in another. The package then needs a manual update. `pkgrel` and the checksums follow the rules of step 4. -6. Any other difference, and a NUL byte in either file, stops the publish with exit code 1. Otherwise steps 10 to 13 of [4.2](#42-step-by-step) follow. When the prepared files equal the AUR copy, only `current` is written to the YAML and the exit code is 2. +6. Any other difference, and a NUL byte in either file, stops the publish with exit code 1. Otherwise the index of the clone takes the tree of `HEAD` and only the two prepared files enter its work tree, so the commit keeps every other file of the AUR repo, and steps 10 to 13 of [4.2](#42-step-by-step) follow. When the prepared files equal the AUR copy, only `current` is written to the YAML and the exit code is 2. `check-all.sh --prepare ` runs no strategy and needs no `GITHUB_TOKEN`. It takes the versions from the statuses that a check run, `check-all.sh` without a mode, wrote to `.status/` before it, and reads them before the first PKGBUILD runs. It prepares every package with `new_version` or `available_downgrade` into `/prepared/` and copies the statuses to `/status/`. A failed preparation becomes `update_failed`, and a package without a status from the check run becomes `check_failed`. The other statuses stay as the check run wrote them. diff --git a/scripts/lib.sh b/scripts/lib.sh index 739df93..db04d7f 100644 --- a/scripts/lib.sh +++ b/scripts/lib.sh @@ -265,13 +265,13 @@ validate_source_urls() { # ============================================================================= # Clone an AUR repo, with 3 attempts -# Args: $1=url, $2=destination +# Args: $1=url, $2=destination, $3...=options for git clone aur_clone() { local attempt # Brief delay to reduce AUR SSH rate-limit pressure from GitHub Actions IP pool sleep 2 for attempt in 1 2 3; do - if git clone "$1" "$2" 2>&1; then + if git clone "${@:3}" "$1" "$2" 2>&1; then return 0 fi if [[ "$attempt" -eq 3 ]]; then @@ -292,15 +292,42 @@ aur_clone() { # origin/master and not on the detached HEAD. Anything unique to the # detached commit is dropped. The workflow's concurrency group prevents # the race; this stays as a safety net. -# Args: $1=clone directory +# Args: $1=clone directory, $2=--no-checkout for a clone made without one aur_reanchor() { if ! git -C "$1" symbolic-ref -q HEAD >/dev/null; then log_warn "Detached HEAD after clone (remote HEAD != refs/heads/master)" log_info "Re-basing on origin/master to allow fast-forward push" - git -C "$1" checkout -B master origin/master + if [[ "${2:-}" == "--no-checkout" ]]; then + git -C "$1" branch -f master origin/master && + git -C "$1" symbolic-ref HEAD refs/heads/master + else + git -C "$1" checkout -B master origin/master + fi fi } +# Write the top-level entries of HEAD of a clone made with --no-checkout +# into a directory, without a checkout, so git applies none of the links, +# attributes or filters of the repo: a regular file with its content, a +# symlink as a link to nowhere and anything else as an empty directory. The +# checks of the prepared update refuse those two where a PKGBUILD, a +# .SRCINFO or a source is expected, as they do in a checkout. +# Args: $1=clone directory, $2=destination directory, created here +aur_export() { + local entry name mode sha + mkdir "$2" || return 1 + while IFS= read -r -d '' entry; do + name="${entry#*$'\t'}" + [[ -n "$name" && "$name" != */* && "$name" != . && "$name" != .. ]] || continue + read -r mode _ sha <<<"${entry%%$'\t'*}" + case "$mode" in + 100*) git -C "$1" cat-file blob "$sha" >"$2/$name" || return 1 ;; + 120000) ln -s /nonexistent "$2/$name" || return 1 ;; + *) mkdir "$2/$name" || return 1 ;; + esac + done < <(git -C "$1" ls-tree -z HEAD) +} + # Commit PKGBUILD and .SRCINFO, sign when GPG_KEY_ID is set, push with 3 # attempts. # Args: $1=clone directory, $2=version, $3=strategy (for the commit message) diff --git a/scripts/publish-pkg.sh b/scripts/publish-pkg.sh index 33e587f..e93e0fd 100755 --- a/scripts/publish-pkg.sh +++ b/scripts/publish-pkg.sh @@ -45,8 +45,8 @@ trap 'rm -rf "$tmp_dir"' EXIT aur_dir="${tmp_dir}/aur-${PKG_NAME}" log_info "Cloning AUR repo: $PKG_NAME" -aur_clone "ssh://aur@aur.archlinux.org/${PKG_NAME}.git" "$aur_dir" || exit 1 -aur_reanchor "$aur_dir" +aur_clone "ssh://aur@aur.archlinux.org/${PKG_NAME}.git" "$aur_dir" --no-checkout || exit 1 +aur_reanchor "$aur_dir" --no-checkout head=$(git -C "$aur_dir" rev-parse HEAD) if [[ "$head" != "$base" ]]; then @@ -58,10 +58,17 @@ fi # CHECK AND PUSH # ============================================================================= -check_prepared_files "$aur_dir" "$PREPARED" "$VERSION" || exit 1 +aur_files="${tmp_dir}/aur-files" +aur_export "$aur_dir" "$aur_files" || exit 1 +check_prepared_files "$aur_files" "$PREPARED" "$VERSION" || exit 1 + +# The clone has no checkout: the index takes the tree of HEAD, and only the +# two prepared files enter the work tree +git -C "$aur_dir" read-tree HEAD || exit 1 cp "${PREPARED}/PKGBUILD" "${PREPARED}/.SRCINFO" "$aur_dir/" +git -C "$aur_dir" add PKGBUILD .SRCINFO || exit 1 -if git -C "$aur_dir" diff --quiet; then +if git -C "$aur_dir" diff --cached --quiet; then log_ok "$PKG_NAME already at $VERSION on AUR, nothing to do" pkg_set "$PKG_FILE" .current "\"${VERSION}\"" exit 2 @@ -70,7 +77,7 @@ fi if [[ "$DRY_RUN" == "--dry-run" ]]; then log_warn "[DRY RUN] Would push $PKG_NAME $VERSION to AUR" log_info "PKGBUILD diff:" - git -C "$aur_dir" diff PKGBUILD || true + git -C "$aur_dir" diff --cached PKGBUILD || true exit 0 fi diff --git a/scripts/verify-pkg.sh b/scripts/verify-pkg.sh index e1afbae..a7c8971 100755 --- a/scripts/verify-pkg.sh +++ b/scripts/verify-pkg.sh @@ -31,8 +31,8 @@ trap 'rm -rf "$tmp_dir"' EXIT aur_dir="${tmp_dir}/aur-${PKG_NAME}" log_info "Cloning AUR repo: $PKG_NAME" -aur_clone "https://aur.archlinux.org/${PKG_NAME}.git" "$aur_dir" || exit 1 -aur_reanchor "$aur_dir" +aur_clone "https://aur.archlinux.org/${PKG_NAME}.git" "$aur_dir" --no-checkout || exit 1 +aur_reanchor "$aur_dir" --no-checkout head=$(git -C "$aur_dir" rev-parse HEAD) if [[ "$head" != "$base" ]]; then @@ -40,7 +40,9 @@ if [[ "$head" != "$base" ]]; then exit 3 fi -check_prepared_files "$aur_dir" "$PREPARED" "$version" || exit 1 +aur_files="${tmp_dir}/aur-files" +aur_export "$aur_dir" "$aur_files" || exit 1 +check_prepared_files "$aur_files" "$PREPARED" "$version" || exit 1 mkdir "${tmp_dir}/downloads" -verify_prepared_sums "$aur_dir" "$PREPARED" "${tmp_dir}/downloads" || exit 1 +verify_prepared_sums "$aur_files" "$PREPARED" "${tmp_dir}/downloads" || exit 1 log_ok "Checksums of $PKG_NAME $version match the sources" diff --git a/tests/publish.bats b/tests/publish.bats index 88c89d7..02114fe 100644 --- a/tests/publish.bats +++ b/tests/publish.bats @@ -86,14 +86,28 @@ if [[ "\$1" == "clone" && " \$* " == *" --mirror "* ]]; then exec env -u GIT_ALLOW_PROTOCOL /usr/bin/git clone -q --mirror "$TEST_TMPDIR/upstream.git" "\${@: -1}" fi if [[ "\$1" == "clone" ]]; then - echo "\$2" >>"$TEST_TMPDIR/clone-urls" - exec /usr/bin/git clone -q "$AUR_REMOTE" "\$3" + echo "\${@: -2:1}" >>"$TEST_TMPDIR/clone-urls" + echo "\${*:2:\$#-3}" >>"$TEST_TMPDIR/clone-opts" + exec /usr/bin/git clone -q "\${@:2:\$#-3}" "$AUR_REMOTE" "\${@: -1}" fi exec /usr/bin/git "\$@" EOF chmod +x "$MOCK_DIR/git" } +# Helper: add a file to the AUR repo of demo and hand the new commit to the +# prepared update as its base +# Usage: aur_add_file +aur_add_file() { + local work="$TEST_TMPDIR/aur-work-demo" + printf '%s\n' "$2" >"$work/$1" + git -C "$work" add "$1" + git -C "$work" -c user.name=aur -c user.email=aur@example.invalid \ + -c commit.gpgsign=false commit -qm "add $1" + git -C "$work" push -q "$AUR_REMOTE" master + git -C "$AUR_REMOTE" rev-parse HEAD >"$PREPARED/base" +} + # Helper: makepkg that prints a fixed .SRCINFO, namcap that finds nothing mock_pkg_tools() { printf '#!/usr/bin/env bash\necho "pkgbase = demo"\n' >"$MOCK_DIR/makepkg" @@ -175,6 +189,17 @@ current: '1.0.0'" [ "$(pkg_get "$TEST_TMPDIR/packages/demo.yml" .current)" = "1.1.0" ] } +@test "publish-pkg: clones the AUR repo without a checkout and keeps its other files" { + setup_publish + aur_add_file demo.install "post_install() { :; }" + run "$TEST_TMPDIR/scripts/publish-pkg.sh" demo 1.1.0 "$PREPARED" + [ "$status" -eq 0 ] + grep -qx -- '--no-checkout' "$TEST_TMPDIR/clone-opts" + git -C "$AUR_REMOTE" show HEAD:PKGBUILD | grep -qx 'pkgver=1.1.0' + [ "$(git -C "$AUR_REMOTE" show HEAD:demo.install)" = "post_install() { :; }" ] + [ "$(git -C "$AUR_REMOTE" ls-tree --name-only HEAD | LC_ALL=C sort | tr '\n' ' ')" = ".SRCINFO PKGBUILD demo.install " ] +} + @test "publish-pkg: refuses a prepared PKGBUILD that changes other lines" { setup_publish sed -i 's|https://example.com/demo|https://evil.example/demo|' "$PREPARED/PKGBUILD" @@ -327,6 +352,13 @@ commit_aur_state() { git -C "$AUR_REMOTE" rev-parse HEAD >"$PREPARED/base" } +@test "verify-pkg: clones the AUR repo without a checkout" { + setup_verify + run "$TEST_TMPDIR/scripts/verify-pkg.sh" demo "$PREPARED" + [ "$status" -eq 0 ] + grep -qx -- '--no-checkout' "$TEST_TMPDIR/clone-opts" +} + @test "verify-pkg: accepts checksums that match the sources" { setup_verify run "$TEST_TMPDIR/scripts/verify-pkg.sh" demo "$PREPARED" @@ -1129,6 +1161,51 @@ EOF [ "${lines[6]}" = "$tail" ] } +# === aur_export and aur_reanchor === + +# Helper: a repo with a file, an executable, a symlink out of the repo and a +# directory, cloned to $TEST_TMPDIR/clone without a checkout +make_export_clone() { + local repo="$TEST_TMPDIR/tree" + git init -q -b master "$repo" + echo "pkgver=1.0.0" >"$repo/PKGBUILD" + printf '#!/bin/sh\n' >"$repo/run.sh" + chmod +x "$repo/run.sh" + ln -s /etc/hostname "$repo/link" + mkdir "$repo/dir" + echo inside >"$repo/dir/file" + git -C "$repo" add . + git -C "$repo" -c user.name=aur -c user.email=aur@example.invalid \ + -c commit.gpgsign=false commit -qm tree + git clone -q --no-checkout "$repo" "$TEST_TMPDIR/clone" +} + +@test "aur_export: writes the files of HEAD and stands in for links and directories" { + make_export_clone + run aur_export "$TEST_TMPDIR/clone" "$TEST_TMPDIR/files" + [ "$status" -eq 0 ] + [ "$(cat "$TEST_TMPDIR/files/PKGBUILD")" = "pkgver=1.0.0" ] + [ -f "$TEST_TMPDIR/files/run.sh" ] + [ ! -L "$TEST_TMPDIR/files/run.sh" ] + [ -L "$TEST_TMPDIR/files/link" ] + [ "$(readlink "$TEST_TMPDIR/files/link")" != /etc/hostname ] + [ -d "$TEST_TMPDIR/files/dir" ] + [ -z "$(ls -A "$TEST_TMPDIR/files/dir")" ] + [ ! -e "$TEST_TMPDIR/clone/PKGBUILD" ] +} + +@test "aur_reanchor: puts a clone without a checkout on master and checks nothing out" { + make_export_clone + local clone="$TEST_TMPDIR/clone" + git -C "$clone" update-ref --no-deref HEAD "$(git -C "$clone" rev-parse HEAD)" + git -C "$clone" branch -q -D master + run aur_reanchor "$clone" --no-checkout + [ "$status" -eq 0 ] + [ "$(git -C "$clone" symbolic-ref HEAD)" = "refs/heads/master" ] + [ "$(git -C "$clone" rev-parse master)" = "$(git -C "$clone" rev-parse origin/master)" ] + [ ! -e "$clone/PKGBUILD" ] +} + # === check_prepared_pkgbuild === @test "check_prepared_pkgbuild: accepts new pkgver, pkgrel and checksums" { From 838a6e85fb87a936a068267bdb44db12b2aa7f25 Mon Sep 17 00:00:00 2001 From: noctuum <25441068+noctuum@users.noreply.github.com> Date: Fri, 16 Oct 2026 18:56:53 +0500 Subject: [PATCH 3/3] fix(workflow): run verify as builder --- .github/workflows/check-updates.yml | 62 ++++++++++++++++++++++++++--- docs/WORKFLOW.md | 11 ++--- tests/workflows.bats | 10 +++++ 3 files changed, 73 insertions(+), 10 deletions(-) diff --git a/.github/workflows/check-updates.yml b/.github/workflows/check-updates.yml index 35daf61..2d8e933 100644 --- a/.github/workflows/check-updates.yml +++ b/.github/workflows/check-updates.yml @@ -138,8 +138,8 @@ jobs: - name: Hand over run: | - # A process started by a PKGBUILD can outlive its step. Stop all - # of them before root reads anything builder wrote. A zombie runs + # A process builder started can outlive its step. Stop all of + # them before root reads anything builder wrote. A zombie runs # nothing, and pid 1 of a job container never reaps one, so only # the other processes count, by effective or real user. live exits # with 0 for a running process, 1 for none and 2 when the list @@ -207,7 +207,8 @@ jobs: run: | pacman-key --init pacman -Sy --noconfirm --needed archlinux-keyring - pacman -Su --noconfirm --needed git jq go-yq + pacman -Su --noconfirm --needed sudo git jq go-yq + useradd -m builder - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -220,14 +221,65 @@ jobs: name: handoff path: handoff + # git and curl read the AUR repo and the sources here, as builder, + # who cannot reach the Docker socket that root has in a job container - name: Verify checksums - run: ./scripts/check-all.sh --verify handoff + run: | + chown -R builder:builder . + sudo -u builder ./scripts/check-all.sh --verify handoff + + - name: Hand over + run: | + # A process builder started can outlive its step. Stop all of + # them before root reads anything builder wrote. A zombie runs + # nothing, and pid 1 of a job container never reaps one, so only + # the other processes count, by effective or real user. live exits + # with 0 for a running process, 1 for none and 2 when the list + # fails, and only 1 lets the step go on. + uid=$(id -u builder) + live() { + procs=$(ps -e -o uid=,ruid=,stat=) || return 2 + printf '%s\n' "$procs" | + awk -v u="$uid" '($1 == u || $2 == u) && $3 !~ /^Z/ { n++ } END { exit !n }' + } + for _ in $(seq 30); do + rc=0 + live || rc=$? + [ "$rc" -eq 1 ] && break + pkill -KILL -u builder || true + pkill -KILL -U builder || true + sleep 1 + done + rc=0 + live || rc=$? + if [ "$rc" -ne 1 ]; then + echo "::error::A process of builder is still running, or the process list failed" + exit 1 + fi + + # Copy regular files from the expected paths only, and nothing + # else gets uploaded. find does not follow symlinks; -print0, + # because a name may hold a newline. A package name never starts + # with a dot, so . and .. do not match. + mkdir upload + if [[ -d handoff && ! -L handoff ]]; then + pkg_re='[A-Za-z0-9@_+][A-Za-z0-9@._+-]*' + (cd handoff && find prepared status -type f -print0 2>/dev/null) | + while IFS= read -r -d '' f; do + if [[ "$f" =~ ^prepared/${pkg_re}/(PKGBUILD|\.SRCINFO|base)$ ]] || + [[ "$f" =~ ^status/${pkg_re}$ ]]; then + [[ -f "handoff/$f" && ! -L "handoff/$f" ]] || continue + install -D -m 0644 "handoff/$f" "upload/$f" + fi + done + fi - name: Upload verified handoff uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: verified - path: handoff/ + path: upload/ + # .SRCINFO starts with a dot include-hidden-files: true if-no-files-found: error retention-days: 1 diff --git a/docs/WORKFLOW.md b/docs/WORKFLOW.md index 6fab28b..1eb9ebc 100644 --- a/docs/WORKFLOW.md +++ b/docs/WORKFLOW.md @@ -41,7 +41,7 @@ flowchart TD H5 --> H6[hand over PKGBUILD, .SRCINFO, AUR commit] H -->|failure| J[write error to .status/] D -->|failure| J - B2 -->|all done| V[verify job: no secrets] + B2 -->|all done| V[verify job: no secrets, as builder] V --> V1[check-all.sh --verify] V1 --> V2[verify-pkg.sh: download the sources, check the checksums] V2 -->|failure| V3[take the update out, write update_failed] @@ -62,7 +62,7 @@ flowchart TD - **One YAML per package.** Adding or removing a package is one file. - **Strategy-based version detection.** Each type of upstream source has its own script. A new source is a new script plus its entry in `scripts/validate-pkg.sh`. - **Patch, do not generate.** aurtomator trusts your PKGBUILD and changes only `pkgver`, `pkgrel`, checksums and `.SRCINFO`. Build steps, dependencies and install files stay untouched. -- **Four jobs, one with secrets.** A PKGBUILD is code that every step reading it executes. `check` runs the strategies with the read-only workflow token, executes no PKGBUILD and decides what the run does. The jobs after it take the status of every package from `check`. `prepare` runs the PKGBUILD steps without secrets and without a token, and hands over only prepared files and update failures, because code from one PKGBUILD can write the files of every package there. `verify` downloads the sources of each prepared update and checks its checksums, without secrets, so code from one PKGBUILD cannot forge the checksums of another package. `publish` holds the SSH key, the GPG key and the write token, never executes a PKGBUILD, runs no strategy and never downloads a source, and checks what `verify` passes on before it pushes. See [4.4](#44-prepare-and-publish). +- **Four jobs, one with secrets.** A PKGBUILD is code that every step reading it executes. `check` runs the strategies with the read-only workflow token, executes no PKGBUILD and decides what the run does. The jobs after it take the status of every package from `check`. `prepare` runs the PKGBUILD steps without secrets and without a token, and hands over only prepared files and update failures, because code from one PKGBUILD can write the files of every package there. `verify` downloads the sources of each prepared update and checks its checksums, without secrets and as an unprivileged user, so code from one PKGBUILD cannot forge the checksums of another package. `publish` holds the SSH key, the GPG key and the write token, never executes a PKGBUILD, runs no strategy and never downloads a source, and checks what `verify` passes on before it pushes. See [4.4](#44-prepare-and-publish). - **Failure isolation.** A broken package does not block the others. Each failure is logged for its package and opens a GitHub Issue. Each package has a time budget, so one that hangs does not hold up the run either. ## 2. Package YAML Format @@ -690,10 +690,11 @@ Every job starts with **Checkout:** `actions/checkout`, pinned by commit SHA lik `verify`: -1. **Install dependencies:** `git jq go-yq`. +1. **Install dependencies:** `sudo git jq go-yq`, and the `builder` user. 2. **Download handoff:** the `handoff` artifact to `handoff/`. -3. **Verify checksums:** runs `check-all.sh --verify handoff`, see [4.4](#44-prepare-and-publish). -4. **Upload verified handoff:** uploads `handoff/` as the `verified` artifact with `include-hidden-files: true` and a retention of 1 day. +3. **Verify checksums:** gives the workspace to `builder` and runs `check-all.sh --verify handoff` as `builder`, see [4.4](#44-prepare-and-publish). git and curl read the AUR repo and the sources here. A job container has the Docker socket of the runner, which root can use and `builder` cannot. +4. **Hand over:** the same step as in `prepare`. +5. **Upload verified handoff:** uploads `upload/` as the `verified` artifact with `include-hidden-files: true` and a retention of 1 day. `publish`: diff --git a/tests/workflows.bats b/tests/workflows.bats index e419b27..6ede1f2 100644 --- a/tests/workflows.bats +++ b/tests/workflows.bats @@ -402,6 +402,16 @@ EOF [ "$output" = "0" ] } +@test "check-updates: verify runs as builder and hands over as prepare does" { + run step_run verify "Verify checksums" + [[ "$output" == *"sudo -u builder ./scripts/check-all.sh --verify handoff"* ]] + [ "$(step_run verify "Hand over")" = "$(step_run prepare "Hand over")" ] + run yq '[.jobs.verify.steps[].name] | join(",")' "$CHECK_UPDATES" + [[ "$output" == *"Verify checksums,Hand over,Upload verified handoff"* ]] + run yq '.jobs.verify.steps[] | select(.name == "Upload verified handoff") | .with.path' "$CHECK_UPDATES" + [ "$output" = "upload/" ] +} + @test "check-updates: the publish step gets no GitHub token" { run step_run publish "Publish updates" [ "$status" -eq 0 ]