diff --git a/docs/SUPPORTED-PACKAGES.md b/docs/SUPPORTED-PACKAGES.md index 16cd10f..6f381e5 100644 --- a/docs/SUPPORTED-PACKAGES.md +++ b/docs/SUPPORTED-PACKAGES.md @@ -100,6 +100,8 @@ upstream: The version comes from the tag name, the release body, the tag date or the publish date. `strategies/github-nightly.sh` lists the fields. +A fixed tag keeps its download URL, so a PKGBUILD can save the download under a name with the date, as `neovim-nightly-bin` does with `_date="$(date -u +%Y%m%d)"` and `"$_pkgname-$_date.tar.gz::"`. This works when the checksum of that source is `SKIP`: only the digits of the name may change from one update to the next. See step 5 of publish in [WORKFLOW.md section 4.4](WORKFLOW.md#44-prepare-and-publish). + ## Not supported (blocked or detected) aurtomator detects these cases and blocks the update or warns. diff --git a/docs/WORKFLOW.md b/docs/WORKFLOW.md index 26437cc..102c8e4 100644 --- a/docs/WORKFLOW.md +++ b/docs/WORKFLOW.md @@ -636,7 +636,7 @@ A PKGBUILD is a bash script, and every step that reads it executes it: the audit 2. When the prepared `pkgver` is not ``, upstream moved after the update was prepared: exit code 3. 3. Clone over SSH. When `HEAD` is not `base`, the AUR repo moved after the update was prepared: exit code 3. `PKGBUILD` and `.SRCINFO` in the clone must be regular files, not symlinks. 4. The prepared `PKGBUILD` may differ from the AUR copy only in `pkgver=`, which must be ``, in `pkgrel=`, which must be `1`, and in the values of the checksum arrays. The arrays keep their names and lengths, hold only quoted hex checksums or `SKIP`, and have a `SKIP` exactly where the AUR copy has one. Comments and line continuations may stand between the checksums, read as bash reads them: a `#` at the start of a line, right after the opening parenthesis, or after a space or a tab comments out the rest of the line, and a backslash that ends a line without a comment, as a word of its own, continues the array on the next line. A backslash on the line that closes the array stops the publish, because bash joins the next line to the assignment. -5. Each `source` line of the prepared `.SRCINFO`, with or without an arch suffix, must be the line of the AUR copy with the old version replaced by ``. Any other line must either equal the line of the AUR copy or be that line with the version replaced in the same way, because a license, a dependency or a description can hold the old version without taking it from `pkgver`, as `GPL-2.0` does in a package at version `2.0`. The replacement also covers three forms a PKGBUILD often derives from `pkgver`: dots turned into underscores (`1_86_0`), the part before the first `+` (`0.13.0` of `0.13.0+dev+1853+g6edb38ca98`), and the version without its last component (`6.19` of `6.19.0`, as in `frameworks/${pkgver%.*}/` of KDE Frameworks). The last form counts only while it keeps a dot. Each part of a line is replaced at most once, so `5.1` inside an already replaced `5.10.0` stays. A line built from any other form of the version, a lone major version for example, stops the publish, and so does a line that has the version replaced in one place and kept in another. The package then needs a manual update. `pkgrel` and the checksums follow the rules of step 4. +5. Each `source` line of the prepared `.SRCINFO`, with or without an arch suffix, must be the line of the AUR copy with the old version replaced by ``. A source that is `SKIP` in every checksum array of its arch may also change the digits in the name it is saved under, the part before `::`, as `neovim-20261014.tar.gz` becomes `neovim-20261016.tar.gz` in a PKGBUILD that names a nightly download by `$(date -u +%Y%m%d)`. Its URL after `::` must still be the AUR URL with the version replaced. Verify never downloads a `SKIP` source, and makepkg takes the sources from the `PKGBUILD`. Any other line must either equal the line of the AUR copy or be that line with the version replaced in the same way, because a license, a dependency or a description can hold the old version without taking it from `pkgver`, as `GPL-2.0` does in a package at version `2.0`. The replacement also covers three forms a PKGBUILD often derives from `pkgver`: dots turned into underscores (`1_86_0`), the part before the first `+` (`0.13.0` of `0.13.0+dev+1853+g6edb38ca98`), and the version without its last component (`6.19` of `6.19.0`, as in `frameworks/${pkgver%.*}/` of KDE Frameworks). The last form counts only while it keeps a dot. Each part of a line is replaced at most once, so `5.1` inside an already replaced `5.10.0` stays. A line built from any other form of the version, a lone major version for example, stops the publish, and so does a line that has the version replaced in one place and kept in another. The package then needs a manual update. `pkgrel` and the checksums follow the rules of step 4. 6. Any other difference, and a NUL byte in either file, stops the publish with exit code 1. Otherwise steps 10 to 13 of [4.2](#42-step-by-step) follow. When the prepared files equal the AUR copy, only `current` is written to the YAML and the exit code is 2. `check-all.sh --prepare ` runs no strategy and needs no `GITHUB_TOKEN`. It takes the versions from the statuses that a check run, `check-all.sh` without a mode, wrote to `.status/` before it, and reads them before the first PKGBUILD runs. It prepares every package with `new_version` or `available_downgrade` into `/prepared/` and copies the statuses to `/status/`. A failed preparation becomes `update_failed`, and a package without a status from the check run becomes `check_failed`. The other statuses stay as the check run wrote them. diff --git a/scripts/lib.sh b/scripts/lib.sh index cde8b3a..739df93 100644 --- a/scripts/lib.sh +++ b/scripts/lib.sh @@ -503,13 +503,39 @@ _bump_version() { printf '%s\n' "$out" } +# Print ":" for every source entry of a .SRCINFO that is +# SKIP in each checksum array with the same arch suffix +# Args: $1=.SRCINFO +_skip_sources() { + awk -F' = ' ' + { key = $1; sub(/^[ \t]+/, "", key) } + key ~ /^(ck|md5|sha1|sha224|sha256|sha384|sha512|b2)sums(_[A-Za-z0-9_]+)?$/ { + entry = substr(key, index(key, "sums") + 4) ":" count[key]++ + listed[entry] = 1 + if ($2 != "SKIP") hashed[entry] = 1 + } + END { for (entry in listed) if (!(entry in hashed)) print entry } + ' "$1" +} + +# Args: $1=expected source line, $2=prepared source line +# Returns: 0 when both save their file under a name before ::, have the same +# URL after it, and their names differ in nothing but digits +_name_differs_in_digits() { + [[ "$1" == *::* && "$2" == *::* && "${1#*::}" == "${2#*::}" ]] || return 1 + [[ "$(sed -E 's/[0-9]+/0/g' <<<"${1%%::*}")" == "$(sed -E 's/[0-9]+/0/g' <<<"${2%%::*}")" ]] +} + # A .SRCINFO is generated, so line N of the prepared file must be line N of # the AUR file, with these changes: pkgver is the new version, pkgrel is 1, # a checksum keeps its key and stays a SKIP or a hash, a source has the old # version replaced by the new one (see _bump_version), and any other line # either stays or has the version replaced in the same way. A license, a # dependency or a pkgdesc can hold the old version without deriving it -# from pkgver. +# from pkgver. A source that is SKIP in every checksum array may also change +# the digits in the name of its file, as a nightly package names its +# download by the date: verify_prepared_sums never fetches it, and makepkg +# takes the sources from the PKGBUILD. # Args: $1=AUR .SRCINFO, $2=prepared .SRCINFO, $3=expected version # Returns: 0 when that holds, 1 otherwise (reason on stderr) check_prepared_srcinfo() { @@ -533,7 +559,13 @@ check_prepared_srcinfo() { local old_ver old_ver=$(awk -F' = ' '$1 ~ /^[ \t]*pkgver$/ { print $2; exit }' "$1") - local i old new key kind old_val new_val + local -A skip_sources=() source_counts=() + local entry + while read -r entry; do + skip_sources["$entry"]=1 + done < <(_skip_sources "$1") + + local i old new key kind old_val new_val n expected for i in "${!new_lines[@]}"; do old="${old_lines[$i]}" new="${new_lines[$i]}" @@ -576,7 +608,12 @@ check_prepared_srcinfo() { fi ;; source) - if [[ "$new" != "$(_bump_version "$old" "$old_ver" "$version")" ]]; then + n="${source_counts[$key]:-0}" + source_counts[$key]=$((n + 1)) + expected=$(_bump_version "$old" "$old_ver" "$version") + if [[ "$new" != "$expected" ]] && + ! { [[ -n "${skip_sources["${key#source}:$n"]:-}" ]] && + _name_differs_in_digits "$expected" "$new"; }; then log_err "Prepared .SRCINFO line $((i + 1)) is not the AUR source with the new version: $new" return 1 fi diff --git a/tests/publish.bats b/tests/publish.bats index 8aeffb9..057939f 100644 --- a/tests/publish.bats +++ b/tests/publish.bats @@ -1379,6 +1379,83 @@ write_srcinfo_lookalike() { [[ "$output" == *"changes more than the version"* ]] } +# Helper: .SRCINFO of a nightly package with two sources, the first saved +# under . gives the checksum arrays as array=checksum pairs, +# with the checksum of the first source; the second source has $H2. +# Usage: write_srcinfo_nightly [url] +write_srcinfo_nightly() { + local url="${5:-https://example.com/releases/download/nightly/demo-linux.tar.gz}" + local pair + { + printf '%s\n' "pkgbase = demo-nightly-bin" $'\tpkgver = '"$2" $'\tpkgrel = 1' \ + $'\tsource = '"$3::$url" $'\tsource = demo.vim' + for pair in $4; do + printf '\t%s = %s\n' "${pair%%=*}" "${pair#*=}" "${pair%%=*}" "$H2" + done + printf '%s\n' "" "pkgname = demo-nightly-bin" + } >"$1" +} + +@test "check_prepared_srcinfo: accepts new digits in the name of a SKIP source" { + write_srcinfo_nightly "$TEST_TMPDIR/old.SRCINFO" 1.0.0 demo-20261014.tar.gz b2sums=SKIP + write_srcinfo_nightly "$TEST_TMPDIR/new.SRCINFO" 1.1.0 demo-20261016.tar.gz b2sums=SKIP + run check_prepared_srcinfo "$TEST_TMPDIR/old.SRCINFO" "$TEST_TMPDIR/new.SRCINFO" 1.1.0 + [ "$status" -eq 0 ] +} + +@test "check_prepared_srcinfo: rejects new letters in the name of a SKIP source" { + write_srcinfo_nightly "$TEST_TMPDIR/old.SRCINFO" 1.0.0 demo-20261014.tar.gz b2sums=SKIP + write_srcinfo_nightly "$TEST_TMPDIR/new.SRCINFO" 1.1.0 demo-oct16.tar.gz b2sums=SKIP + run check_prepared_srcinfo "$TEST_TMPDIR/old.SRCINFO" "$TEST_TMPDIR/new.SRCINFO" 1.1.0 + [ "$status" -eq 1 ] +} + +@test "check_prepared_srcinfo: rejects a new URL for a SKIP source" { + write_srcinfo_nightly "$TEST_TMPDIR/old.SRCINFO" 1.0.0 demo-20261014.tar.gz b2sums=SKIP + write_srcinfo_nightly "$TEST_TMPDIR/new.SRCINFO" 1.1.0 demo-20261016.tar.gz b2sums=SKIP \ + https://evil.example/demo-linux.tar.gz + run check_prepared_srcinfo "$TEST_TMPDIR/old.SRCINFO" "$TEST_TMPDIR/new.SRCINFO" 1.1.0 + [ "$status" -eq 1 ] +} + +@test "check_prepared_srcinfo: rejects new digits in the name of a hashed source" { + write_srcinfo_nightly "$TEST_TMPDIR/old.SRCINFO" 1.0.0 demo-20261014.tar.gz "b2sums=$H1" + write_srcinfo_nightly "$TEST_TMPDIR/new.SRCINFO" 1.1.0 demo-20261016.tar.gz "b2sums=$H3" + run check_prepared_srcinfo "$TEST_TMPDIR/old.SRCINFO" "$TEST_TMPDIR/new.SRCINFO" 1.1.0 + [ "$status" -eq 1 ] +} + +@test "check_prepared_srcinfo: rejects new digits when another checksum array has a hash" { + write_srcinfo_nightly "$TEST_TMPDIR/old.SRCINFO" 1.0.0 demo-20261014.tar.gz "b2sums=SKIP sha256sums=$H1" + write_srcinfo_nightly "$TEST_TMPDIR/new.SRCINFO" 1.1.0 demo-20261016.tar.gz "b2sums=SKIP sha256sums=$H3" + run check_prepared_srcinfo "$TEST_TMPDIR/old.SRCINFO" "$TEST_TMPDIR/new.SRCINFO" 1.1.0 + [ "$status" -eq 1 ] +} + +# Helper: .SRCINFO of a nightly package with a generic source and an +# x86_64 source saved under , and the checksums of each +# Usage: write_srcinfo_arch +write_srcinfo_arch() { + printf '%s\n' "pkgbase = demo-nightly-bin" $'\tpkgver = '"$2" $'\tpkgrel = 1' \ + $'\tsource = demo.vim' \ + $'\tsource_x86_64 = '"$3"'::https://example.com/releases/download/nightly/demo-x86_64.tar.gz' \ + $'\tb2sums = '"$4" $'\tb2sums_x86_64 = '"$5" "" "pkgname = demo-nightly-bin" >"$1" +} + +@test "check_prepared_srcinfo: accepts new digits in the name of a SKIP arch source" { + write_srcinfo_arch "$TEST_TMPDIR/old.SRCINFO" 1.0.0 demo-20261014.tar.gz "$H1" SKIP + write_srcinfo_arch "$TEST_TMPDIR/new.SRCINFO" 1.1.0 demo-20261016.tar.gz "$H1" SKIP + run check_prepared_srcinfo "$TEST_TMPDIR/old.SRCINFO" "$TEST_TMPDIR/new.SRCINFO" 1.1.0 + [ "$status" -eq 0 ] +} + +@test "check_prepared_srcinfo: pairs an arch source with the checksums of its arch" { + write_srcinfo_arch "$TEST_TMPDIR/old.SRCINFO" 1.0.0 demo-20261014.tar.gz SKIP "$H1" + write_srcinfo_arch "$TEST_TMPDIR/new.SRCINFO" 1.1.0 demo-20261016.tar.gz SKIP "$H3" + run check_prepared_srcinfo "$TEST_TMPDIR/old.SRCINFO" "$TEST_TMPDIR/new.SRCINFO" 1.1.0 + [ "$status" -eq 1 ] +} + @test "check_prepared_srcinfo: rejects an added dependency" { write_srcinfo "$TEST_TMPDIR/old.SRCINFO" 1.0.0 2 "$H1" "$H2" write_srcinfo "$TEST_TMPDIR/new.SRCINFO" 1.1.0 1 "$H3" "$H4"