diff --git a/CLAUDE.md b/CLAUDE.md index e8e94ea..7a2d439 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -33,7 +33,7 @@ a run did the last reconcile. `rsync://ftp.gnu.org/alpha/`. The alpha list is on GNU's mirror page, in "Mirroring the GNU Alpha release server". - **Paths are GNU's, at the root of the bucket, with the dot-files.** Only the mirror uses - the prefix `.state/` and the file name `gnu-alpha.katoptra.org.directory.index.html`. The + the prefix `.state/` and the file name `katoptra.org.directory.index.html`. The alpha tree does not have these two names. - **GNU's mirror guidelines are applicable to each page that the mirror serves.** Text must be "as short as possible, and strictly explanatory". Images and logos are not permitted. diff --git a/README.md b/README.md index a12e5d8..74166fe 100644 --- a/README.md +++ b/README.md @@ -150,14 +150,15 @@ A run gets four values from one vault item, `gnu-alpha`: ### 4. The zone -The zone has three Cloudflare rules. Each rule is only for the hostname of the mirror. You -set these rules one time, out of the pipeline. The pipeline does not change them. +The zone has three Cloudflare rules for the mirror. Each rule is only for the hostnames of +the rsync mirrors of katoptra. You set these rules one time, out of the pipeline. The +pipeline does not change them. | Rule | Function | |---|---| | Configuration Rule | It sets Email Obfuscation, Rocket Loader, Automatic HTTPS Rewrites and Browser Integrity Check to off. The first three change the HTML between the bucket and the client. The fourth sends 403 to Perl and Python clients. If one of the four is on again, the canary check stops the run. | | Cache Rule | Bypass. Without this rule, a client can get a previous copy of a page or of the timestamp from the cache. | -| Transform Rule | It changes each path with `/` at the end to `concat(http.request.uri.path, http.host, ".directory.index.html")`. This includes the root. | +| Transform Rule | It changes each path with `/` at the end to `concat(http.request.uri.path, "katoptra.org.directory.index.html")`. This includes the root. | ### 5. Do the checks, run it, schedule it diff --git a/Taskfile.yml b/Taskfile.yml index 936b3e1..71a4d51 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -31,9 +31,10 @@ vars: # A listing must have more lines than this value, or it is a truncated listing. The tree # has approximately 7,200 files. LIST_FLOOR: 6500 - # INDEX is the key suffix of the directory pages. The Transform Rule of the zone uses only - # this name. If an upstream file has this name, the bucket keeps the page, not the file. - INDEX: '{{.HOST}}.directory.index.html' + # INDEX is the key suffix of the directory pages. The rsync mirrors of katoptra use one + # name, and the Transform Rules of the zone use it for their hosts. If an upstream file has + # this name, the bucket keeps the page, not the file. + INDEX: katoptra.org.directory.index.html # PAGE_FOOT is the last line of each directory page. In GNU's mirror guidelines, text # must be short and "strictly explanatory". The only link that is permitted is a link for # bug reports. That link is an email address, which must receive mail. The value is one diff --git a/render.txt b/render.txt index 13bf114..efb84d4 100644 --- a/render.txt +++ b/render.txt @@ -110,7 +110,7 @@ task: [verify] cd /work/staging && { test ! -f /tlpkg/texlive.tlpdb || { cmp /tl /^srccontainerchecksum /{print TL "/archive/" n ".source.tar.xz"}' /tlpkg/texlive.tlpdb \ | LC_ALL=C sort | LC_ALL=C comm -23 - /work/.run/tl/after.txt > /work/.run/tl/missing.txt && test ! -s /work/.run/tl/missing.txt; }; } task: [label] python3 -I - /work/staging $'application/x-gzip .gz .tgz .taz .tz .svgz\napplication/x-bzip2 .bz2 .tbz2 .tbz\napplication/x-xz .xz .txz\napplication/x-compress .Z\napplication/zstd .zst .tzst\napplication/x-lzip .lz\napplication/x-lzma .lzma' > /work/.run/labels.raw <<'PY' -import mimetypes, os, sys +import codecs, mimetypes, os, sys root, table = sys.argv[1], sys.argv[2] endings = {s: f[0] for f in map(str.split, table.splitlines()) if f for s in f[1:]} @@ -142,13 +142,28 @@ def label(name, head): return t if agrees else seen return t +# One MiB at a time: some text files are large (FILES.byname of CTAN is 27 MB). +def utf8(path): + d = codecs.getincrementaldecoder("utf-8")() + try: + with open(path, "rb") as h: + for part in iter(lambda: h.read(1 << 20), b""): + d.decode(part) + d.decode(b"", final=True) + except UnicodeDecodeError: + return False + return True + out = sys.stdout.buffer for d, _, files in os.walk(root): for f in files: p = os.path.join(d, f) with open(p, "rb") as h: head = h.read(1024) - out.write(os.fsencode(os.path.relpath(p, root)) + b"\t" + label(f, head).encode() + b"\n") + t = label(f, head) + if t.startswith("text/") and utf8(p): + t += "; charset=utf-8" + out.write(os.fsencode(os.path.relpath(p, root)) + b"\t" + t.encode() + b"\n") PY task: [label] LC_ALL=C sort /work/.run/labels.raw > /work/.run/labels.txt && rm /work/.run/labels.raw @@ -243,7 +258,7 @@ task: [rebuild] mv /work/.run/applied.new /work/.run/applied.txt task: [rebuild] rm -f /work/.run/rebuild-now task: [reconcile] awk -F'\t' '!("" in s) { s[""] = 1; print "" } { n = split($1, c, "/"); d = ""; for (i = 1; i < n; i++) { d = d c[i] "/"; if (!(d in s)) { s[d] = 1; print d } } }' /work/.run/applied.txt | awk '$0 != "" { print substr($0, 1, length($0) - 1) }' | LC_ALL=C sort > /work/.run/index-keys.txt -task: [reconcile] cut -f1 /work/.run/bucket.txt | grep -vE '^\.state/' | awk -v i="/gnu-alpha.katoptra.org.directory.index.html" '{ s = "/" $0 } substr(s, length(s) - length(i) + 1) != i' | LC_ALL=C comm -23 - /work/.run/index-keys.txt | LC_ALL=C comm -23 - /work/.run/paths.txt > /work/.run/orphans.txt +task: [reconcile] cut -f1 /work/.run/bucket.txt | grep -vE '^\.state/' | awk -v i="/katoptra.org.directory.index.html" '{ s = "/" $0 } substr(s, length(s) - length(i) + 1) != i' | LC_ALL=C comm -23 - /work/.run/index-keys.txt | LC_ALL=C comm -23 - /work/.run/paths.txt > /work/.run/orphans.txt task: [remove] rm -f /work/.run/del-*; split -l 1000 -a 4 /work/.run/orphans.txt /work/.run/del- task: [remove] for f in /work/.run/del-*; do awk 'BEGIN { printf "{\"Objects\":[" } { gsub(/\\/, "\\\\"); gsub(/"/, "\\\"") printf "%s{\"Key\":\"%s\"}", (NR > 1 ? "," : ""), $0 } END { print "],\"Quiet\":true}" }' "$f" \ @@ -264,12 +279,12 @@ case $rc in *) exit $rc ;; esac -task: [pages] mkdir -p /work/staging /work/.run/slash && find /work/staging /work/.run/slash -mindepth 1 -delete +task: [pages] mkdir -p /work/staging /work/.run/slash && find /work/staging /work/.run/slash -mindepth 1 -delete && : > /work/.run/index-real.txt task: [pages] LC_ALL=C comm -3 /work/.run/indexed.txt /work/.run/applied.txt | awk -F'\t' '{ print ($1 == "" ? $2 : $1) }' | awk -F'\t' '!("" in s) { s[""] = 1; print "" } { n = split($1, c, "/"); d = ""; for (i = 1; i < n; i++) { d = d c[i] "/"; if (!(d in s)) { s[d] = 1; print d } } }' | LC_ALL=C sort > /work/.run/index-dirty.txt task: [pages] awk -F'\t' '!("" in s) { s[""] = 1; print "" } { n = split($1, c, "/"); d = ""; for (i = 1; i < n; i++) { d = d c[i] "/"; if (!(d in s)) { s[d] = 1; print d } } }' /work/.run/applied.txt | LC_ALL=C sort > /work/.run/index-dirs.txt -task: [pages] LC_ALL=C comm -23 /work/.run/index-dirty.txt /work/.run/index-dirs.txt | awk -v i="gnu-alpha.katoptra.org.directory.index.html" '{ print $0 i } $0 != "" { print substr($0, 1, length($0) - 1) }' > /work/.run/index-gone.txt +task: [pages] LC_ALL=C comm -23 /work/.run/index-dirty.txt /work/.run/index-dirs.txt | awk -v i="katoptra.org.directory.index.html" '{ print $0 i } $0 != "" { print substr($0, 1, length($0) - 1) }' > /work/.run/index-gone.txt task: [pages] LC_ALL=C comm -12 /work/.run/index-dirty.txt /work/.run/index-dirs.txt > /work/.run/index-make.txt task: [pages] test ! -s /work/.run/index-make.txt || awk -v s="/work/staging/" '{ print s $0 }' /work/.run/index-make.txt | tr '\n' '\0' | xargs -0 mkdir -p task: [pages] test ! -s /work/.run/index-make.txt || awk -v r="/work/.run/slash/" '$0 != "" { n = split($0, c, "/"); d = "" @@ -279,26 +294,30 @@ task: [pages] test ! -s /work/.run/index-make.txt || awk -F'\t' -v OFS='\t' 'NR { n = split($1, c, "/"); d = "" for (i = 1; i < n; i++) { if ((d in want) && !((d c[i]) in seen)) { seen[d c[i]] = 1; print d, "d", c[i], "", "" }; d = d c[i] "/" } if (d in want) print d, "f", c[n], $2, $3 }' /work/.run/index-make.txt /work/.run/applied.txt \ -| LC_ALL=C sort | awk -F'\t' -v S="/work/staging/" -v R="/work/.run/slash/" -v I="gnu-alpha.katoptra.org.directory.index.html" -v F='A mirror of https://alpha.gnu.org/gnu/, updated twice a day. Sizes in bytes, times as upstream lists them. Report a problem to gnu-alpha@katoptra.org.' 'BEGIN { for (i = 0; i < 256; i++) ord[sprintf("%c", i)] = i } +| LC_ALL=C sort | awk -F'\t' -v S="/work/staging/" -v R="/work/.run/slash/" -v I="katoptra.org.directory.index.html" -v F='A mirror of https://alpha.gnu.org/gnu/, updated twice a day. Sizes in bytes, times as upstream lists them. Report a problem to gnu-alpha@katoptra.org.' \ + -v N='' -v X="/work/.run/index-real.txt" 'BEGIN { for (i = 0; i < 256; i++) ord[sprintf("%c", i)] = i } function enc(s, out, j, c) { out = ""; for (j = 1; j <= length(s); j++) { c = substr(s, j, 1) out = out ((c ~ /[A-Za-z0-9\/._~-]/) ? c : sprintf("%%%02X", ord[c])) } return out } + BEGIN { n = split(N, nm, " "); for (i = n; i >= 1; i--) rank[nm[i]] = i } function esc(s) { gsub(/&/, "\\&", s); gsub(//, "\\>", s); gsub(/"/, "\\"", s); return s } - function open(d) { cur = d; live = 1 + function open(d) { cur = d; live = 1; real = "" p = "Index of /" esc(d) "\n" p = p "\n" p = p "\n" p = p "

Index of /" esc(d) "

\n" if (d != "") p = p "\n" } function shut( f, b, k, e) { p = p "
NameLast modifiedSize
../
" (F == "" ? "" : "

" sprintf(F, enc(cur)) "

") "\n" - f = S cur I; printf("%s", p) > f; close(f) + if (real == "") { f = S cur I; printf("%s", p) > f; close(f) } else print cur real "\t" cur I > X if (cur != "") { b = substr(cur, 1, length(cur) - 1); k = split(b, e, "/") f = R k "/" b; printf("%s", p) > f; close(f) } } !live || $1 != cur { if (live) shut(); open($1) } $2 == "d" { p = p "" esc($3) "/\n"; next } + ($3 in rank) && (real == "" || rank[$3] < rank[real]) { real = $3 } { p = p "" esc($3) "" $5 "" $4 "\n" } END { if (live) shut() }' -task: [index] test ! -s /work/.run/index-make.txt || aws s3 cp --no-progress --cli-connect-timeout 60 --cli-read-timeout 300 --recursive --content-type text/html /work/staging/ s3://gnu-alpha/ -task: [index] for d in /work/.run/slash/*/; do test -d "$d" || continue; aws s3 cp --no-progress --cli-connect-timeout 60 --cli-read-timeout 300 --recursive --content-type text/html "$d" s3://gnu-alpha/; done +task: [index] test ! -s /work/.run/index-make.txt || aws s3 cp --no-progress --cli-connect-timeout 60 --cli-read-timeout 300 --recursive --content-type 'text/html; charset=utf-8' /work/staging/ s3://gnu-alpha/ +task: [index] for d in /work/.run/slash/*/; do test -d "$d" || continue; aws s3 cp --no-progress --cli-connect-timeout 60 --cli-read-timeout 300 --recursive --content-type 'text/html; charset=utf-8' "$d" s3://gnu-alpha/; done +task: [index-copy] while IFS="$(printf '\t')" read -r s k; do aws s3api copy-object --bucket gnu-alpha --copy-source "gnu-alpha/$s" --key "$k" --metadata-directive COPY > /dev/null; echo "copy: $s to $k"; done < /work/.run/index-real.txt task: [remove] rm -f /work/.run/del-*; split -l 1000 -a 4 /work/.run/index-gone.txt /work/.run/del- task: [remove] for f in /work/.run/del-*; do awk 'BEGIN { printf "{\"Objects\":[" } { gsub(/\\/, "\\\\"); gsub(/"/, "\\\"") printf "%s{\"Key\":\"%s\"}", (NR > 1 ? "," : ""), $0 } END { print "],\"Quiet\":true}" }' "$f" \ @@ -309,14 +328,14 @@ function enc(s, out, j, c) { out = ""; for (j = 1; j <= length(s); j++) { c = s out = out ((c ~ /[A-Za-z0-9\/._~-]/) ? c : sprintf("%%%02X", ord[c])) } return out } { print $0 "\t" enc($0) }' | cut -f2); want=$(awk -F'\t' -v k="$k" '$1 == k { print $2 }' /work/.run/upstream.txt); h=$(curl -fsS --connect-timeout 15 --max-time 60 --retry 6 --retry-connrefused --retry-max-time 600 -r 0-0 -D - -o /dev/null "https://gnu-alpha.katoptra.org/$u"); got=$(printf '%s\n' "$h" | awk 'tolower($1) == "content-range:" { split($3, a, "/"); r = a[2] } tolower($1) == "content-length:" { c = $2 } $2 == "416" { z = 1 } - END { sub(/\r/, "", r); sub(/\r/, "", c); print (r != "" ? r : (z ? 0 : c)) }'); ct=$(printf '%s\n' "$h" | awk 'tolower($1) == "content-type:" { sub(/\r$/, ""); print $2 }'); echo "$k: $got bytes (listing: $want), served as ${ct:-no type} (label: ${t:-none})"; test "$got" = "$want" || exit 1; case "https://gnu-alpha.katoptra.org" in http*) test -z "$t" || test "$ct" = "$t" || exit 1 ;; esac; done + END { sub(/\r/, "", r); sub(/\r/, "", c); print (r != "" ? r : (z ? 0 : c)) }'); ct=$(printf '%s\n' "$h" | awk 'tolower($1) == "content-type:" { sub(/\r$/, ""); sub(/^[^:]*:[ \t]*/, ""); print }'); echo "$k: $got bytes (listing: $want), served as ${ct:-no type} (label: ${t:-none})"; test "$got" = "$want" || exit 1; case "https://gnu-alpha.katoptra.org" in http*) test -z "$t" || test "$ct" = "$t" || exit 1 ;; esac; done task: [smoke] echo "no TL subtree: no tlpdb to read back" -task: [smoke] test ! -s /work/.run/index-make.txt || { d=$(tail -1 /work/.run/index-make.txt); want=$(wc -c < "/work/staging/${d}gnu-alpha.katoptra.org.directory.index.html" | tr -d ' '); test -n "$want"; case "https://gnu-alpha.katoptra.org" in http*) b=1 ;; *) b=0 ;; esac; printf '%s\n' "$d" | awk -v i="gnu-alpha.katoptra.org.directory.index.html" -v b="$b" '{ print $0 i } b == 1 && $0 != "" { print substr($0, 1, length($0) - 1) }' | awk 'BEGIN { for (i = 0; i < 256; i++) ord[sprintf("%c", i)] = i } +task: [smoke] test ! -s /work/.run/index-make.txt || { d=$(tail -1 /work/.run/index-make.txt); r() { h=$(curl -fsS --connect-timeout 15 --max-time 60 --retry 6 --retry-connrefused --retry-max-time 600 -r 0-0 -D - -o /dev/null "https://gnu-alpha.katoptra.org/$(printf '%s\n' "$1" | awk 'BEGIN { for (i = 0; i < 256; i++) ord[sprintf("%c", i)] = i } function enc(s, out, j, c) { out = ""; for (j = 1; j <= length(s); j++) { c = substr(s, j, 1) - out = out ((c ~ /[A-Za-z0-9\/._~-]/) ? c : sprintf("%%%02X", ord[c])) } return out } { print $0 "\t" enc($0) }' | while IFS="$(printf '\t')" read -r k u; do got=$(curl -fsS --connect-timeout 15 --max-time 60 --retry 6 --retry-connrefused --retry-max-time 600 -r 0-0 -D - -o /dev/null "https://gnu-alpha.katoptra.org/$u" | awk 'tolower($1) == "content-range:" { split($3, a, "/"); r = a[2] } + out = out ((c ~ /[A-Za-z0-9\/._~-]/) ? c : sprintf("%%%02X", ord[c])) } return out } { print $0 "\t" enc($0) }' | cut -f2)"); echo "$(printf '%s\n' "$h" | awk 'tolower($1) == "content-range:" { split($3, a, "/"); r = a[2] } tolower($1) == "content-length:" { c = $2 } $2 == "416" { z = 1 } - END { sub(/\r/, "", r); sub(/\r/, "", c); print (r != "" ? r : (z ? 0 : c)) }'); echo "$k: $got bytes (page: $want)"; test "$got" = "$want" || exit 1; done; } + END { sub(/\r/, "", r); sub(/\r/, "", c); print (r != "" ? r : (z ? 0 : c)) }') bytes, type $(printf '%s\n' "$h" | awk 'tolower($1) == "content-type:" { sub(/\r$/, ""); sub(/^[^:]*:[ \t]*/, ""); print }')"; }; s=$(awk -F'\t' -v k="${d}katoptra.org.directory.index.html" '$2 == k { print $1 }' /work/.run/index-real.txt); if test -n "$s"; then want=$(r "$s"); else want=$(wc -c < "/work/staging/${d}katoptra.org.directory.index.html" | tr -d ' '); test -n "$want"; want="$want bytes"; fi; got=$(r "${d}katoptra.org.directory.index.html"); test -n "$s" || got=${got%%,*}; echo "${d}katoptra.org.directory.index.html: $got (${s:-page}: $want)"; test "$got" = "$want"; case "https://gnu-alpha.katoptra.org" in http*) test -z "$d" || { p=${d%/}; want=$(wc -c < "/work/.run/slash/$(printf '%s\n' "$p" | awk -F/ '{ print NF }')/$p" | tr -d ' '); test -n "$want"; got=$(r "$p"); got=${got%%,*}; echo "$p: $got (listing: $want bytes)"; test "$got" = "$want bytes"; } ;; esac; } task: [smoke] h=emacs/.readme.shtml; n=$(awk -F'\t' -v k="$h" '$1 == k { print $2 }' /work/.run/applied.txt 2>/dev/null); case "https://gnu-alpha.katoptra.org" in http*) ;; *) n= ;; esac; if test -z "$n"; then echo "canary skipped ($h is not in the state, or the URL is not http)"; else aws s3 cp --no-progress --cli-connect-timeout 60 --cli-read-timeout 300 "s3://gnu-alpha/$h" /work/.run/canary.html && curl -fsS --connect-timeout 15 --max-time 60 --retry 6 --retry-connrefused --retry-max-time 600 -A "libwww-perl/6.67" "https://gnu-alpha.katoptra.org/$h" | cmp - /work/.run/canary.html && echo "$h: $n bytes, served to a libwww-perl client as the bucket holds them"; fi task: [smoke] g=$(awk -F'\t' '$1 ~ /\.tar\.gz$/ && $2 > 0 { print $1; exit }' /work/.run/applied.txt 2>/dev/null); case "https://gnu-alpha.katoptra.org" in http*) ;; *) g= ;; esac; if test -z "$g"; then echo "content-encoding check skipped (no non-empty .tar.gz in the state, or the URL is not http)"; else u=$(printf '%s\n' "$g" | awk 'BEGIN { for (i = 0; i < 256; i++) ord[sprintf("%c", i)] = i } function enc(s, out, j, c) { out = ""; for (j = 1; j <= length(s); j++) { c = substr(s, j, 1)