diff --git a/CLAUDE.md b/CLAUDE.md index 29ea2d4..64c20f2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -29,7 +29,7 @@ a run did the last reconcile. mirror, to decrease the load on `ftp.gnu.org`, but the owner selected the primary site. Do not change `SOURCE` to a secondary mirror without the approval of the owner. - **Paths are GNU's, at the root of the bucket, with the dot-files.** Only the mirror uses - the prefix `.state/` and the file name `gnu.katoptra.org.directory.index.html`. The GNU + the prefix `.state/` and the file name `katoptra.org.directory.index.html`. The GNU tree does not have these two names. - **GNU's mirror guidelines are applicable to each page that the mirror serves.** Text must be "as short as possible, and strictly explanatory". Images and logos are not permitted. diff --git a/README.md b/README.md index 7dafef5..ce271aa 100644 --- a/README.md +++ b/README.md @@ -78,6 +78,8 @@ flowchart LR directory. `PAGE_FOOT` is the last line of each page. This line identifies the mirror and tells how frequently the mirror gets an update. It also gives an address for problem reports. + `INDEX_FILES` is `index.html index.htm`. In a directory with one of these files, the URL + of the directory serves that file, as on ftp.gnu.org. - **The canary.** `CANARY` is `aspell/dict/0index.html`. This file has plain `http://` links, and each HTML rewriter of Cloudflare changes such links. - **Freshness.** `FRESH_KEY` is `mirror-updated-timestamp.txt`, the file where ftp.gnu.org @@ -145,14 +147,15 @@ A run gets four values from one vault item, `gnu`: ### 4. The zone -The zone has three Cloudflare rules. Each rule is only for the hostname of the mirror. You -set these rules one time, out of the pipeline. The pipeline does not change them. +The zone has three Cloudflare rules for the mirror. Each rule is only for the hostnames of +the rsync mirrors of katoptra. You set these rules one time, out of the pipeline. The +pipeline does not change them. | Rule | Function | |---|---| | Configuration Rule | It sets Email Obfuscation, Rocket Loader, Automatic HTTPS Rewrites and Browser Integrity Check to off. The first three change the HTML between the bucket and the client. The fourth sends 403 to Perl and Python clients. If one of the four is on again, the canary check stops the run. | | Cache Rule | Bypass. Without this rule, a client can get a previous copy of a page or of the timestamp from the cache. | -| Transform Rule | It changes each path with `/` at the end to `concat(http.request.uri.path, http.host, ".directory.index.html")`. This includes the root. | +| Transform Rule | It changes each path with `/` at the end to `concat(http.request.uri.path, "katoptra.org.directory.index.html")`. This includes the root. | ### 5. Do the checks, run it, schedule it diff --git a/Taskfile.yml b/Taskfile.yml index aac20d4..0cc0fe0 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -31,9 +31,13 @@ vars: # A listing must have more lines than this value, or it is a truncated listing. The tree # has approximately 43,850 files. LIST_FLOOR: 40000 - # INDEX is the key suffix of the directory pages. The Transform Rule of the zone uses only - # this name. If an upstream file has this name, the bucket keeps the page, not the file. - INDEX: '{{.HOST}}.directory.index.html' + # INDEX is the key suffix of the directory pages. The rsync mirrors of katoptra use one + # name, and the Transform Rules of the zone use it for their hosts. If an upstream file has + # this name, the bucket keeps the page, not the file. + INDEX: katoptra.org.directory.index.html + # INDEX_FILES are the upstream index files, in sequence. The page key of a directory with + # such a file is a copy of that file. Thus, the URL of that directory serves the file. + INDEX_FILES: index.html index.htm # PAGE_FOOT is the last line of each directory page. In GNU's mirror guidelines, text # must be short and "strictly explanatory". The only link that is permitted is a link for # bug reports. That link is an email address, which must receive mail. diff --git a/render.txt b/render.txt index a78a79f..9a75360 100644 --- a/render.txt +++ b/render.txt @@ -110,7 +110,7 @@ task: [verify] cd /work/staging && { test ! -f /tlpkg/texlive.tlpdb || { cmp /tl /^srccontainerchecksum /{print TL "/archive/" n ".source.tar.xz"}' /tlpkg/texlive.tlpdb \ | LC_ALL=C sort | LC_ALL=C comm -23 - /work/.run/tl/after.txt > /work/.run/tl/missing.txt && test ! -s /work/.run/tl/missing.txt; }; } task: [label] python3 -I - /work/staging $'application/x-gzip .gz .tgz .taz .tz .svgz\napplication/x-bzip2 .bz2 .tbz2 .tbz\napplication/x-xz .xz .txz\napplication/x-compress .Z\napplication/zstd .zst .tzst\napplication/x-lzip .lz\napplication/x-lzma .lzma' > /work/.run/labels.raw <<'PY' -import mimetypes, os, sys +import codecs, mimetypes, os, sys root, table = sys.argv[1], sys.argv[2] endings = {s: f[0] for f in map(str.split, table.splitlines()) if f for s in f[1:]} @@ -142,13 +142,28 @@ def label(name, head): return t if agrees else seen return t +# One MiB at a time: some text files are large (FILES.byname of CTAN is 27 MB). +def utf8(path): + d = codecs.getincrementaldecoder("utf-8")() + try: + with open(path, "rb") as h: + for part in iter(lambda: h.read(1 << 20), b""): + d.decode(part) + d.decode(b"", final=True) + except UnicodeDecodeError: + return False + return True + out = sys.stdout.buffer for d, _, files in os.walk(root): for f in files: p = os.path.join(d, f) with open(p, "rb") as h: head = h.read(1024) - out.write(os.fsencode(os.path.relpath(p, root)) + b"\t" + label(f, head).encode() + b"\n") + t = label(f, head) + if t.startswith("text/") and utf8(p): + t += "; charset=utf-8" + out.write(os.fsencode(os.path.relpath(p, root)) + b"\t" + t.encode() + b"\n") PY task: [label] LC_ALL=C sort /work/.run/labels.raw > /work/.run/labels.txt && rm /work/.run/labels.raw @@ -243,7 +258,7 @@ task: [rebuild] mv /work/.run/applied.new /work/.run/applied.txt task: [rebuild] rm -f /work/.run/rebuild-now task: [reconcile] awk -F'\t' '!("" in s) { s[""] = 1; print "" } { n = split($1, c, "/"); d = ""; for (i = 1; i < n; i++) { d = d c[i] "/"; if (!(d in s)) { s[d] = 1; print d } } }' /work/.run/applied.txt | awk '$0 != "" { print substr($0, 1, length($0) - 1) }' | LC_ALL=C sort > /work/.run/index-keys.txt -task: [reconcile] cut -f1 /work/.run/bucket.txt | grep -vE '^\.state/' | awk -v i="/gnu.katoptra.org.directory.index.html" '{ s = "/" $0 } substr(s, length(s) - length(i) + 1) != i' | LC_ALL=C comm -23 - /work/.run/index-keys.txt | LC_ALL=C comm -23 - /work/.run/paths.txt > /work/.run/orphans.txt +task: [reconcile] cut -f1 /work/.run/bucket.txt | grep -vE '^\.state/' | awk -v i="/katoptra.org.directory.index.html" '{ s = "/" $0 } substr(s, length(s) - length(i) + 1) != i' | LC_ALL=C comm -23 - /work/.run/index-keys.txt | LC_ALL=C comm -23 - /work/.run/paths.txt > /work/.run/orphans.txt task: [remove] rm -f /work/.run/del-*; split -l 1000 -a 4 /work/.run/orphans.txt /work/.run/del- task: [remove] for f in /work/.run/del-*; do awk 'BEGIN { printf "{\"Objects\":[" } { gsub(/\\/, "\\\\"); gsub(/"/, "\\\"") printf "%s{\"Key\":\"%s\"}", (NR > 1 ? "," : ""), $0 } END { print "],\"Quiet\":true}" }' "$f" \ @@ -264,12 +279,12 @@ case $rc in *) exit $rc ;; esac -task: [pages] mkdir -p /work/staging /work/.run/slash && find /work/staging /work/.run/slash -mindepth 1 -delete +task: [pages] mkdir -p /work/staging /work/.run/slash && find /work/staging /work/.run/slash -mindepth 1 -delete && : > /work/.run/index-real.txt task: [pages] LC_ALL=C comm -3 /work/.run/indexed.txt /work/.run/applied.txt | awk -F'\t' '{ print ($1 == "" ? $2 : $1) }' | awk -F'\t' '!("" in s) { s[""] = 1; print "" } { n = split($1, c, "/"); d = ""; for (i = 1; i < n; i++) { d = d c[i] "/"; if (!(d in s)) { s[d] = 1; print d } } }' | LC_ALL=C sort > /work/.run/index-dirty.txt task: [pages] awk -F'\t' '!("" in s) { s[""] = 1; print "" } { n = split($1, c, "/"); d = ""; for (i = 1; i < n; i++) { d = d c[i] "/"; if (!(d in s)) { s[d] = 1; print d } } }' /work/.run/applied.txt | LC_ALL=C sort > /work/.run/index-dirs.txt -task: [pages] LC_ALL=C comm -23 /work/.run/index-dirty.txt /work/.run/index-dirs.txt | awk -v i="gnu.katoptra.org.directory.index.html" '{ print $0 i } $0 != "" { print substr($0, 1, length($0) - 1) }' > /work/.run/index-gone.txt +task: [pages] LC_ALL=C comm -23 /work/.run/index-dirty.txt /work/.run/index-dirs.txt | awk -v i="katoptra.org.directory.index.html" '{ print $0 i } $0 != "" { print substr($0, 1, length($0) - 1) }' > /work/.run/index-gone.txt task: [pages] LC_ALL=C comm -12 /work/.run/index-dirty.txt /work/.run/index-dirs.txt > /work/.run/index-make.txt task: [pages] test ! -s /work/.run/index-make.txt || awk -v s="/work/staging/" '{ print s $0 }' /work/.run/index-make.txt | tr '\n' '\0' | xargs -0 mkdir -p task: [pages] test ! -s /work/.run/index-make.txt || awk -v r="/work/.run/slash/" '$0 != "" { n = split($0, c, "/"); d = "" @@ -279,26 +294,30 @@ task: [pages] test ! -s /work/.run/index-make.txt || awk -F'\t' -v OFS='\t' 'NR { n = split($1, c, "/"); d = "" for (i = 1; i < n; i++) { if ((d in want) && !((d c[i]) in seen)) { seen[d c[i]] = 1; print d, "d", c[i], "", "" }; d = d c[i] "/" } if (d in want) print d, "f", c[n], $2, $3 }' /work/.run/index-make.txt /work/.run/applied.txt \ -| LC_ALL=C sort | awk -F'\t' -v S="/work/staging/" -v R="/work/.run/slash/" -v I="gnu.katoptra.org.directory.index.html" -v F='A mirror of https://ftp.gnu.org/gnu/, updated twice a day. Sizes in bytes, times as upstream lists them. Report a problem to gnu@katoptra.org.' 'BEGIN { for (i = 0; i < 256; i++) ord[sprintf("%c", i)] = i } +| LC_ALL=C sort | awk -F'\t' -v S="/work/staging/" -v R="/work/.run/slash/" -v I="katoptra.org.directory.index.html" -v F='A mirror of https://ftp.gnu.org/gnu/, updated twice a day. Sizes in bytes, times as upstream lists them. Report a problem to gnu@katoptra.org.' \ + -v N='index.html index.htm' -v X="/work/.run/index-real.txt" 'BEGIN { for (i = 0; i < 256; i++) ord[sprintf("%c", i)] = i } function enc(s, out, j, c) { out = ""; for (j = 1; j <= length(s); j++) { c = substr(s, j, 1) out = out ((c ~ /[A-Za-z0-9\/._~-]/) ? c : sprintf("%%%02X", ord[c])) } return out } + BEGIN { n = split(N, nm, " "); for (i = n; i >= 1; i--) rank[nm[i]] = i } function esc(s) { gsub(/&/, "\\&", s); gsub(//, "\\>", s); gsub(/"/, "\\"", s); return s } - function open(d) { cur = d; live = 1 + function open(d) { cur = d; live = 1; real = "" p = "Index of /" esc(d) "\n" p = p "\n" p = p "\n" p = p "

Index of /" esc(d) "

\n" if (d != "") p = p "\n" } function shut( f, b, k, e) { p = p "
NameLast modifiedSize
../
" (F == "" ? "" : "

" sprintf(F, enc(cur)) "

") "\n" - f = S cur I; printf("%s", p) > f; close(f) + if (real == "") { f = S cur I; printf("%s", p) > f; close(f) } else print cur real "\t" cur I > X if (cur != "") { b = substr(cur, 1, length(cur) - 1); k = split(b, e, "/") f = R k "/" b; printf("%s", p) > f; close(f) } } !live || $1 != cur { if (live) shut(); open($1) } $2 == "d" { p = p "" esc($3) "/\n"; next } + ($3 in rank) && (real == "" || rank[$3] < rank[real]) { real = $3 } { p = p "" esc($3) "" $5 "" $4 "\n" } END { if (live) shut() }' -task: [index] test ! -s /work/.run/index-make.txt || aws s3 cp --no-progress --cli-connect-timeout 60 --cli-read-timeout 300 --recursive --content-type text/html /work/staging/ s3://gnu/ -task: [index] for d in /work/.run/slash/*/; do test -d "$d" || continue; aws s3 cp --no-progress --cli-connect-timeout 60 --cli-read-timeout 300 --recursive --content-type text/html "$d" s3://gnu/; done +task: [index] test ! -s /work/.run/index-make.txt || aws s3 cp --no-progress --cli-connect-timeout 60 --cli-read-timeout 300 --recursive --content-type 'text/html; charset=utf-8' /work/staging/ s3://gnu/ +task: [index] for d in /work/.run/slash/*/; do test -d "$d" || continue; aws s3 cp --no-progress --cli-connect-timeout 60 --cli-read-timeout 300 --recursive --content-type 'text/html; charset=utf-8' "$d" s3://gnu/; done +task: [index-copy] while IFS="$(printf '\t')" read -r s k; do aws s3api copy-object --bucket gnu --copy-source "gnu/$s" --key "$k" --metadata-directive COPY > /dev/null; echo "copy: $s to $k"; done < /work/.run/index-real.txt task: [remove] rm -f /work/.run/del-*; split -l 1000 -a 4 /work/.run/index-gone.txt /work/.run/del- task: [remove] for f in /work/.run/del-*; do awk 'BEGIN { printf "{\"Objects\":[" } { gsub(/\\/, "\\\\"); gsub(/"/, "\\\"") printf "%s{\"Key\":\"%s\"}", (NR > 1 ? "," : ""), $0 } END { print "],\"Quiet\":true}" }' "$f" \ @@ -309,14 +328,14 @@ function enc(s, out, j, c) { out = ""; for (j = 1; j <= length(s); j++) { c = s out = out ((c ~ /[A-Za-z0-9\/._~-]/) ? c : sprintf("%%%02X", ord[c])) } return out } { print $0 "\t" enc($0) }' | cut -f2); want=$(awk -F'\t' -v k="$k" '$1 == k { print $2 }' /work/.run/upstream.txt); h=$(curl -fsS --connect-timeout 15 --max-time 60 --retry 6 --retry-connrefused --retry-max-time 600 -r 0-0 -D - -o /dev/null "https://gnu.katoptra.org/$u"); got=$(printf '%s\n' "$h" | awk 'tolower($1) == "content-range:" { split($3, a, "/"); r = a[2] } tolower($1) == "content-length:" { c = $2 } $2 == "416" { z = 1 } - END { sub(/\r/, "", r); sub(/\r/, "", c); print (r != "" ? r : (z ? 0 : c)) }'); ct=$(printf '%s\n' "$h" | awk 'tolower($1) == "content-type:" { sub(/\r$/, ""); print $2 }'); echo "$k: $got bytes (listing: $want), served as ${ct:-no type} (label: ${t:-none})"; test "$got" = "$want" || exit 1; case "https://gnu.katoptra.org" in http*) test -z "$t" || test "$ct" = "$t" || exit 1 ;; esac; done + END { sub(/\r/, "", r); sub(/\r/, "", c); print (r != "" ? r : (z ? 0 : c)) }'); ct=$(printf '%s\n' "$h" | awk 'tolower($1) == "content-type:" { sub(/\r$/, ""); sub(/^[^:]*:[ \t]*/, ""); print }'); echo "$k: $got bytes (listing: $want), served as ${ct:-no type} (label: ${t:-none})"; test "$got" = "$want" || exit 1; case "https://gnu.katoptra.org" in http*) test -z "$t" || test "$ct" = "$t" || exit 1 ;; esac; done task: [smoke] echo "no TL subtree: no tlpdb to read back" -task: [smoke] test ! -s /work/.run/index-make.txt || { d=$(tail -1 /work/.run/index-make.txt); want=$(wc -c < "/work/staging/${d}gnu.katoptra.org.directory.index.html" | tr -d ' '); test -n "$want"; case "https://gnu.katoptra.org" in http*) b=1 ;; *) b=0 ;; esac; printf '%s\n' "$d" | awk -v i="gnu.katoptra.org.directory.index.html" -v b="$b" '{ print $0 i } b == 1 && $0 != "" { print substr($0, 1, length($0) - 1) }' | awk 'BEGIN { for (i = 0; i < 256; i++) ord[sprintf("%c", i)] = i } +task: [smoke] test ! -s /work/.run/index-make.txt || { d=$(tail -1 /work/.run/index-make.txt); r() { h=$(curl -fsS --connect-timeout 15 --max-time 60 --retry 6 --retry-connrefused --retry-max-time 600 -r 0-0 -D - -o /dev/null "https://gnu.katoptra.org/$(printf '%s\n' "$1" | awk 'BEGIN { for (i = 0; i < 256; i++) ord[sprintf("%c", i)] = i } function enc(s, out, j, c) { out = ""; for (j = 1; j <= length(s); j++) { c = substr(s, j, 1) - out = out ((c ~ /[A-Za-z0-9\/._~-]/) ? c : sprintf("%%%02X", ord[c])) } return out } { print $0 "\t" enc($0) }' | while IFS="$(printf '\t')" read -r k u; do got=$(curl -fsS --connect-timeout 15 --max-time 60 --retry 6 --retry-connrefused --retry-max-time 600 -r 0-0 -D - -o /dev/null "https://gnu.katoptra.org/$u" | awk 'tolower($1) == "content-range:" { split($3, a, "/"); r = a[2] } + out = out ((c ~ /[A-Za-z0-9\/._~-]/) ? c : sprintf("%%%02X", ord[c])) } return out } { print $0 "\t" enc($0) }' | cut -f2)"); echo "$(printf '%s\n' "$h" | awk 'tolower($1) == "content-range:" { split($3, a, "/"); r = a[2] } tolower($1) == "content-length:" { c = $2 } $2 == "416" { z = 1 } - END { sub(/\r/, "", r); sub(/\r/, "", c); print (r != "" ? r : (z ? 0 : c)) }'); echo "$k: $got bytes (page: $want)"; test "$got" = "$want" || exit 1; done; } + END { sub(/\r/, "", r); sub(/\r/, "", c); print (r != "" ? r : (z ? 0 : c)) }') bytes, type $(printf '%s\n' "$h" | awk 'tolower($1) == "content-type:" { sub(/\r$/, ""); sub(/^[^:]*:[ \t]*/, ""); print }')"; }; s=$(awk -F'\t' -v k="${d}katoptra.org.directory.index.html" '$2 == k { print $1 }' /work/.run/index-real.txt); if test -n "$s"; then want=$(r "$s"); else want=$(wc -c < "/work/staging/${d}katoptra.org.directory.index.html" | tr -d ' '); test -n "$want"; want="$want bytes"; fi; got=$(r "${d}katoptra.org.directory.index.html"); test -n "$s" || got=${got%%,*}; echo "${d}katoptra.org.directory.index.html: $got (${s:-page}: $want)"; test "$got" = "$want"; case "https://gnu.katoptra.org" in http*) test -z "$d" || { p=${d%/}; want=$(wc -c < "/work/.run/slash/$(printf '%s\n' "$p" | awk -F/ '{ print NF }')/$p" | tr -d ' '); test -n "$want"; got=$(r "$p"); got=${got%%,*}; echo "$p: $got (listing: $want bytes)"; test "$got" = "$want bytes"; } ;; esac; } task: [smoke] h=aspell/dict/0index.html; n=$(awk -F'\t' -v k="$h" '$1 == k { print $2 }' /work/.run/applied.txt 2>/dev/null); case "https://gnu.katoptra.org" in http*) ;; *) n= ;; esac; if test -z "$n"; then echo "canary skipped ($h is not in the state, or the URL is not http)"; else aws s3 cp --no-progress --cli-connect-timeout 60 --cli-read-timeout 300 "s3://gnu/$h" /work/.run/canary.html && curl -fsS --connect-timeout 15 --max-time 60 --retry 6 --retry-connrefused --retry-max-time 600 -A "libwww-perl/6.67" "https://gnu.katoptra.org/$h" | cmp - /work/.run/canary.html && echo "$h: $n bytes, served to a libwww-perl client as the bucket holds them"; fi task: [smoke] g=$(awk -F'\t' '$1 ~ /\.tar\.gz$/ && $2 > 0 { print $1; exit }' /work/.run/applied.txt 2>/dev/null); case "https://gnu.katoptra.org" in http*) ;; *) g= ;; esac; if test -z "$g"; then echo "content-encoding check skipped (no non-empty .tar.gz in the state, or the URL is not http)"; else u=$(printf '%s\n' "$g" | awk 'BEGIN { for (i = 0; i < 256; i++) ord[sprintf("%c", i)] = i } function enc(s, out, j, c) { out = ""; for (j = 1; j <= length(s); j++) { c = substr(s, j, 1)