From 0e8cffa4f18a17ca87451cad1729ccc9de2a79c5 Mon Sep 17 00:00:00 2001
From: Josh Vaughen <josh@vaughen.net>
Date: Fri, 16 Oct 2026 18:01:16 -0700
Subject: [PATCH 1/2] feat(rsync): serve index files as directory pages, label
 UTF-8 charset

INDEX_FILES (empty by default) names the upstream index files, in
sequence. For a directory with such a file, pages writes the listing
only to the slashless key and records "<source key> TAB <page key>" in
.run/index-real.txt. index then copies the file to the page key with
one server-side copy-object (--metadata-directive COPY), so /dir/
serves the same bytes, Content-Type and ETag as the index file.

label adds "; charset=utf-8" to a text/* type when the whole file
decodes as strict UTF-8. Generated pages upload as
"text/html; charset=utf-8".

smoke compares the full Content-Type value (CTYPE), not the first
word, and for an index directory checks the copy against its source.

index-real.txt starts each line with a full key: read with a TAB in
IFS drops an empty first field, which the root directory would be.
---
 CLAUDE.md                                     |  6 ++
 README.md                                     | 49 ++++++----
 engines/rsync.yml                             | 95 +++++++++++++++----
 examples/rsync/Taskfile.yml                   | 44 +++++++--
 examples/rsync/fixtures/run-pages/applied.txt |  2 +
 .../fixtures/run-pages/want/index-make.txt    |  1 +
 .../fixtures/run-pages/want/index-real.txt    |  1 +
 .../rsync/fixtures/run-pages/want/slash/1/a   |  1 +
 .../fixtures/run-pages/want/slash/2/a/doc     |  8 ++
 .../a/ctan.katoptra.org.directory.index.html  |  1 +
 .../rsync/fixtures/run-smoke/index-real.txt   |  0
 examples/rsync/render.txt                     | 45 ++++++---
 12 files changed, 197 insertions(+), 56 deletions(-)
 create mode 100644 examples/rsync/fixtures/run-pages/want/index-real.txt
 create mode 100644 examples/rsync/fixtures/run-pages/want/slash/2/a/doc
 create mode 100644 examples/rsync/fixtures/run-smoke/index-real.txt

diff --git a/CLAUDE.md b/CLAUDE.md
index 26a9ac4..44616fb 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -103,6 +103,12 @@ Read that section before you write.
   cause, `SESSION` and `UPLOAD` in `proton.yml` give `ROOT_DIR/.run` in full. Also, if a
   mirror sets `IMAGE` in its root vars, a run can use the default of the engine. If the
   toolbox entry of `includes:` sets `IMAGE`, the run always uses that value.
+- Do not put an empty field first in a line that `read` gets with a TAB in `IFS`. A TAB is
+  white space for `IFS`. Thus, `read` removes the empty field, and the next field moves to
+  the first variable (verified, 3.53.1). The root directory is an empty path. For this
+  cause, each line of `.run/index-real.txt` starts with a full key.
+- Do not give `--no-progress` (in `AWS_FLAGS`) to `aws s3api`. Only `aws s3` accepts it.
+  `s3api` stops with `Unknown options` (verified, aws-cli 2.36.24).
 
 ## Verifying a change
 
diff --git a/README.md b/README.md
index f54b540..20d9012 100644
--- a/README.md
+++ b/README.md
@@ -345,12 +345,12 @@ The verbs with a broken line are hooks. These hooks do no work until a mirror se
 | `fetch` | Copies the files of the batch into `staging/` with rsync, and copies the target of each symlink. If upstream deletes a path after the listing, `fetch` does not copy it |
 | `verify` | A hook. If `TL_KEY` has a value: it compares each signed file and each container in the batch with the tlpdb |
 | `publish` | Runs `label` on the batch. Then it runs `aws s3 cp --recursive` one time for each type, with the files of each type in a different tree. It sends each file as one PutObject, and it makes no listing of the bucket. The engine does not use `aws s3 sync`. The `FRESH_KEY` file goes last |
-| `label` | Writes `.run/labels.txt`: the Content-Type of each file in staging, from its name and its first KiB. Refer to [Content types](#content-types) |
+| `label` | Writes `.run/labels.txt`: the Content-Type of each file in staging, from its name and its first KiB. A text type gets a charset if the file is UTF-8. Refer to [Content types](#content-types) |
 | `checkpoint` | Runs `merge` to add the uploaded files to the state. Then it uploads the state as one PutObject, and it deletes the files in staging |
 | `delete` | Removes the keys in `deleted.txt` from the bucket, 1,000 in each call, and removes them from the state. It waits for the run that uploads the last batch |
 | `reconcile` | If `due` wrote `.run/reconcile`, it runs `rebuild`. Then it deletes each key that is not a path of upstream, a key in `OWN`, or a directory of the state. Then it runs `reconciled` |
-| `index` | A hook. If `INDEX` has a value: it runs `pages`, uploads the two sets of keys, and removes the keys of the directories that became empty. Then it moves `.state/indexed.txt.xz` to the new state. A mirror with a different landing page overrides `index` |
-| `pages` | Writes a page for each directory that the run changed, from the state, into staging. The keys with no slash at the end go in one tree for each depth |
+| `index` | A hook. If `INDEX` has a value: it runs `pages`, uploads the two sets of keys, copies the index files of `.run/index-real.txt` to their page keys, and removes the keys of the directories that became empty. Then it moves `.state/indexed.txt.xz` to the new state. A mirror with a different landing page overrides `index` |
+| `pages` | Writes a page for each directory that the run changed, from the state, into staging. The keys with no slash at the end go in one tree for each depth. A directory with a file in `INDEX_FILES` gets only the key with no slash, and a line in `.run/index-real.txt` |
 | `smoke` | Reads a sample back from `HOST`. The list below the table gives each item of the sample. Then it runs `fresh` and `smoke-mirror` |
 | `fresh` | If `FRESH_KEY` has a value: it reads the clock of upstream in that file, from `HOST`. If the clock is more than `FRESH_HOURS` before the time of the run, it stops the run |
 | `smoke-mirror` | A hook, empty here. A mirror with more items to read back writes it |
@@ -365,12 +365,13 @@ between the bucket and `.run`. A verb of a mirror can also run them.
 correct:
 
 - One key of each type that the run published. Its length in bytes must agree with the
-  listing, and its Content-Type must agree with the label.
+  listing, and its full Content-Type, with the charset, must agree with the label.
 - The `FRESH_KEY` file, for its length. `smoke` reads it only if the state has it. While
   the engine fills an empty bucket, the runs that stop at `MAX_BATCHES` do not upload the
   decision batch. The bucket gives 404 for a key that it did not contain before.
 - If `TL` has a value: `texlive.tlpdb.sha512`. It must be the same as the verified copy.
-- If `INDEX` has a value: one page that `index` wrote, at its two keys.
+- If `INDEX` has a value: one page that `index` wrote, at its two keys. If the page key is
+  a copy of an index file, its length and its Content-Type must agree with that file.
 - If `CANARY` has a value: that file, read as `libwww-perl`. It must be the same as the copy
   in the bucket, byte for byte.
 - The first `.tar.gz` that is not empty. It must have no `Content-Encoding`.
@@ -494,8 +495,17 @@ each page has two keys:
 A `<base href>` lets one document serve the two URLs. A file system cannot contain `a/b` and
 `a/b/` at the same time. Thus, `pages` puts the keys with no slash in one tree for each
 depth, and `index` uploads one tree at a time. `index` gives these keys
-`--content-type text/html`. Without it, a key with no suffix gets `binary/octet-stream`,
-and a browser downloads the page and does not show it.
+`--content-type 'text/html; charset=utf-8'`. Without it, a key with no suffix gets
+`binary/octet-stream`, and a browser downloads the page and does not show it.
+
+`INDEX_FILES` gives the names of the index files, in sequence, for example
+`index.html index.htm`. A directory can contain a file with one of these names. Then:
+
+- `<dir>/<INDEX>` is a server-side copy of the first such file. The copy has the same
+  bytes, the same Content-Type and the same ETag. Thus, `/dir/` serves the index file, as
+  on a mirror with Apache.
+- `<dir>` keeps the listing.
+- If upstream removes the index file, the next run writes the listing to the two keys.
 
 No page goes into the state. `merge` adds to the state only the files that are in staging
 and also in the batch, and a page is in no batch.
@@ -506,15 +516,17 @@ next run writes all the pages again, one time. Thus, the pages of the directorie
 not change also get a change to the HTML of the page or to `PAGE_FOOT`. `reconcile` does
 not delete the two keys.
 
-The zone has one rule for each host:
+The rsync mirrors of katoptra use one `INDEX`, `katoptra.org.directory.index.html`. Thus,
+one Transform Rule of the zone serves the pages of their hosts:
 
 ```text
 when:         ends_with(http.request.uri.path, "/")
-rewrite path: concat(http.request.uri.path, http.host, ".directory.index.html")
+rewrite path: concat(http.request.uri.path, "katoptra.org.directory.index.html")
 ```
 
-This rule also serves the page of the root for `/`. In ctan, the rule does not include `/`,
-because `/` serves the `index.html` of CTAN.
+The rule does not use `http.host`. Thus, two hosts of one bucket get the same key for each
+URL. This rule also serves the page of the root for `/`. In ctan, that page is a copy of
+the `index.html` of CTAN.
 
 #### Content types
 
@@ -539,14 +551,17 @@ CLI sets no type:
 4. A name with no type gets its type from the bytes. A known signature gives its type. If
    the bytes have no NUL, the type is `text/plain`. If they have a NUL, the type is
    `application/octet-stream`.
+5. A `text/*` type gets `; charset=utf-8` if the full file is valid UTF-8. ASCII is valid
+   UTF-8. The bytes of other text do not tell their encoding. Thus, other text gets no
+   charset.
 
 A control byte is not a NUL. Thus, a `.sty` file of CTAN with a `^Z` at its end stays
 `text/x-tex`. Three checks make sure that `label` obeys these rules:
 
 - `offline` gives a set of names and bytes to `label`, and stops on each change. If an
   update of the image changes `mime.types`, this check shows it.
-- `smoke` reads one key of each type back from the domain, and compares the served type
-  with the label.
+- `smoke` reads one key of each type back from the domain, and compares the full served
+  type, with the charset, with the label.
 - The `.tar.gz` check makes sure that tarballs have no `Content-Encoding`.
 
 #### The vars
@@ -566,6 +581,7 @@ inline default, and a mirror sets only the vars that are different:
 | `FILTER` | Empty | rsync filter arguments that make the listing smaller, for a mirror of a subtree |
 | `OWN` | Empty | Keys at the root of the bucket that the mirror writes, not upstream. A space is between two keys. `reconcile` does not delete them |
 | `INDEX` | Empty | The key suffix of the directory pages. If it has a value, `index` writes the pages. `reconcile` does not delete the pages, or a key with the name of a directory of the state |
+| `INDEX_FILES` | Empty | The names of the index files, in sequence. A space is between two names. The page key of a directory with such a file is a copy of the first one. Refer to [Directory pages](#directory-pages) |
 | `PAGE_FOOT` | Empty | The HTML at the end of each directory page. `%s` is the encoded path of the directory, and `%%` gives one `%` |
 | `CANARY` | Empty | A file that `smoke` reads from the domain as a Perl client, and compares with the bucket, byte for byte. All files can be the canary. A file with `http://` links (not `https://`) or `mailto:` addresses also finds the HTML rewriters of the zone. The path must have no character that a URL encodes |
 | `FRESH_KEY` | Empty | A file in which upstream records its clock, as Unix time or as `YYYY-MM-DD-HH-MM` (the format of CTAN). If it has a value, `fresh` stops the run when the clock is more than `FRESH_HOURS` before the time of the run |
@@ -884,7 +900,7 @@ engine.
 |---|---|---|
 | rsync | Each path of upstream, at the root | The mirror. A public domain serves the bucket |
 | rsync | `.state/applied.txt.xz` | The state: the keys that the bucket contains, with the length and the mtime of upstream |
-| rsync, with `INDEX` | `.state/indexed.txt.xz`, `<dir>/<INDEX>`, `<dir>` | The state that the directory pages showed last, and the pages with their two keys |
+| rsync, with `INDEX` | `.state/indexed.txt.xz`, `<dir>/<INDEX>`, `<dir>` | The state that the directory pages showed last, and the pages with their two keys. With `INDEX_FILES`, `<dir>/<INDEX>` can be a copy of an index file |
 | rsync, a key of the mirror | `index.html` | The landing page of tlnet. `reconcile` does not delete it, because it is in `OWN` |
 | proton | `.state/session.tar.age` | The CLI session, encrypted. This is the only key |
 | A mirror with its pipeline | `.state/state.sqlite.xz.age`, `.state/history/<epoch>-<label>...` | The state of dropbox and its copies with a date. A lifecycle rule of the bucket deletes the history after a time |
@@ -1182,8 +1198,9 @@ The other public mirrors have these structures:
   of `includes:`. It writes an `index` that uploads its landing page, and a `report-mirror`
   with its row.
 - A mirror with directories that a browser can show, for example ctan, gnu and nongnu, adds
-  `INDEX` and `PAGE_FOOT` to its root vars. It adds the Transform Rule of
-  [Directory pages](#directory-pages) to its zone. The engine does all the other work.
+  `INDEX` and `PAGE_FOOT` to its root vars. If its upstream serves index files, it also
+  adds `INDEX_FILES`. Its host must be in the host list of the Transform Rule of
+  [Directory pages](#directory-pages). The engine does all the other work.
 
 ### A proton mirror
 
diff --git a/engines/rsync.yml b/engines/rsync.yml
index 114a1bb..8850270 100644
--- a/engines/rsync.yml
+++ b/engines/rsync.yml
@@ -29,6 +29,10 @@
 #       none.
 #     - INDEX: the key suffix of the directory pages. When INDEX is set, index makes a
 #       page for each directory.
+#     - INDEX_FILES: the names of the upstream index files, in sequence, with a space
+#       between each two names. When a directory has such a file, its page key is a copy
+#       of the first one, and its key without the slash keeps the listing. An empty value
+#       gives a listing for each directory.
 #     - PAGE_FOOT: the HTML at the end of each page, with %s for the encoded path of the
 #       directory. An empty value gives none.
 #     - CANARY: a file with bytes that the HTML rewriters of the zone can change. When
@@ -126,6 +130,10 @@ vars:
       tolower($1) == "content-length:" { c = $2 }
       $2 == "416" { z = 1 }
       END { sub(/\r/, "", r); sub(/\r/, "", c); print (r != "" ? r : (z ? 0 : c)) }'
+  # The full Content-Type value from the headers, with its parameters (for example
+  # "text/plain; charset=utf-8").
+  CTYPE: |-
+    awk 'tolower($1) == "content-type:" { sub(/\r$/, ""); sub(/^[^:]*:[ \t]*/, ""); print }'
   # Percent-encoding for awk. enc(s) keeps the unreserved set and "/", and it encodes all
   # other characters. Upstream paths contain spaces and ( ) [ ] { } & # > ~ $. This text
   # goes into ENCODE, and into the awk program of the pages verb.
@@ -514,11 +522,14 @@ tasks:
       #   4. A name that has no type in mime.types gets a type from its bytes. This type
       #      is a known signature, text/plain without a NUL, or application/octet-stream
       #      with a NUL.
+      #   5. A text/* type gets "; charset=utf-8" when the full file is UTF-8 (ASCII is
+      #      UTF-8). A browser then shows its characters correctly. The bytes of other
+      #      text do not tell their encoding. Thus, other text gets no charset.
       # A control byte is not a NUL: some .sty files of CTAN contain ^Z, and they stay
       # text. Without "./", guess_type reads a name with a colon as a URL scheme.
       - |
         python3 -I - {{shellQuote .DIR}} {{shellQuote .COMPRESSED}} > {{.RUN}}/labels.raw <<'PY'
-        import mimetypes, os, sys
+        import codecs, mimetypes, os, sys
 
         root, table = sys.argv[1], sys.argv[2]
         endings = {s: f[0] for f in map(str.split, table.splitlines()) if f for s in f[1:]}
@@ -550,13 +561,28 @@ tasks:
                 return t if agrees else seen
             return t
 
+        # One MiB at a time: some text files are large (FILES.byname of CTAN is 27 MB).
+        def utf8(path):
+            d = codecs.getincrementaldecoder("utf-8")()
+            try:
+                with open(path, "rb") as h:
+                    for part in iter(lambda: h.read(1 << 20), b""):
+                        d.decode(part)
+                d.decode(b"", final=True)
+            except UnicodeDecodeError:
+                return False
+            return True
+
         out = sys.stdout.buffer
         for d, _, files in os.walk(root):
             for f in files:
                 p = os.path.join(d, f)
                 with open(p, "rb") as h:
                     head = h.read(1024)
-                out.write(os.fsencode(os.path.relpath(p, root)) + b"\t" + label(f, head).encode() + b"\n")
+                t = label(f, head)
+                if t.startswith("text/") and utf8(p):
+                    t += "; charset=utf-8"
+                out.write(os.fsencode(os.path.relpath(p, root)) + b"\t" + t.encode() + b"\n")
         PY
       - LC_ALL=C sort {{.RUN}}/labels.raw > {{.RUN}}/labels.txt && rm {{.RUN}}/labels.raw
 
@@ -711,14 +737,24 @@ tasks:
       # pages stay for smoke. The split verb of the next run removes them. The verb does
       # not append them to publish.txt, because smoke compares the size of that sample
       # with upstream, which has no pages.
-      - test ! -s {{.RUN}}/index-make.txt || aws s3 cp {{.AWS_FLAGS}} --recursive --content-type text/html {{.STAGING}}/ {{.S3}}/
+      - test ! -s {{.RUN}}/index-make.txt || aws s3 cp {{.AWS_FLAGS}} --recursive --content-type 'text/html; charset=utf-8' {{.STAGING}}/ {{.S3}}/
       # One upload for each depth, because no tree can hold a/b and a/b/ together. A
       # slashless key has no suffix that the CLI can use. Thus, the command gives the
       # type. Without it, the CLI sends binary/octet-stream, and a browser downloads the
       # page and does not show it.
       - >-
         for d in {{.SLASH}}/*/; do test -d "$d" || continue;
-        aws s3 cp {{.AWS_FLAGS}} --recursive --content-type text/html "$d" {{.S3}}/; done
+        aws s3 cp {{.AWS_FLAGS}} --recursive --content-type 'text/html; charset=utf-8' "$d" {{.S3}}/; done
+      # The page key of a directory with an index file (INDEX_FILES) is one server-side
+      # copy of that file. COPY keeps its Content-Type, and the copy has the same bytes
+      # and ETag. Thus, the URL of the directory serves the upstream file, as other
+      # mirrors do. The source is in the bucket, because pages read it from the state that
+      # checkpoint put there. The copy source is a URL path. s3api does not accept
+      # --no-progress. Thus, this command does not use AWS_FLAGS.
+      - >-
+        while IFS="$(printf '\t')" read -r s k; do u=$(printf '%s\n' "$s" | {{.ENCODE}} | cut -f2);
+        aws s3api copy-object --bucket {{.BUCKET}} --copy-source "{{.BUCKET}}/$u" --key "$k" --metadata-directive COPY > /dev/null;
+        echo "copy: $s to $k"; done < {{.RUN}}/index-real.txt
       - {task: remove, vars: {K: '{{.RUN}}/index-gone.txt'}}
       # At this point, the pages show the state, and the bucket holds that state, because
       # checkpoint put it there before this verb started. One server-side copy makes
@@ -727,19 +763,22 @@ tasks:
       - aws s3 cp {{.AWS_FLAGS}} {{.S3}}/{{.STATE}} {{.S3}}/{{.INDEXED}}
 
   pages:
-    desc: RUN/indexed.txt against RUN/applied.txt -> a page under both of its keys for every directory a changed line touches, RUN/index-gone.txt for the keys of directories that emptied
+    desc: RUN/indexed.txt against RUN/applied.txt -> a page under both of its keys for every directory a changed line touches, RUN/index-gone.txt for the keys of directories that emptied, RUN/index-real.txt for the directories with an index file
     set: [pipefail]
     vars:
       # The verb writes each page to two keys:
       #   - <dir>/INDEX. For a directory URL, the rewrite of the zone sends this key.
       #   - <dir> without its slash at the end. This key is for the same URL without the
       #     slash. For that URL, all other mirrors send 301.
+      # A directory with a file in INDEX_FILES gets only the second key. RUN/index-real.txt
+      # then has the line "<dir><file> TAB <dir>INDEX", and index copies that file to its
+      # page key.
       # No filesystem can hold a/b and a/b/ together. Thus, the second copies go into one
       # tree for each directory depth, and the upload sends one tree at a time. SLASH is a
       # task var, because a global var cannot follow a RUN that the caller gives.
       SLASH: '{{.RUN}}/slash'
     cmds:
-      - mkdir -p {{.STAGING}} {{.SLASH}} && find {{.STAGING}} {{.SLASH}} -mindepth 1 -delete
+      - 'mkdir -p {{.STAGING}} {{.SLASH}} && find {{.STAGING}} {{.SLASH}} -mindepth 1 -delete && : > {{.RUN}}/index-real.txt'
       # A page changes when a file in its directory, or a directory below it, is added,
       # changed or removed. Thus, the verb makes the page of each ancestor of each changed
       # line again.
@@ -787,26 +826,32 @@ tasks:
       # and for /dir. Without it, a relative href in the second resolves against the
       # parent. PAGE_FOOT is at the end of each page, with %s for the encoded path of the
       # directory. When PAGE_FOOT is empty, the table is the last part of the page.
+      #
+      # rank gives the position of each name in INDEX_FILES. real is the index file of the
+      # open directory with the lowest rank. Without a real, shut writes the page key.
       - >-
         test ! -s {{.RUN}}/index-make.txt ||
         awk -F'\t' -v OFS='\t' 'NR == FNR { want[$0] = 1; next }
           { n = split($1, c, "/"); d = ""
             for (i = 1; i < n; i++) { if ((d in want) && !((d c[i]) in seen)) { seen[d c[i]] = 1; print d, "d", c[i], "", "" }; d = d c[i] "/" }
             if (d in want) print d, "f", c[n], $2, $3 }' {{.RUN}}/index-make.txt {{.RUN}}/applied.txt \
-        | LC_ALL=C sort | awk -F'\t' -v S="{{.STAGING}}/" -v R="{{.SLASH}}/" -v I="{{.INDEX}}" -v F={{shellQuote (.PAGE_FOOT | default "")}} '{{.URLENC}}
+        | LC_ALL=C sort | awk -F'\t' -v S="{{.STAGING}}/" -v R="{{.SLASH}}/" -v I="{{.INDEX}}" -v F={{shellQuote (.PAGE_FOOT | default "")}} \
+          -v N={{shellQuote (.INDEX_FILES | default "")}} -v X="{{.RUN}}/index-real.txt" '{{.URLENC}}
+          BEGIN { n = split(N, nm, " "); for (i = n; i >= 1; i--) rank[nm[i]] = i }
           function esc(s) { gsub(/&/, "\\&amp;", s); gsub(/</, "\\&lt;", s); gsub(/>/, "\\&gt;", s); gsub(/"/, "\\&quot;", s); return s }
-          function open(d) { cur = d; live = 1
+          function open(d) { cur = d; live = 1; real = ""
             p = "<!DOCTYPE html><html><head><meta charset=\"utf-8\"><title>Index of /" esc(d) "</title>\n"
             p = p "<base href=\"/" enc(d) "\">\n"
             p = p "<style>body{font-family:monospace;margin:2em}td,th{text-align:left;padding:0 1.5em 0 0}</style></head><body>\n"
             p = p "<h1>Index of /" esc(d) "</h1><table><tr><th>Name</th><th>Last modified</th><th>Size</th></tr>\n"
             if (d != "") p = p "<tr><td><a href=\"../\">../</a></td><td></td><td></td></tr>\n" }
           function shut(  f, b, k, e) { p = p "</table>" (F == "" ? "" : "<p>" sprintf(F, enc(cur)) "</p>") "</body></html>\n"
-            f = S cur I; printf("%s", p) > f; close(f)
+            if (real == "") { f = S cur I; printf("%s", p) > f; close(f) } else print cur real "\t" cur I > X
             if (cur != "") { b = substr(cur, 1, length(cur) - 1); k = split(b, e, "/")
               f = R k "/" b; printf("%s", p) > f; close(f) } }
           !live || $1 != cur { if (live) shut(); open($1) }
           $2 == "d" { p = p "<tr><td><a href=\"" enc($3) "/\">" esc($3) "/</a></td><td></td><td></td></tr>\n"; next }
+          ($3 in rank) && (real == "" || rank[$3] < rank[real]) { real = $3 }
           { p = p "<tr><td><a href=\"" enc($3) "\">" esc($3) "</a></td><td>" $5 "</td><td>" $4 "</td></tr>\n" }
           END { if (live) shut() }'
 
@@ -842,7 +887,7 @@ tasks:
         want=$(awk -F'\t' -v k="$k" '$1 == k { print $2 }' {{.RUN}}/upstream.txt);
         h=$({{.CURL}} -r 0-0 -D - -o /dev/null "{{.URL}}/$u");
         got=$(printf '%s\n' "$h" | {{.SIZE}});
-        ct=$(printf '%s\n' "$h" | awk 'tolower($1) == "content-type:" { sub(/\r$/, ""); print $2 }');
+        ct=$(printf '%s\n' "$h" | {{.CTYPE}});
         echo "$k: $got bytes (listing: $want), served as ${ct:-no type} (label: ${t:-none})"; test "$got" = "$want" || exit 1;
         case "{{.URL}}" in http*) test -z "$t" || test "$ct" = "$t" || exit 1 ;; esac; done
       # The tlpdb key is in the decision batch, the last batch. Thus, a run that stopped
@@ -852,11 +897,14 @@ tasks:
         {{if .TL}}test ! -f {{.RUN}}/tl/tlpkg/texlive.tlpdb.sha512 ||
         test -f {{.RUN}}/chain ||
         {{.CURL}} {{.URL}}/{{.TL}}/tlpkg/texlive.tlpdb.sha512 | cmp - {{.RUN}}/tl/tlpkg/texlive.tlpdb.sha512{{else}}echo "no TL subtree: no tlpdb to read back"{{end}}
-      # This command reads one of the pages that this run made. It reads that page at its
-      # two keys, with each key in the URL (no zone rule is necessary). It compares the
-      # size with the copy in staging, with the same ranged read of one byte as the
-      # sample. The two keys hold the same bytes. Thus, one copy in staging gives the size
-      # for the two. The root has no slashless key.
+      # This command reads one of the pages that this run made, at its two keys, with each
+      # key in the URL (no zone rule is necessary). r gives the size and the type of a
+      # key, with the same ranged read of one byte as the sample.
+      #   - The page key: for a directory in index-real.txt, the size and the type must be
+      #     those of its index file. For other directories, the size must be that of the
+      #     page in staging.
+      #   - The slashless key: the size must be that of the listing in the SLASH tree. The
+      #     root has no slashless key.
       #
       # ${d} is in braces, because the INDEX after it starts with a letter. $d{{.INDEX}}
       # is the name of a variable that is not there. test -n then stops the command, and
@@ -865,12 +913,17 @@ tasks:
       # together.
       - >-
         {{if .INDEX}}test ! -s {{.RUN}}/index-make.txt || { d=$(tail -1 {{.RUN}}/index-make.txt);
-        want=$(wc -c < "{{.STAGING}}/${d}{{.INDEX}}" | tr -d ' '); test -n "$want";
-        case "{{.URL}}" in http*) b=1 ;; *) b=0 ;; esac;
-        printf '%s\n' "$d" | awk -v i="{{.INDEX}}" -v b="$b" '{ print $0 i } b == 1 && $0 != "" { print substr($0, 1, length($0) - 1) }'
-        | {{.ENCODE}} | while IFS="$(printf '\t')" read -r k u; do
-        got=$({{.CURL}} -r 0-0 -D - -o /dev/null "{{.URL}}/$u" | {{.SIZE}});
-        echo "$k: $got bytes (page: $want)"; test "$got" = "$want" || exit 1; done; }{{else}}echo "no INDEX: no directory pages to read back"{{end}}
+        r() { h=$({{.CURL}} -r 0-0 -D - -o /dev/null "{{.URL}}/$(printf '%s\n' "$1" | {{.ENCODE}} | cut -f2)");
+        echo "$(printf '%s\n' "$h" | {{.SIZE}}) bytes, type $(printf '%s\n' "$h" | {{.CTYPE}})"; };
+        s=$(awk -F'\t' -v k="${d}{{.INDEX}}" '$2 == k { print $1 }' {{.RUN}}/index-real.txt);
+        if test -n "$s"; then want=$(r "$s");
+        else want=$(wc -c < "{{.STAGING}}/${d}{{.INDEX}}" | tr -d ' '); test -n "$want"; want="$want bytes"; fi;
+        got=$(r "${d}{{.INDEX}}"); test -n "$s" || got=${got%%,*};
+        echo "${d}{{.INDEX}}: $got (${s:-page}: $want)"; test "$got" = "$want";
+        case "{{.URL}}" in http*) test -z "$d" || { p=${d%/};
+        want=$(wc -c < "{{.RUN}}/slash/$(printf '%s\n' "$p" | awk -F/ '{ print NF }')/$p" | tr -d ' '); test -n "$want";
+        got=$(r "$p"); got=${got%%,*}; echo "$p: $got (listing: $want bytes)"; test "$got" = "$want bytes"; } ;; esac;
+        }{{else}}echo "no INDEX: no directory pages to read back"{{end}}
       # The HTML rewriters of Cloudflare are between R2 and the client. Thus, the check
       # compares the object with the file that the domain sends. A size check is not
       # sufficient. Automatic HTTPS Rewrites can make a link one byte longer, while the
diff --git a/examples/rsync/Taskfile.yml b/examples/rsync/Taskfile.yml
index 98fa825..e4b2288 100644
--- a/examples/rsync/Taskfile.yml
+++ b/examples/rsync/Taskfile.yml
@@ -13,6 +13,7 @@ vars:
   HOST: example.katoptra.org
   LIST_FLOOR: 1
   INDEX: '{{.HOST}}.directory.index.html'
+  INDEX_FILES: index.html index.htm
   PAGE_FOOT: 'A mirror of rsync.example.org/pub/. <a href="https://example.org/issues">Report a problem</a>.'
   CANARY: pub/index.html
   FRESH_KEY: timestamp
@@ -140,11 +141,18 @@ tasks:
       # run-pages: the new state has a file with & < > in its name, and a directory with a
       # space in its name. gone/ is not in the new state. pages makes the pages of the
       # root, a/, a/b/ and a/sp ace/, each with its two keys (the root with one). It also
-      # gives the two keys of gone/ for removal. Each byte must agree with want/.
+      # gives the two keys of gone/ for removal. a/doc/ has index.htm and index.html. Thus,
+      # its listing goes only to its key without the slash, and index-real.txt gives
+      # index.html, the first name in INDEX_FILES, as the source of its page key. Each byte
+      # must agree with want/. staging keeps the empty directory a/doc/, which git cannot
+      # keep in want/. The upload sends no empty directory.
       - {task: pages, vars: {RUN: '{{.F}}/run-pages', STAGING: '{{.F}}/run-pages/staging', INDEX: '{{.CTAN_INDEX}}', PAGE_FOOT: '{{.CTAN_FOOT}}'}}
-      - diff -r {{.F}}/run-pages/want/staging {{.F}}/run-pages/staging
+      - find {{.F}}/run-pages/staging -type d -empty -delete && diff -r {{.F}}/run-pages/want/staging {{.F}}/run-pages/staging
       - diff -r {{.F}}/run-pages/want/slash {{.F}}/run-pages/slash
       - cmp {{.F}}/run-pages/want/index-make.txt {{.F}}/run-pages/index-make.txt && cmp {{.F}}/run-pages/want/index-gone.txt {{.F}}/run-pages/index-gone.txt
+      - cmp {{.F}}/run-pages/want/index-real.txt {{.F}}/run-pages/index-real.txt
+      # CTYPE gives the full value of the header, with its charset.
+      - "test \"$(printf 'HTTP/2 200\\r\\nContent-Type: text/plain; charset=utf-8\\r\\n' | {{.CTYPE}})\" = 'text/plain; charset=utf-8'"
       # run-smoke: an hour with one change only, timestamp, and one new page only, the
       # root. smoke reads timestamp again, and it compares its size with the listing. Then
       # it reads the root page at its INDEX key (a filesystem cannot hold the slashless
@@ -158,6 +166,15 @@ tasks:
       - {task: smoke, vars: {RUN: '{{.F}}/run-smoke', STAGING: '{{.F}}/run-smoke/staging', URL: 'file://{{.F}}/run-smoke/staging'}}
       # smoke rejects a page that index made and that is not in staging.
       - '! task smoke-refuses 2>/dev/null'
+      # run-copy: the one new page is doc/, a directory with an index file. Its page key is
+      # a copy of doc/index.html. smoke reads the two keys and compares their sizes and
+      # types. smoke-refuses-copy makes the copy one byte longer.
+      - |
+        r={{.F}}/run-copy; mkdir -p $r/staging/doc; printf '<p>doc</p>\n' > $r/staging/doc/index.html
+        cp $r/staging/doc/index.html "$r/staging/doc/{{.INDEX}}"; printf 'doc/\n' > $r/index-make.txt
+        printf 'doc/index.html\tdoc/{{.INDEX}}\n' > $r/index-real.txt; : > $r/applied.txt
+      - {task: smoke, vars: {RUN: '{{.F}}/run-copy', STAGING: '{{.F}}/run-copy/staging', URL: 'file://{{.F}}/run-copy/staging'}}
+      - '! task smoke-refuses-copy 2>/dev/null'
       # run-label: the command makes it here, because this test is about its names and its
       # bytes. It has a test case for each rule of label:
       #   - A COMPRESSED ending is more important than the remaining part of the name
@@ -170,6 +187,9 @@ tasks:
       #   - An ending that Python reads as compressed and that COMPRESSED does not have is
       #     not text (q.txt.br).
       #   - A colon is not a URL scheme (data:p.html).
+      #   - A text type gets "; charset=utf-8" only when the full file is UTF-8: ASCII
+      #     (README), non-ASCII UTF-8 (r.txt) and the empty k. Latin-1 (s.txt) and UTF-16
+      #     (u.txt) get no charset.
       # label-trees then moves each file other than timestamp out, one tree for each type,
       # and then back. After that, staging has the same bytes that it had at the start.
       - |
@@ -180,18 +200,21 @@ tasks:
         printf '\211\001\000' > $d/g.sig; printf 'PK\003\004' > $d/h.xpi; printf '<p></p>\n' > $d/sub/i.html
         printf 'x\032y\n' > $d/j.sty; : > $d/k; printf '\037\235' > $d/l.Z; printf 'x\000' > $d/m
         printf 'BZh9' > $d/n; printf '<p></p>\n' > "$d/data:p.html"; printf 'abc' > $d/q.txt.br
+        printf 'caf\303\251\n' > $d/r.txt; printf 'caf\351\n' > $d/s.txt; printf '\377\376a\000' > $d/u.txt
         printf '2026-10-08-05-02\n' > $d/timestamp
         cp -R $d {{.F}}/run-label/before
       - {task: label, vars: {RUN: '{{.F}}/run-label', DIR: '{{.F}}/run-label/staging'}}
       - |
-        printf '%s\t%s\n' README text/plain a.diff.gz application/x-gzip b.tar.xz application/x-xz \
-          c.tgz application/x-gzip d.txt application/octet-stream data:p.html text/html e.pdf application/pdf \
+        u='; charset=utf-8'
+        printf '%s\t%s\n' README "text/plain$u" a.diff.gz application/x-gzip b.tar.xz application/x-xz \
+          c.tgz application/x-gzip d.txt application/octet-stream data:p.html "text/html$u" e.pdf application/pdf \
           f.pdf application/x-gzip g.sig application/pgp-signature h.xpi application/x-xpinstall \
-          j.sty text/x-tex k text/plain l.Z application/x-compress m application/octet-stream \
-          n application/x-bzip2 q.txt.br application/octet-stream sub/i.html text/html timestamp text/plain \
+          j.sty "text/x-tex$u" k "text/plain$u" l.Z application/x-compress m application/octet-stream \
+          n application/x-bzip2 q.txt.br application/octet-stream r.txt "text/plain$u" s.txt text/plain \
+          sub/i.html "text/html$u" timestamp "text/plain$u" u.txt text/plain \
         | diff - {{.F}}/run-label/labels.txt
       - {task: label-trees, vars: {TO: up, RUN: '{{.F}}/run-label', STAGING: '{{.F}}/run-label/staging'}}
-      - test "$(cd {{.F}}/run-label/staging && find . -type f)" = ./timestamp && test "$(ls {{.F}}/run-label/up/*.type | wc -l)" -eq 11
+      - test "$(cd {{.F}}/run-label/staging && find . -type f)" = ./timestamp && test "$(ls {{.F}}/run-label/up/*.type | wc -l)" -eq 12
       - {task: label-trees, vars: {TO: staging, RUN: '{{.F}}/run-label', STAGING: '{{.F}}/run-label/staging'}}
       - diff -r {{.F}}/run-label/before {{.F}}/run-label/staging && test ! -e {{.F}}/run-label/up
       # fresh on three clocks, from a filesystem:
@@ -224,6 +247,13 @@ tasks:
     cmds:
       - rm -f {{.R}}/staging/{{.INDEX}}
       - {task: smoke, vars: {RUN: '{{.R}}', STAGING: '{{.R}}/staging', URL: 'file://{{.R}}/staging'}}
+  smoke-refuses-copy:
+    desc: smoke over run-copy with the page key one byte longer than its source, after offline made run-copy; offline expects it to fail
+    vars:
+      R: '{{.RUN}}/fixtures/run-copy'
+    cmds:
+      - echo >> "{{.R}}/staging/doc/{{.INDEX}}"
+      - {task: smoke, vars: {RUN: '{{.R}}', STAGING: '{{.R}}/staging', URL: 'file://{{.R}}/staging'}}
   verify-refuses:
     desc: verify over a batch naming a container the tlpdb does not describe, after offline staged run-tl; offline expects it to fail
     vars:
diff --git a/examples/rsync/fixtures/run-pages/applied.txt b/examples/rsync/fixtures/run-pages/applied.txt
index 49a691d..8a38edf 100644
--- a/examples/rsync/fixtures/run-pages/applied.txt
+++ b/examples/rsync/fixtures/run-pages/applied.txt
@@ -1,4 +1,6 @@
 a/b/f.txt	10	2026/08/27 01:00:00
 a/b/new & <odd>.txt	7	2026/08/28 06:00:00
+a/doc/index.htm	4	2026/08/28 06:00:00
+a/doc/index.html	5	2026/08/28 06:00:00
 a/sp ace/g.tar.gz	3	2026/08/28 06:00:00
 timestamp	20	2026/08/28 06:00:00
diff --git a/examples/rsync/fixtures/run-pages/want/index-make.txt b/examples/rsync/fixtures/run-pages/want/index-make.txt
index fd09e78..dc77e4a 100644
--- a/examples/rsync/fixtures/run-pages/want/index-make.txt
+++ b/examples/rsync/fixtures/run-pages/want/index-make.txt
@@ -1,4 +1,5 @@
 
 a/
 a/b/
+a/doc/
 a/sp ace/
diff --git a/examples/rsync/fixtures/run-pages/want/index-real.txt b/examples/rsync/fixtures/run-pages/want/index-real.txt
new file mode 100644
index 0000000..47a3f6d
--- /dev/null
+++ b/examples/rsync/fixtures/run-pages/want/index-real.txt
@@ -0,0 +1 @@
+a/doc/index.html	a/doc/ctan.katoptra.org.directory.index.html
diff --git a/examples/rsync/fixtures/run-pages/want/slash/1/a b/examples/rsync/fixtures/run-pages/want/slash/1/a
index 1dac6ad..640dd06 100644
--- a/examples/rsync/fixtures/run-pages/want/slash/1/a
+++ b/examples/rsync/fixtures/run-pages/want/slash/1/a
@@ -4,5 +4,6 @@
 <h1>Index of /a/</h1><table><tr><th>Name</th><th>Last modified</th><th>Size</th></tr>
 <tr><td><a href="../">../</a></td><td></td><td></td></tr>
 <tr><td><a href="b/">b/</a></td><td></td><td></td></tr>
+<tr><td><a href="doc/">doc/</a></td><td></td><td></td></tr>
 <tr><td><a href="sp%20ace/">sp ace/</a></td><td></td><td></td></tr>
 </table><p><a href="https://ctan.org/tex-archive/a/">This directory on ctan.org</a>. Sizes in bytes, times as CTAN lists them.</p></body></html>
diff --git a/examples/rsync/fixtures/run-pages/want/slash/2/a/doc b/examples/rsync/fixtures/run-pages/want/slash/2/a/doc
new file mode 100644
index 0000000..1146f63
--- /dev/null
+++ b/examples/rsync/fixtures/run-pages/want/slash/2/a/doc
@@ -0,0 +1,8 @@
+<!DOCTYPE html><html><head><meta charset="utf-8"><title>Index of /a/doc/</title>
+<base href="/a/doc/">
+<style>body{font-family:monospace;margin:2em}td,th{text-align:left;padding:0 1.5em 0 0}</style></head><body>
+<h1>Index of /a/doc/</h1><table><tr><th>Name</th><th>Last modified</th><th>Size</th></tr>
+<tr><td><a href="../">../</a></td><td></td><td></td></tr>
+<tr><td><a href="index.htm">index.htm</a></td><td>2026/08/28 06:00:00</td><td>4</td></tr>
+<tr><td><a href="index.html">index.html</a></td><td>2026/08/28 06:00:00</td><td>5</td></tr>
+</table><p><a href="https://ctan.org/tex-archive/a/doc/">This directory on ctan.org</a>. Sizes in bytes, times as CTAN lists them.</p></body></html>
diff --git a/examples/rsync/fixtures/run-pages/want/staging/a/ctan.katoptra.org.directory.index.html b/examples/rsync/fixtures/run-pages/want/staging/a/ctan.katoptra.org.directory.index.html
index 1dac6ad..640dd06 100644
--- a/examples/rsync/fixtures/run-pages/want/staging/a/ctan.katoptra.org.directory.index.html
+++ b/examples/rsync/fixtures/run-pages/want/staging/a/ctan.katoptra.org.directory.index.html
@@ -4,5 +4,6 @@
 <h1>Index of /a/</h1><table><tr><th>Name</th><th>Last modified</th><th>Size</th></tr>
 <tr><td><a href="../">../</a></td><td></td><td></td></tr>
 <tr><td><a href="b/">b/</a></td><td></td><td></td></tr>
+<tr><td><a href="doc/">doc/</a></td><td></td><td></td></tr>
 <tr><td><a href="sp%20ace/">sp ace/</a></td><td></td><td></td></tr>
 </table><p><a href="https://ctan.org/tex-archive/a/">This directory on ctan.org</a>. Sizes in bytes, times as CTAN lists them.</p></body></html>
diff --git a/examples/rsync/fixtures/run-smoke/index-real.txt b/examples/rsync/fixtures/run-smoke/index-real.txt
new file mode 100644
index 0000000..e69de29
diff --git a/examples/rsync/render.txt b/examples/rsync/render.txt
index f26b153..1dca582 100644
--- a/examples/rsync/render.txt
+++ b/examples/rsync/render.txt
@@ -108,7 +108,7 @@ task: [verify] cd /work/examples/rsync/staging && { test ! -f /tlpkg/texlive.tlp
   /^srccontainerchecksum /{print TL "/archive/" n ".source.tar.xz"}' /tlpkg/texlive.tlpdb \
 | LC_ALL=C sort | LC_ALL=C comm -23 - /work/examples/rsync/.run/tl/after.txt > /work/examples/rsync/.run/tl/missing.txt && test ! -s /work/examples/rsync/.run/tl/missing.txt; }; }
 task: [label] python3 -I - /work/examples/rsync/staging $'application/x-gzip .gz .tgz .taz .tz .svgz\napplication/x-bzip2 .bz2 .tbz2 .tbz\napplication/x-xz .xz .txz\napplication/x-compress .Z\napplication/zstd .zst .tzst\napplication/x-lzip .lz\napplication/x-lzma .lzma' > /work/examples/rsync/.run/labels.raw <<'PY'
-import mimetypes, os, sys
+import codecs, mimetypes, os, sys
 
 root, table = sys.argv[1], sys.argv[2]
 endings = {s: f[0] for f in map(str.split, table.splitlines()) if f for s in f[1:]}
@@ -140,13 +140,28 @@ def label(name, head):
         return t if agrees else seen
     return t
 
+# One MiB at a time: some text files are large (FILES.byname of CTAN is 27 MB).
+def utf8(path):
+    d = codecs.getincrementaldecoder("utf-8")()
+    try:
+        with open(path, "rb") as h:
+            for part in iter(lambda: h.read(1 << 20), b""):
+                d.decode(part)
+        d.decode(b"", final=True)
+    except UnicodeDecodeError:
+        return False
+    return True
+
 out = sys.stdout.buffer
 for d, _, files in os.walk(root):
     for f in files:
         p = os.path.join(d, f)
         with open(p, "rb") as h:
             head = h.read(1024)
-        out.write(os.fsencode(os.path.relpath(p, root)) + b"\t" + label(f, head).encode() + b"\n")
+        t = label(f, head)
+        if t.startswith("text/") and utf8(p):
+            t += "; charset=utf-8"
+        out.write(os.fsencode(os.path.relpath(p, root)) + b"\t" + t.encode() + b"\n")
 PY
 
 task: [label] LC_ALL=C sort /work/examples/rsync/.run/labels.raw > /work/examples/rsync/.run/labels.txt && rm /work/examples/rsync/.run/labels.raw
@@ -262,7 +277,7 @@ case $rc in
   *) exit $rc ;;
 esac
 
-task: [pages] mkdir -p /work/examples/rsync/staging /work/examples/rsync/.run/slash && find /work/examples/rsync/staging /work/examples/rsync/.run/slash -mindepth 1 -delete
+task: [pages] mkdir -p /work/examples/rsync/staging /work/examples/rsync/.run/slash && find /work/examples/rsync/staging /work/examples/rsync/.run/slash -mindepth 1 -delete && : > /work/examples/rsync/.run/index-real.txt
 task: [pages] LC_ALL=C comm -3 /work/examples/rsync/.run/indexed.txt /work/examples/rsync/.run/applied.txt | awk -F'\t' '{ print ($1 == "" ? $2 : $1) }' | awk -F'\t' '!("" in s) { s[""] = 1; print "" }
   { n = split($1, c, "/"); d = ""; for (i = 1; i < n; i++) { d = d c[i] "/"; if (!(d in s)) { s[d] = 1; print d } } }' | LC_ALL=C sort > /work/examples/rsync/.run/index-dirty.txt
 task: [pages] awk -F'\t' '!("" in s) { s[""] = 1; print "" }
@@ -277,26 +292,32 @@ task: [pages] test ! -s /work/examples/rsync/.run/index-make.txt || awk -F'\t' -
   { n = split($1, c, "/"); d = ""
     for (i = 1; i < n; i++) { if ((d in want) && !((d c[i]) in seen)) { seen[d c[i]] = 1; print d, "d", c[i], "", "" }; d = d c[i] "/" }
     if (d in want) print d, "f", c[n], $2, $3 }' /work/examples/rsync/.run/index-make.txt /work/examples/rsync/.run/applied.txt \
-| LC_ALL=C sort | awk -F'\t' -v S="/work/examples/rsync/staging/" -v R="/work/examples/rsync/.run/slash/" -v I="example.katoptra.org.directory.index.html" -v F='A mirror of rsync.example.org/pub/. <a href="https://example.org/issues">Report a problem</a>.' 'BEGIN { for (i = 0; i < 256; i++) ord[sprintf("%c", i)] = i }
+| LC_ALL=C sort | awk -F'\t' -v S="/work/examples/rsync/staging/" -v R="/work/examples/rsync/.run/slash/" -v I="example.katoptra.org.directory.index.html" -v F='A mirror of rsync.example.org/pub/. <a href="https://example.org/issues">Report a problem</a>.' \
+  -v N='index.html index.htm' -v X="/work/examples/rsync/.run/index-real.txt" 'BEGIN { for (i = 0; i < 256; i++) ord[sprintf("%c", i)] = i }
 function enc(s,  out, j, c) { out = ""; for (j = 1; j <= length(s); j++) { c = substr(s, j, 1)
     out = out ((c ~ /[A-Za-z0-9\/._~-]/) ? c : sprintf("%%%02X", ord[c])) } return out }
+  BEGIN { n = split(N, nm, " "); for (i = n; i >= 1; i--) rank[nm[i]] = i }
   function esc(s) { gsub(/&/, "\\&amp;", s); gsub(/</, "\\&lt;", s); gsub(/>/, "\\&gt;", s); gsub(/"/, "\\&quot;", s); return s }
-  function open(d) { cur = d; live = 1
+  function open(d) { cur = d; live = 1; real = ""
     p = "<!DOCTYPE html><html><head><meta charset=\"utf-8\"><title>Index of /" esc(d) "</title>\n"
     p = p "<base href=\"/" enc(d) "\">\n"
     p = p "<style>body{font-family:monospace;margin:2em}td,th{text-align:left;padding:0 1.5em 0 0}</style></head><body>\n"
     p = p "<h1>Index of /" esc(d) "</h1><table><tr><th>Name</th><th>Last modified</th><th>Size</th></tr>\n"
     if (d != "") p = p "<tr><td><a href=\"../\">../</a></td><td></td><td></td></tr>\n" }
   function shut(  f, b, k, e) { p = p "</table>" (F == "" ? "" : "<p>" sprintf(F, enc(cur)) "</p>") "</body></html>\n"
-    f = S cur I; printf("%s", p) > f; close(f)
+    if (real == "") { f = S cur I; printf("%s", p) > f; close(f) } else print cur real "\t" cur I > X
     if (cur != "") { b = substr(cur, 1, length(cur) - 1); k = split(b, e, "/")
       f = R k "/" b; printf("%s", p) > f; close(f) } }
   !live || $1 != cur { if (live) shut(); open($1) }
   $2 == "d" { p = p "<tr><td><a href=\"" enc($3) "/\">" esc($3) "/</a></td><td></td><td></td></tr>\n"; next }
+  ($3 in rank) && (real == "" || rank[$3] < rank[real]) { real = $3 }
   { p = p "<tr><td><a href=\"" enc($3) "\">" esc($3) "</a></td><td>" $5 "</td><td>" $4 "</td></tr>\n" }
   END { if (live) shut() }'
-task: [index] test ! -s /work/examples/rsync/.run/index-make.txt || aws s3 cp --no-progress --cli-connect-timeout 60 --cli-read-timeout 300 --recursive --content-type text/html /work/examples/rsync/staging/ s3://example/
-task: [index] for d in /work/examples/rsync/.run/slash/*/; do test -d "$d" || continue; aws s3 cp --no-progress --cli-connect-timeout 60 --cli-read-timeout 300 --recursive --content-type text/html "$d" s3://example/; done
+task: [index] test ! -s /work/examples/rsync/.run/index-make.txt || aws s3 cp --no-progress --cli-connect-timeout 60 --cli-read-timeout 300 --recursive --content-type 'text/html; charset=utf-8' /work/examples/rsync/staging/ s3://example/
+task: [index] for d in /work/examples/rsync/.run/slash/*/; do test -d "$d" || continue; aws s3 cp --no-progress --cli-connect-timeout 60 --cli-read-timeout 300 --recursive --content-type 'text/html; charset=utf-8' "$d" s3://example/; done
+task: [index] while IFS="$(printf '\t')" read -r s k; do u=$(printf '%s\n' "$s" | awk 'BEGIN { for (i = 0; i < 256; i++) ord[sprintf("%c", i)] = i }
+function enc(s,  out, j, c) { out = ""; for (j = 1; j <= length(s); j++) { c = substr(s, j, 1)
+    out = out ((c ~ /[A-Za-z0-9\/._~-]/) ? c : sprintf("%%%02X", ord[c])) } return out } { print $0 "\t" enc($0) }' | cut -f2); aws s3api copy-object --bucket example --copy-source "example/$u" --key "$k" --metadata-directive COPY > /dev/null; echo "copy: $s to $k"; done < /work/examples/rsync/.run/index-real.txt
 task: [remove] rm -f /work/examples/rsync/.run/del-*; split -l 1000 -a 4 /work/examples/rsync/.run/index-gone.txt /work/examples/rsync/.run/del-
 task: [remove] for f in /work/examples/rsync/.run/del-*; do awk 'BEGIN { printf "{\"Objects\":[" } { gsub(/\\/, "\\\\"); gsub(/"/, "\\\"")
      printf "%s{\"Key\":\"%s\"}", (NR > 1 ? "," : ""), $0 } END { print "],\"Quiet\":true}" }' "$f" \
@@ -307,14 +328,14 @@ function enc(s,  out, j, c) { out = ""; for (j = 1; j <= length(s); j++) { c = s
     out = out ((c ~ /[A-Za-z0-9\/._~-]/) ? c : sprintf("%%%02X", ord[c])) } return out } { print $0 "\t" enc($0) }' | cut -f2); want=$(awk -F'\t' -v k="$k" '$1 == k { print $2 }' /work/examples/rsync/.run/upstream.txt); h=$(curl -fsS --connect-timeout 15 --max-time 60 --retry 6 --retry-connrefused --retry-max-time 600 -r 0-0 -D - -o /dev/null "https://example.katoptra.org/$u"); got=$(printf '%s\n' "$h" | awk 'tolower($1) == "content-range:" { split($3, a, "/"); r = a[2] }
   tolower($1) == "content-length:" { c = $2 }
   $2 == "416" { z = 1 }
-  END { sub(/\r/, "", r); sub(/\r/, "", c); print (r != "" ? r : (z ? 0 : c)) }'); ct=$(printf '%s\n' "$h" | awk 'tolower($1) == "content-type:" { sub(/\r$/, ""); print $2 }'); echo "$k: $got bytes (listing: $want), served as ${ct:-no type} (label: ${t:-none})"; test "$got" = "$want" || exit 1; case "https://example.katoptra.org" in http*) test -z "$t" || test "$ct" = "$t" || exit 1 ;; esac; done
+  END { sub(/\r/, "", r); sub(/\r/, "", c); print (r != "" ? r : (z ? 0 : c)) }'); ct=$(printf '%s\n' "$h" | awk 'tolower($1) == "content-type:" { sub(/\r$/, ""); sub(/^[^:]*:[ \t]*/, ""); print }'); echo "$k: $got bytes (listing: $want), served as ${ct:-no type} (label: ${t:-none})"; test "$got" = "$want" || exit 1; case "https://example.katoptra.org" in http*) test -z "$t" || test "$ct" = "$t" || exit 1 ;; esac; done
 task: [smoke] echo "no TL subtree: no tlpdb to read back"
-task: [smoke] test ! -s /work/examples/rsync/.run/index-make.txt || { d=$(tail -1 /work/examples/rsync/.run/index-make.txt); want=$(wc -c < "/work/examples/rsync/staging/${d}example.katoptra.org.directory.index.html" | tr -d ' '); test -n "$want"; case "https://example.katoptra.org" in http*) b=1 ;; *) b=0 ;; esac; printf '%s\n' "$d" | awk -v i="example.katoptra.org.directory.index.html" -v b="$b" '{ print $0 i } b == 1 && $0 != "" { print substr($0, 1, length($0) - 1) }' | awk 'BEGIN { for (i = 0; i < 256; i++) ord[sprintf("%c", i)] = i }
+task: [smoke] test ! -s /work/examples/rsync/.run/index-make.txt || { d=$(tail -1 /work/examples/rsync/.run/index-make.txt); r() { h=$(curl -fsS --connect-timeout 15 --max-time 60 --retry 6 --retry-connrefused --retry-max-time 600 -r 0-0 -D - -o /dev/null "https://example.katoptra.org/$(printf '%s\n' "$1" | awk 'BEGIN { for (i = 0; i < 256; i++) ord[sprintf("%c", i)] = i }
 function enc(s,  out, j, c) { out = ""; for (j = 1; j <= length(s); j++) { c = substr(s, j, 1)
-    out = out ((c ~ /[A-Za-z0-9\/._~-]/) ? c : sprintf("%%%02X", ord[c])) } return out } { print $0 "\t" enc($0) }' | while IFS="$(printf '\t')" read -r k u; do got=$(curl -fsS --connect-timeout 15 --max-time 60 --retry 6 --retry-connrefused --retry-max-time 600 -r 0-0 -D - -o /dev/null "https://example.katoptra.org/$u" | awk 'tolower($1) == "content-range:" { split($3, a, "/"); r = a[2] }
+    out = out ((c ~ /[A-Za-z0-9\/._~-]/) ? c : sprintf("%%%02X", ord[c])) } return out } { print $0 "\t" enc($0) }' | cut -f2)"); echo "$(printf '%s\n' "$h" | awk 'tolower($1) == "content-range:" { split($3, a, "/"); r = a[2] }
   tolower($1) == "content-length:" { c = $2 }
   $2 == "416" { z = 1 }
-  END { sub(/\r/, "", r); sub(/\r/, "", c); print (r != "" ? r : (z ? 0 : c)) }'); echo "$k: $got bytes (page: $want)"; test "$got" = "$want" || exit 1; done; }
+  END { sub(/\r/, "", r); sub(/\r/, "", c); print (r != "" ? r : (z ? 0 : c)) }') bytes, type $(printf '%s\n' "$h" | awk 'tolower($1) == "content-type:" { sub(/\r$/, ""); sub(/^[^:]*:[ \t]*/, ""); print }')"; }; s=$(awk -F'\t' -v k="${d}example.katoptra.org.directory.index.html" '$2 == k { print $1 }' /work/examples/rsync/.run/index-real.txt); if test -n "$s"; then want=$(r "$s"); else want=$(wc -c < "/work/examples/rsync/staging/${d}example.katoptra.org.directory.index.html" | tr -d ' '); test -n "$want"; want="$want bytes"; fi; got=$(r "${d}example.katoptra.org.directory.index.html"); test -n "$s" || got=${got%%,*}; echo "${d}example.katoptra.org.directory.index.html: $got (${s:-page}: $want)"; test "$got" = "$want"; case "https://example.katoptra.org" in http*) test -z "$d" || { p=${d%/}; want=$(wc -c < "/work/examples/rsync/.run/slash/$(printf '%s\n' "$p" | awk -F/ '{ print NF }')/$p" | tr -d ' '); test -n "$want"; got=$(r "$p"); got=${got%%,*}; echo "$p: $got (listing: $want bytes)"; test "$got" = "$want bytes"; } ;; esac; }
 task: [smoke] h=pub/index.html; n=$(awk -F'\t' -v k="$h" '$1 == k { print $2 }' /work/examples/rsync/.run/applied.txt 2>/dev/null); case "https://example.katoptra.org" in http*) ;; *) n= ;; esac; if test -z "$n"; then echo "canary skipped ($h is not in the state, or the URL is not http)"; else aws s3 cp --no-progress --cli-connect-timeout 60 --cli-read-timeout 300 "s3://example/$h" /work/examples/rsync/.run/canary.html && curl -fsS --connect-timeout 15 --max-time 60 --retry 6 --retry-connrefused --retry-max-time 600 -A "libwww-perl/6.67" "https://example.katoptra.org/$h" | cmp - /work/examples/rsync/.run/canary.html && echo "$h: $n bytes, served to a libwww-perl client as the bucket holds them"; fi
 task: [smoke] g=$(awk -F'\t' '$1 ~ /\.tar\.gz$/ && $2 > 0 { print $1; exit }' /work/examples/rsync/.run/applied.txt 2>/dev/null); case "https://example.katoptra.org" in http*) ;; *) g= ;; esac; if test -z "$g"; then echo "content-encoding check skipped (no non-empty .tar.gz in the state, or the URL is not http)"; else u=$(printf '%s\n' "$g" | awk 'BEGIN { for (i = 0; i < 256; i++) ord[sprintf("%c", i)] = i }
 function enc(s,  out, j, c) { out = ""; for (j = 1; j <= length(s); j++) { c = substr(s, j, 1)

From d6879cad22d47058494190237ca16bd2dcced9e1 Mon Sep 17 00:00:00 2001
From: Josh Vaughen <josh@vaughen.net>
Date: Fri, 16 Oct 2026 18:24:08 -0700
Subject: [PATCH 2/2] fix(rsync): pass the raw key to copy-object, which
 encodes it itself

The CLI percent-encodes --copy-source, so the encoded key went out
encoded twice and the copy read a different key. The copy loop is now
index-copy; offline runs it against a stub aws and compares the exact
arguments for a key with a space and a plus.

The run summary counts the copied page keys. The README says to set
INDEX_FILES only where upstream serves the index file, and that ctan
does not: CTAN asks its mirrors for DirectoryIndex disabled.
---
 CLAUDE.md                   |  7 +++++++
 README.md                   | 19 ++++++++++++-------
 engines/rsync.yml           | 30 +++++++++++++++++++-----------
 examples/rsync/Taskfile.yml | 18 ++++++++++++++++++
 examples/rsync/render.txt   |  4 +---
 5 files changed, 57 insertions(+), 21 deletions(-)

diff --git a/CLAUDE.md b/CLAUDE.md
index 44616fb..13b9588 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -109,6 +109,13 @@ Read that section before you write.
   cause, each line of `.run/index-real.txt` starts with a full key.
 - Do not give `--no-progress` (in `AWS_FLAGS`) to `aws s3api`. Only `aws s3` accepts it.
   `s3api` stops with `Unknown options` (verified, aws-cli 2.36.24).
+- Give `aws s3api copy-object --copy-source` the raw key. The CLI encodes it. A key that is
+  encoded first is encoded two times (`a%20b` goes out as `a%2520b`), and the copy then
+  reads a different key (verified with `aws --debug`, aws-cli 2.36.24).
+- In a check, give `RUN` to a verb as a call var, not as `RUN=...` on the command line. The
+  toolbox sets `RUN` as a global var, and go-task uses the command-line value in some runs
+  only (verified, 3.53.1). For this cause, `offline` starts `copy-cli`, which gives `RUN`
+  to `index-copy` as a call var.
 
 ## Verifying a change
 
diff --git a/README.md b/README.md
index 20d9012..052711c 100644
--- a/README.md
+++ b/README.md
@@ -349,7 +349,7 @@ The verbs with a broken line are hooks. These hooks do no work until a mirror se
 | `checkpoint` | Runs `merge` to add the uploaded files to the state. Then it uploads the state as one PutObject, and it deletes the files in staging |
 | `delete` | Removes the keys in `deleted.txt` from the bucket, 1,000 in each call, and removes them from the state. It waits for the run that uploads the last batch |
 | `reconcile` | If `due` wrote `.run/reconcile`, it runs `rebuild`. Then it deletes each key that is not a path of upstream, a key in `OWN`, or a directory of the state. Then it runs `reconciled` |
-| `index` | A hook. If `INDEX` has a value: it runs `pages`, uploads the two sets of keys, copies the index files of `.run/index-real.txt` to their page keys, and removes the keys of the directories that became empty. Then it moves `.state/indexed.txt.xz` to the new state. A mirror with a different landing page overrides `index` |
+| `index` | A hook. If `INDEX` has a value: it runs `pages`, uploads the two sets of keys, runs `index-copy`, which copies the index files of `.run/index-real.txt` to their page keys, and removes the keys of the directories that became empty. Then it moves `.state/indexed.txt.xz` to the new state. A mirror with a different landing page overrides `index` |
 | `pages` | Writes a page for each directory that the run changed, from the state, into staging. The keys with no slash at the end go in one tree for each depth. A directory with a file in `INDEX_FILES` gets only the key with no slash, and a line in `.run/index-real.txt` |
 | `smoke` | Reads a sample back from `HOST`. The list below the table gives each item of the sample. Then it runs `fresh` and `smoke-mirror` |
 | `fresh` | If `FRESH_KEY` has a value: it reads the clock of upstream in that file, from `HOST`. If the clock is more than `FRESH_HOURS` before the time of the run, it stops the run |
@@ -358,7 +358,7 @@ The verbs with a broken line are hooks. These hooks do no work until a mirror se
 | `retry` | Runs a command a maximum of five times. After a transport error of rsync (exit 5, 10, 12, 30 or 35), it runs the command again, with backoff. It accepts exit 23 and exit 24, and the run continues. rsync exits 23 if it cannot read a path, and 24 if upstream deletes a file while rsync sends it. Thus, a directory that rsync cannot read does not stop the run, and `delete` removes its files, the same as files that upstream deleted. Only `LIST_FLOOR` stops a large loss |
 
 The verbs in the table also run these verbs: `normalise`, `pull`, `push`, `batch`,
-`label-trees`, `merge` and `remove`. `pull` and `push` move a key with xz compression
+`label-trees`, `merge`, `remove` and `index-copy`. `pull` and `push` move a key with xz compression
 between the bucket and `.run`. A verb of a mirror can also run them.
 
 `smoke` reads these items back from `HOST`, and it stops the run if one of them is not
@@ -503,10 +503,14 @@ depth, and `index` uploads one tree at a time. `index` gives these keys
 
 - `<dir>/<INDEX>` is a server-side copy of the first such file. The copy has the same
   bytes, the same Content-Type and the same ETag. Thus, `/dir/` serves the index file, as
-  on a mirror with Apache.
+  the upstream server does.
 - `<dir>` keeps the listing.
 - If upstream removes the index file, the next run writes the listing to the two keys.
 
+Set `INDEX_FILES` only if the upstream server sends the index file for the directory URL.
+CTAN tells its mirrors not to do this (`DirectoryIndex disabled`). Thus, ctan does not set
+it.
+
 No page goes into the state. `merge` adds to the state only the files that are in staging
 and also in the batch, and a page is in no batch.
 
@@ -517,16 +521,17 @@ not change also get a change to the HTML of the page or to `PAGE_FOOT`. `reconci
 not delete the two keys.
 
 The rsync mirrors of katoptra use one `INDEX`, `katoptra.org.directory.index.html`. Thus,
-one Transform Rule of the zone serves the pages of their hosts:
+the Transform Rules of the zone for their hosts use one rewrite:
 
 ```text
 when:         ends_with(http.request.uri.path, "/")
 rewrite path: concat(http.request.uri.path, "katoptra.org.directory.index.html")
 ```
 
-The rule does not use `http.host`. Thus, two hosts of one bucket get the same key for each
-URL. This rule also serves the page of the root for `/`. In ctan, that page is a copy of
-the `index.html` of CTAN.
+The rewrite does not use `http.host`. Thus, two hosts of one bucket get the same key for
+each URL. This rule also serves the page of the root for `/`. In ctan, the rule does not
+include `/`, because a second rule serves the `index.html` of CTAN for `/`. No two rules
+match one URL.
 
 #### Content types
 
diff --git a/engines/rsync.yml b/engines/rsync.yml
index 8850270..d5648fd 100644
--- a/engines/rsync.yml
+++ b/engines/rsync.yml
@@ -745,16 +745,7 @@ tasks:
       - >-
         for d in {{.SLASH}}/*/; do test -d "$d" || continue;
         aws s3 cp {{.AWS_FLAGS}} --recursive --content-type 'text/html; charset=utf-8' "$d" {{.S3}}/; done
-      # The page key of a directory with an index file (INDEX_FILES) is one server-side
-      # copy of that file. COPY keeps its Content-Type, and the copy has the same bytes
-      # and ETag. Thus, the URL of the directory serves the upstream file, as other
-      # mirrors do. The source is in the bucket, because pages read it from the state that
-      # checkpoint put there. The copy source is a URL path. s3api does not accept
-      # --no-progress. Thus, this command does not use AWS_FLAGS.
-      - >-
-        while IFS="$(printf '\t')" read -r s k; do u=$(printf '%s\n' "$s" | {{.ENCODE}} | cut -f2);
-        aws s3api copy-object --bucket {{.BUCKET}} --copy-source "{{.BUCKET}}/$u" --key "$k" --metadata-directive COPY > /dev/null;
-        echo "copy: $s to $k"; done < {{.RUN}}/index-real.txt
+      - {task: index-copy}
       - {task: remove, vars: {K: '{{.RUN}}/index-gone.txt'}}
       # At this point, the pages show the state, and the bucket holds that state, because
       # checkpoint put it there before this verb started. One server-side copy makes
@@ -762,6 +753,23 @@ tasks:
       # before this line, the next run makes the same pages again.
       - aws s3 cp {{.AWS_FLAGS}} {{.S3}}/{{.STATE}} {{.S3}}/{{.INDEXED}}
 
+  # Not internal: the offline check of lib starts it with its name.
+  index-copy:
+    desc: Copy the index file of each directory in RUN/index-real.txt to the page key of that directory, in the bucket
+    cmds:
+      # The page key of a directory with an index file (INDEX_FILES) is one server-side
+      # copy of that file. COPY keeps its Content-Type, and the copy has the same bytes
+      # and ETag. Thus, the URL of the directory serves the upstream file, as its upstream
+      # does. The source is in the bucket, because pages read it from the state that
+      # checkpoint put there. The CLI encodes --copy-source. Thus, the command gives the
+      # key with no change. A key that is encoded first is encoded two times, and the copy
+      # then reads a different key. s3api does not accept --no-progress. Thus, this command does
+      # not use AWS_FLAGS.
+      - >-
+        while IFS="$(printf '\t')" read -r s k; do
+        aws s3api copy-object --bucket {{.BUCKET}} --copy-source "{{.BUCKET}}/$s" --key "$k" --metadata-directive COPY > /dev/null;
+        echo "copy: $s to $k"; done < {{.RUN}}/index-real.txt
+
   pages:
     desc: RUN/indexed.txt against RUN/applied.txt -> a page under both of its keys for every directory a changed line touches, RUN/index-gone.txt for the keys of directories that emptied, RUN/index-real.txt for the directories with an index file
     set: [pipefail]
@@ -1015,7 +1023,7 @@ tasks:
       # prepare made RUN/tl. Thus, the delta had a path in the signed subtree.
       - test ! -d {{.RUN}}/tl || echo "| Signature | $(test -f {{.RUN}}/tl/tlpkg/texlive.tlpdb && echo "tlpdb signed by {{.TL_KEY}}; $(test -f {{.RUN}}/tl/missing.txt && wc -l < {{.RUN}}/tl/missing.txt || echo 0) named containers missing from the bucket" || echo "the tlpdb did not verify") |" >> "${GITHUB_STEP_SUMMARY:-/dev/stdout}"
       # With INDEX set, the pages that index made and the keys that it removed.
-      - test -z "{{.INDEX}}" || echo "| Directory pages | $(test -f {{.RUN}}/index-make.txt && wc -l < {{.RUN}}/index-make.txt || echo 0) redrawn under both keys, $(test -f {{.RUN}}/index-gone.txt && wc -l < {{.RUN}}/index-gone.txt || echo 0) keys removed |" >> "${GITHUB_STEP_SUMMARY:-/dev/stdout}"
+      - test -z "{{.INDEX}}" || echo "| Directory pages | $(test -f {{.RUN}}/index-make.txt && wc -l < {{.RUN}}/index-make.txt || echo 0) redrawn, $(test -f {{.RUN}}/index-real.txt && wc -l < {{.RUN}}/index-real.txt || echo 0) of them with a copy of an index file as the page key, $(test -f {{.RUN}}/index-gone.txt && wc -l < {{.RUN}}/index-gone.txt || echo 0) keys removed |" >> "${GITHUB_STEP_SUMMARY:-/dev/stdout}"
       # Rows that show only when they have data.
       - test ! -s {{.RUN}}/unlanded.txt || echo "| Never landed | $(wc -l < {{.RUN}}/unlanded.txt | tr -d ' ') listed upstream that rsync could not send, fetched again every run; the first are warnings on the run |" >> "${GITHUB_STEP_SUMMARY:-/dev/stdout}"
       - test ! -f {{.RUN}}/fresh.txt || echo "| Upstream clock | $(awk '{ printf "%d h %d min", $1 / 3600, $1 % 3600 / 60 }' {{.RUN}}/fresh.txt) old in {{.FRESH_KEY}}, limit {{.FRESH_H}} h |" >> "${GITHUB_STEP_SUMMARY:-/dev/stdout}"
diff --git a/examples/rsync/Taskfile.yml b/examples/rsync/Taskfile.yml
index e4b2288..0fe1393 100644
--- a/examples/rsync/Taskfile.yml
+++ b/examples/rsync/Taskfile.yml
@@ -175,6 +175,20 @@ tasks:
         printf 'doc/index.html\tdoc/{{.INDEX}}\n' > $r/index-real.txt; : > $r/applied.txt
       - {task: smoke, vars: {RUN: '{{.F}}/run-copy', STAGING: '{{.F}}/run-copy/staging', URL: 'file://{{.F}}/run-copy/staging'}}
       - '! task smoke-refuses-copy 2>/dev/null'
+      # run-copy-cli: index-copy gives each source key to the CLI with no change, because
+      # the CLI encodes --copy-source. A key that is encoded first is encoded two times,
+      # and the copy then reads a different key. A stub aws, first on PATH, records the arguments.
+      - |
+        r={{.F}}/run-copy-cli; mkdir -p $r/bin
+        cat > $r/bin/aws <<'EOF'
+        #!/bin/sh
+        printf '%s\n' "$@" >> "$(dirname "$0")/../args"
+        EOF
+        chmod +x $r/bin/aws; printf 'a b/c+d/index.html\ta b/c+d/{{.INDEX}}\n' > $r/index-real.txt
+      - PATH="{{.F}}/run-copy-cli/bin:$PATH" task copy-cli
+      - >-
+        printf '%s\n' s3api copy-object --bucket {{.BUCKET}} --copy-source '{{.BUCKET}}/a b/c+d/index.html'
+        --key 'a b/c+d/{{.INDEX}}' --metadata-directive COPY | diff - {{.F}}/run-copy-cli/args
       # run-label: the command makes it here, because this test is about its names and its
       # bytes. It has a test case for each rule of label:
       #   - A COMPRESSED ending is more important than the remaining part of the name
@@ -247,6 +261,10 @@ tasks:
     cmds:
       - rm -f {{.R}}/staging/{{.INDEX}}
       - {task: smoke, vars: {RUN: '{{.R}}', STAGING: '{{.R}}/staging', URL: 'file://{{.R}}/staging'}}
+  copy-cli:
+    desc: index-copy over run-copy-cli, after offline made it; offline runs this task with a stub aws first on PATH
+    cmds:
+      - {task: index-copy, vars: {RUN: '{{.RUN}}/fixtures/run-copy-cli'}}
   smoke-refuses-copy:
     desc: smoke over run-copy with the page key one byte longer than its source, after offline made run-copy; offline expects it to fail
     vars:
diff --git a/examples/rsync/render.txt b/examples/rsync/render.txt
index 1dca582..ece27e4 100644
--- a/examples/rsync/render.txt
+++ b/examples/rsync/render.txt
@@ -315,9 +315,7 @@ function enc(s,  out, j, c) { out = ""; for (j = 1; j <= length(s); j++) { c = s
   END { if (live) shut() }'
 task: [index] test ! -s /work/examples/rsync/.run/index-make.txt || aws s3 cp --no-progress --cli-connect-timeout 60 --cli-read-timeout 300 --recursive --content-type 'text/html; charset=utf-8' /work/examples/rsync/staging/ s3://example/
 task: [index] for d in /work/examples/rsync/.run/slash/*/; do test -d "$d" || continue; aws s3 cp --no-progress --cli-connect-timeout 60 --cli-read-timeout 300 --recursive --content-type 'text/html; charset=utf-8' "$d" s3://example/; done
-task: [index] while IFS="$(printf '\t')" read -r s k; do u=$(printf '%s\n' "$s" | awk 'BEGIN { for (i = 0; i < 256; i++) ord[sprintf("%c", i)] = i }
-function enc(s,  out, j, c) { out = ""; for (j = 1; j <= length(s); j++) { c = substr(s, j, 1)
-    out = out ((c ~ /[A-Za-z0-9\/._~-]/) ? c : sprintf("%%%02X", ord[c])) } return out } { print $0 "\t" enc($0) }' | cut -f2); aws s3api copy-object --bucket example --copy-source "example/$u" --key "$k" --metadata-directive COPY > /dev/null; echo "copy: $s to $k"; done < /work/examples/rsync/.run/index-real.txt
+task: [index-copy] while IFS="$(printf '\t')" read -r s k; do aws s3api copy-object --bucket example --copy-source "example/$s" --key "$k" --metadata-directive COPY > /dev/null; echo "copy: $s to $k"; done < /work/examples/rsync/.run/index-real.txt
 task: [remove] rm -f /work/examples/rsync/.run/del-*; split -l 1000 -a 4 /work/examples/rsync/.run/index-gone.txt /work/examples/rsync/.run/del-
 task: [remove] for f in /work/examples/rsync/.run/del-*; do awk 'BEGIN { printf "{\"Objects\":[" } { gsub(/\\/, "\\\\"); gsub(/"/, "\\\"")
      printf "%s{\"Key\":\"%s\"}", (NR > 1 ? "," : ""), $0 } END { print "],\"Quiet\":true}" }' "$f" \