From 3d70493643c0e809f8f7c9e6e0a38063d28617bf Mon Sep 17 00:00:00 2001 From: tannevaled Date: Wed, 30 Sep 2026 18:03:06 +0200 Subject: [PATCH 01/28] images: count the pictures paired by size, and say so PairedBy's own comment states the requirement: A run whose size share is large is a run whose numbers are worth less, and that has to be visible. It was not visible anywhere. The pairing already records how each picture was matched -- by object number, or by falling back to size -- but Tally threw that away, so neither the report nor the JSON record carried it. A reader of either had no way to tell a run where every picture was matched by its object number from one where the matching was guesswork. Counts gains SizePaired, Tally increments it on PairedBySize only, the filter line ends with "N paired by size", and FilterCounts carries it into the record (omitempty, so existing records that have no such number stay as they are). The test asserts both directions. A counter that increments on every picture reads exactly like one that works, until the run that is all object-paired reports every picture as suspect -- so the JPXDecode row, which has no size-paired picture in it, must come out zero. Checked by mutation: counting unconditionally, dropping the report column, and dropping the record field each make it fail. --- images/images.go | 24 +++++++++++++++++++++--- images/images_test.go | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 3 deletions(-) diff --git a/images/images.go b/images/images.go index 5bd0484..26caf00 100644 --- a/images/images.go +++ b/images/images.go @@ -1157,6 +1157,18 @@ type Bucket struct { type Counts struct { // Pictures is how many were judged. Pictures int + // SizePaired is how many were matched to the judge's picture by SIZE + // rather than by object number. + // + // PairedBy's own note says a run whose size share is large is a run whose + // numbers are worth less, "and that has to be visible". It was not: the + // field was recorded per picture and no report or record carried it, so + // the share could not be read from a run at all. conformance#13 found 144 + // of 173 apparent inversions were this fallback pairing pictures that had + // nothing to do with each other, which is what makes the share worth + // printing beside the counts it qualifies rather than kept for whoever + // thinks to ask. + SizePaired int // Unmatched is how many the other implementation had no picture for. Unmatched int // Remapped is how many carried a /Decode array, which we apply and @@ -1199,6 +1211,9 @@ func Tally(rs []Result) map[string]*Counts { by[key] = c } c.Pictures++ + if r.PairedBy == PairedBySize { + c.SizePaired++ + } switch { case r.RawBits: c.RawBits++ @@ -1251,8 +1266,8 @@ func Report(by map[string]*Counts) string { var sb strings.Builder for _, key := range order(by) { c := by[key] - fmt.Fprintf(&sb, "%-22s %5d pictures %5d unmatched %5d remapped\n", - key, c.Pictures, c.Unmatched, c.Remapped) + fmt.Fprintf(&sb, "%-22s %5d pictures %5d unmatched %5d remapped %5d paired by size\n", + key, c.Pictures, c.Unmatched, c.Remapped, c.SizePaired) reportBucket(&sb, "direct", &c.Direct) reportBucket(&sb, "converted", &c.Converted) } @@ -1337,6 +1352,9 @@ type FilterCounts struct { // Unmatched and Remapped are the pictures no comparison was made of. Unmatched int `json:"unmatched"` Remapped int `json:"remapped"` + // SizePaired is how many were paired by size rather than by object. A + // large share here devalues every figure on the line beside it. + SizePaired int `json:"sizePaired,omitempty"` // RawBits is the pictures the judge wrote as samples rather than as // colour, which is a third way of not being asked the same question. RawBits int `json:"rawBits,omitempty"` @@ -1401,7 +1419,7 @@ func Summarize(population string, documents int, rs []Result) Summary { by := Tally(rs) for _, key := range order(by) { c := by[key] - s.Filters = append(s.Filters, FilterCounts{Filter: key, + s.Filters = append(s.Filters, FilterCounts{Filter: key, SizePaired: c.SizePaired, Pictures: c.Pictures, Unmatched: c.Unmatched, Remapped: c.Remapped, RawBits: c.RawBits, Direct: bucketCounts(&c.Direct), Converted: bucketCounts(&c.Converted)}) diff --git a/images/images_test.go b/images/images_test.go index 8bff692..154a45d 100644 --- a/images/images_test.go +++ b/images/images_test.go @@ -1757,3 +1757,43 @@ func TestAPaletteInAStreamIsReadToo(t *testing.T) { t.Errorf("a black-and-white palette in a stream was counted apart: %v", got) } } + +func TestTheSizePairedShareIsCounted(t *testing.T) { + // PairedBy's own note says a run whose size share is large is a run whose + // numbers are worth less, "and that has to be visible". Until this counter + // it was in no report and no record. + // + // Both sides are asserted. A counter that increments on every picture reads + // exactly like one that works, until the run that is all object-paired says + // every picture is suspect. + rows := []Result{ + {Name: "a", Filter: "DCTDecode", PairedBy: PairedBySize}, + {Name: "b", Filter: "DCTDecode", PairedBy: PairedByObject}, + {Name: "c", Filter: "DCTDecode", PairedBy: PairedBySize}, + {Name: "d", Filter: "JPXDecode", PairedBy: PairedByObject}, + } + by := Tally(rows) + if got := by["DCTDecode"].SizePaired; got != 2 { + t.Errorf("DCTDecode counted %d paired by size, want 2", got) + } + if got := by["JPXDecode"].SizePaired; got != 0 { + t.Errorf("JPXDecode counted %d paired by size, want 0 -- the counter "+ + "fires on pictures it should not", got) + } + // And it reaches the two places anyone reads: the report and the record. + if rep := Report(by); !strings.Contains(rep, "2 paired by size") { + t.Errorf("the report does not carry the share:\n%s", rep) + } + var found bool + for _, f := range Summarize("p", 1, rows).Filters { + if f.Filter == "DCTDecode" { + found = true + if f.SizePaired != 2 { + t.Errorf("the record says %d paired by size, want 2", f.SizePaired) + } + } + } + if !found { + t.Error("the record has no DCTDecode line to carry it") + } +} From c83b4d1007b0d0c2fc9b493dcdca032cd72b27c7 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Thu, 1 Oct 2026 15:31:30 +0200 Subject: [PATCH 02/28] baseline: answer the pairing-audit entry, and date the records MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two entries in "What is not measured, and why" are addressed, one of them by closing it and one by naming what blocks it. The pairing entry asked whether a per-picture pairing audit had been run since conformance#13 closed. It is now counted in every run rather than audited once, and the first reading is ZERO over 1 606 pictures -- 330 across the five pdfscans populations, 1 276 across all eighteen of pdfforms, 53 filter rows, none paired by size. A zero needs a control, because a counter that never fires reads exactly like a fallback that never fires. Ablating the object test -- object forced to 0 inside match, the only input its first loop reads -- pairs EVERY picture of the same sample by size. The counter is live and the zero belongs to the corpus. §30 is the other half, and it is the uncomfortable one. §29 says this file must be read backwards because a section is true of the run that wrote it; that applies to the records too, and there a strikethrough will not do. All twenty-three are dated 2026-09-24 at render v0.35.0 and jpeg2000 v0.1.0, while go.mod holds v0.60.0/v0.9.1 and §29 measures v0.67.0/v0.13.2 -- four subjects in one document, with the machine-readable half the oldest of them. Nineteen jpeg2000 releases have landed since the version the records name, including the decoder of §25 and the three blank pages of §26. That is what blocks the aggregate bound. The twenty-one buckets carrying terms do invite one -- mse.worst 0.654, |mean.worst| 0.579, peak.worst 4 everywhere but gh-openpdf's converted bucket at 20 -- but a bound fitted to them is fitted to v0.35.0, and a gate borrowed from code that no longer exists reddens on the first honest change and is then deleted. The next step is the records, not the bound. No gate is added. A test asserting the records name go.mod's version would redden every dependency bump; the move here is §28's and conformance#92's -- make the number visible and let the reader see that a record is six days and thirty-two releases behind what it is cited for. --- baseline/README.md | 76 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 74 insertions(+), 2 deletions(-) diff --git a/baseline/README.md b/baseline/README.md index b2a9f03..145b2d8 100644 --- a/baseline/README.md +++ b/baseline/README.md @@ -1979,9 +1979,31 @@ it is the only part where the answer is ground the references do not cover. see that an entry here can outlive its subject. [conformance#13](https://github.com/go-pdfkit/conformance/issues/13), cited beside it, is **closed**: the disagreements it found were this repository's - size matcher, and `match` now pairs by OBJECT first. Whether a per-picture + size matcher, and `match` now pairs by OBJECT first. ~~Whether a per-picture pairing audit has been run since is still not recorded, and that part of the - entry stands. + entry stands.~~ **It is recorded now, in every run.** + + `PairedBy` already said what follows from itself -- *"a run whose size share + is large is a run whose numbers are worth less, and that has to be visible"* + -- and `Tally` dropped it, so the share reached no report line and no record. + It is counted as of + [conformance#92](https://github.com/go-pdfkit/conformance/pull/92): + `SizePaired` on the filter line and in the JSON. An audit is a one-off that + ages; a counter does not. + + **First reading: ZERO, over 1 606 pictures.** Measured 2026-10-01 at the + `go.mod` of this run (`render` v0.60.0, `go-images/jpeg2000` v0.9.1): 330 + pictures over the five `pdfscans` populations (`-limit 40`) and 1 276 over all + eighteen of `pdfforms` (`-limit 60`), 53 filter rows, not one paired by size. + Every picture both sides took out was matched by the one identity both + publish. + + **A zero needs a control, because a counter that never fires reads exactly + like a fallback that never fires.** The object test was ablated -- `object` + forced to 0 inside `match`, which is the only input the first loop reads -- + and the same sample came back with **every** picture paired by size (7 of 7 + `DCTDecode`, 16 of 16 `JPXDecode`, and so on down the rows). The counter is + live; the zero is a property of the corpus. - **No aggregate bound is applied.** `mse` and `mean` are recorded in FFmpeg's and pdfium's units and bounded by nothing, because no bound has been measured for pictures that were *extracted* rather than rendered. Choosing one from @@ -2619,3 +2641,53 @@ forwards, as I did, produces a defect hunt for something fixed two months ago. The same hazard, in a smaller form, is in §27: a sweep's "drawn by neither" line counts the two builds it compares, not us and the judge. + +## §30 — §29's hazard, in the half of this document a program reads + +§29 says this file has to be read backwards, because a section is true of the +run that produced it. That is about the prose. The same thing has happened to +the **records**, and there it is worse: a paragraph can be struck through, a +`.json` either describes the code or it does not. + +Read off the files themselves, 2026-10-01: + +| what | render | `go-images/jpeg2000` | dated | +|---|---|---|---| +| the twenty-three `baseline/*.json` | **v0.35.0** | **v0.1.0** | 2026-09-24 | +| `go.mod`, which is what the tool in this repository builds against | v0.60.0 | v0.9.1 | — | +| §24, the timings | v0.60.0 | v0.9.1 | 2026-09-28 | +| §29, measured in a tree with the modules upgraded | **v0.67.0** | **v0.13.2** | 2026-09-30 | + +**The document has four subjects at once and the machine-readable half is the +oldest of them.** Every `taken` in `baseline/` is 2026-09-24T17:08Z..20:07Z, and +`go-images/jpeg2000` has had **nineteen** releases since the version those +records name (v0.2.0 through v0.13.2, counted off the tag list) — among them the whole 8.6-bytes-per-pixel decoder of §25 and the +three pages of §26 that came back blank and now draw. Nothing in the records +says they predate that work, because when they were written there was no later. + +### What this blocks, by name + +The entry above that reads *"No aggregate bound is applied … choosing one from +these records is a job for a later run, and the records carry the terms"* cannot +be done from these records. Their twenty-one buckets with `terms` put +`mse.worst` at 0.654 and `|mean.worst|` at 0.579, with `peak.worst` at 4 in +every bucket but one (`gh-openpdf DCTDecode converted`, at 20) -- a tight, +inviting distribution, and **a bound fitted to it would be fitted to v0.35.0.** +A gate borrowed from a measurement of code that no longer exists goes red on +the first legitimate change and is then deleted, which costs more than never +having had it. + +So the order is: land the `render` v0.67.0 bump that is open as +[conformance#55](https://github.com/go-pdfkit/conformance/pull/55), re-take the +twenty-three records at that `go.mod`, and read the distribution again. **The +bound is not the next step. The records are.** + +### Why this is not a gate + +The obvious fix -- a test asserting every record names the `render` version in +`go.mod` -- would redden CI on every dependency bump, including the ones that +change no pixel, and a gate that reddens on routine upgrades is a gate someone +removes. What is wanted is the move §28 and conformance#92 both make: make the +number **visible**, and let a reader see for themselves that a record is six +days and thirty-two releases behind the code it is cited for. This table is +that, until the records are re-taken. From d03a6485ff9700b92d6ff088f0e8b5829221f26d Mon Sep 17 00:00:00 2001 From: tannevaled Date: Sat, 3 Oct 2026 08:07:14 +0200 Subject: [PATCH 03/28] fix(deps): update dependencies (#55) --- go.mod | 4 +- go.sum | 8 ++-- judges/package-lock.json | 94 ++++++++++++++++++++-------------------- 3 files changed, 53 insertions(+), 53 deletions(-) diff --git a/go.mod b/go.mod index 1e5f224..d522541 100644 --- a/go.mod +++ b/go.mod @@ -5,13 +5,13 @@ go 1.26.4 require ( github.com/go-gfx/gfx v0.34.0 github.com/go-pdfkit/reader v0.6.0 - github.com/go-pdfkit/render v0.60.0 + github.com/go-pdfkit/render v0.67.0 ) require ( github.com/ajroetker/go-highway v0.0.12 // indirect github.com/go-images/jpeg v0.1.0 // indirect - github.com/go-images/jpeg2000 v0.9.1 // indirect + github.com/go-images/jpeg2000 v0.13.2 // indirect github.com/go-opentype/fonts v0.10.0 // indirect github.com/go-opentype/opentype v0.13.0 // indirect github.com/go-pdfkit/pdffont v0.3.1 // indirect diff --git a/go.sum b/go.sum index 7fe2107..c27655d 100644 --- a/go.sum +++ b/go.sum @@ -4,8 +4,8 @@ github.com/go-gfx/gfx v0.34.0 h1:K46srusNnZls/xkmJRvxpBajlptgrXc6d3exa4ZxhoA= github.com/go-gfx/gfx v0.34.0/go.mod h1:TC2X+3GMXm8mYZPmp/v7bFrRa69p+k0xz0Nabbd9sII= github.com/go-images/jpeg v0.1.0 h1:cfg5WNy0FtPz0kmy573vA1MOWuH3G2jlxbHSiPlayDQ= github.com/go-images/jpeg v0.1.0/go.mod h1:2C1DevAM+HaQlysTLbSdyol8X/RlvmsXmHAXs9teAqo= -github.com/go-images/jpeg2000 v0.9.1 h1:vn817Lm2h+cS4UghT+WlX5KUZKv3BTxrE5H65Ff6z4U= -github.com/go-images/jpeg2000 v0.9.1/go.mod h1:Qqxz1gBrUwFDBOGkA4RGyTsX7wifr0wvaTQ7k8FK0wY= +github.com/go-images/jpeg2000 v0.13.2 h1:wthA3iZkyqWXzNXxd6EjjsrsQ7GOYX7vYZbaI4vU6+I= +github.com/go-images/jpeg2000 v0.13.2/go.mod h1:Qqxz1gBrUwFDBOGkA4RGyTsX7wifr0wvaTQ7k8FK0wY= github.com/go-opentype/fonts v0.10.0 h1:9f7ik795SlBEJjz6iqBE/ZV2PaTauAj537MMWw9t4OQ= github.com/go-opentype/fonts v0.10.0/go.mod h1:b+o8v1zDZ6B3GixA398jGILAh09+WwfyoYz52zKe7DA= github.com/go-opentype/opentype v0.13.0 h1:q9K7IGDfwMP/ik51ZkwiYf6jbaqPfocvAIuojetd0UQ= @@ -14,8 +14,8 @@ github.com/go-pdfkit/pdffont v0.3.1 h1:+K7opHGRERDZ5EMkA12s1GT48t8GpncArhnyzyc5b github.com/go-pdfkit/pdffont v0.3.1/go.mod h1:eFJ/7t9AvcX76KY5qNJMMKetz8hUUUdVM1qSWLi3S1g= github.com/go-pdfkit/reader v0.6.0 h1:KAabNOYUcTlZlNBTbG9bEhWP1NiZhjuhzUdavdTdfes= github.com/go-pdfkit/reader v0.6.0/go.mod h1:fQFOVfCMUui1AdvD4qhimdyvvNr9KvvJ1S7IuKZjyV8= -github.com/go-pdfkit/render v0.60.0 h1:YJpcA47fDedQX+RMLoe4V2004GdGCegOe+g0bOxRPuU= -github.com/go-pdfkit/render v0.60.0/go.mod h1:wVfQmpHJpjM3rP3LLq08Y1Q8aVqOqtunPw7rIYtjx2k= +github.com/go-pdfkit/render v0.67.0 h1:5PB8HaWPpWYfcsTMYAFO2CWHQkYO6VTQ+S5455jyuVk= +github.com/go-pdfkit/render v0.67.0/go.mod h1:uq6fkICiu8kxBc2y5JU7OyirEozzFz875U2KkpunDMM= github.com/sergeymakinen/go-bmp v1.0.0 h1:SdGTzp9WvCV0A1V0mBeaS7kQAwNLdVJbmHlqNWq0R+M= github.com/sergeymakinen/go-bmp v1.0.0/go.mod h1:/mxlAQZRLxSvJFNIEGGLBE/m40f3ZnUifpgVDlcUIEY= github.com/sergeymakinen/go-ico v1.0.0 h1:uL3khgvKkY6WfAetA+RqsguClBuu7HpvBB/nq/Jvr80= diff --git a/judges/package-lock.json b/judges/package-lock.json index 02fddd5..8e6e04f 100644 --- a/judges/package-lock.json +++ b/judges/package-lock.json @@ -14,9 +14,9 @@ } }, "node_modules/@napi-rs/canvas": { - "version": "1.0.9", - "resolved": "https://registry.npmjs.org/@napi-rs/canvas/-/canvas-1.0.9.tgz", - "integrity": "sha512-QviPdJImDi/jMAvBfqaw+19BndMd/sizXVW3NnpMd3VJGz++QXkOHcP9kWR/smHG0hNjHeyuHFyrx/5lD0oNcQ==", + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas/-/canvas-1.0.10.tgz", + "integrity": "sha512-V+qptzsGnPMFxEbhwc6eDNMe5eDrfiBIp8qGpuRjAVIX4Kn2zxvwQ1Xco6Fwe+3tYIe3iLW8LM9D+SvjyJhqbQ==", "license": "MIT", "workspaces": [ "e2e/*" @@ -29,23 +29,23 @@ "url": "https://github.com/sponsors/Brooooooklyn" }, "optionalDependencies": { - "@napi-rs/canvas-android-arm64": "1.0.9", - "@napi-rs/canvas-darwin-arm64": "1.0.9", - "@napi-rs/canvas-darwin-x64": "1.0.9", - "@napi-rs/canvas-linux-arm-gnueabihf": "1.0.9", - "@napi-rs/canvas-linux-arm64-gnu": "1.0.9", - "@napi-rs/canvas-linux-arm64-musl": "1.0.9", - "@napi-rs/canvas-linux-riscv64-gnu": "1.0.9", - "@napi-rs/canvas-linux-x64-gnu": "1.0.9", - "@napi-rs/canvas-linux-x64-musl": "1.0.9", - "@napi-rs/canvas-win32-arm64-msvc": "1.0.9", - "@napi-rs/canvas-win32-x64-msvc": "1.0.9" + "@napi-rs/canvas-android-arm64": "1.0.10", + "@napi-rs/canvas-darwin-arm64": "1.0.10", + "@napi-rs/canvas-darwin-x64": "1.0.10", + "@napi-rs/canvas-linux-arm-gnueabihf": "1.0.10", + "@napi-rs/canvas-linux-arm64-gnu": "1.0.10", + "@napi-rs/canvas-linux-arm64-musl": "1.0.10", + "@napi-rs/canvas-linux-riscv64-gnu": "1.0.10", + "@napi-rs/canvas-linux-x64-gnu": "1.0.10", + "@napi-rs/canvas-linux-x64-musl": "1.0.10", + "@napi-rs/canvas-win32-arm64-msvc": "1.0.10", + "@napi-rs/canvas-win32-x64-msvc": "1.0.10" } }, "node_modules/@napi-rs/canvas-android-arm64": { - "version": "1.0.9", - "resolved": "https://registry.npmjs.org/@napi-rs/canvas-android-arm64/-/canvas-android-arm64-1.0.9.tgz", - "integrity": "sha512-4LGXk2/0HVzE29K8SzML5WubgCp++B1FH3qgl35XmSZE+lLdr6P9VRQEnZ0MCLZMTSuJP41yyhtoiVNEuvrTIA==", + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-android-arm64/-/canvas-android-arm64-1.0.10.tgz", + "integrity": "sha512-0sDMvme+7fj6bHERUaBs5SvEyWaORc91VUkaB4RrtG7p6dBrBNGy1of8cyEKDxygd1FlHtW+Thunj8lSixzY7A==", "cpu": [ "arm64" ], @@ -63,9 +63,9 @@ } }, "node_modules/@napi-rs/canvas-darwin-arm64": { - "version": "1.0.9", - "resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-arm64/-/canvas-darwin-arm64-1.0.9.tgz", - "integrity": "sha512-YNdfLBzY0W/Pep9fo2L6RmoNlNksnn05LRnX66W63R3ij58S25QOTcjdtEt2v8+PnCESzqZsYzUo+QPeIR44NA==", + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-arm64/-/canvas-darwin-arm64-1.0.10.tgz", + "integrity": "sha512-pdHL1TEfFMrnchRYqmPWXKN7hR4XtOYmzJqnh4XhpfWD4/eeSokOGVzrdFP/iQ3sfr2rf0939gOGq9uCp8BGmg==", "cpu": [ "arm64" ], @@ -83,9 +83,9 @@ } }, "node_modules/@napi-rs/canvas-darwin-x64": { - "version": "1.0.9", - "resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-x64/-/canvas-darwin-x64-1.0.9.tgz", - "integrity": "sha512-ceZQSknTEcy3dOXoekv59LTCkXjvnLsq+VW5PeNNDHEPQbRS5Ervkm1EaDa7WLAjiYWMLuSQTRTHao1dEX4prg==", + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-x64/-/canvas-darwin-x64-1.0.10.tgz", + "integrity": "sha512-iuuk5plGxAcxurxv2ycF2yR20y5zvaBZ4heQdhPpg3EMFD21PhlY1V+YwSRmYRWzbvKL474UYdw3WIg2qfafyw==", "cpu": [ "x64" ], @@ -103,9 +103,9 @@ } }, "node_modules/@napi-rs/canvas-linux-arm-gnueabihf": { - "version": "1.0.9", - "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm-gnueabihf/-/canvas-linux-arm-gnueabihf-1.0.9.tgz", - "integrity": "sha512-XhfI0Wwv4llhd6nnWDtY3kQKjq0r+y1i91PlJlJI24ag2U9WrnwbG1qS3+fDLEyouwEVFchiKkTEHozK+5iUNA==", + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm-gnueabihf/-/canvas-linux-arm-gnueabihf-1.0.10.tgz", + "integrity": "sha512-kH7GFR5Unpm77pKSEsmkqvzeQCRD7/MQ7upmeLLjg4XpQ62dX/R6THFsp6vkF1Pkco2aKUq9KhoLM4W5twdk7g==", "cpu": [ "arm" ], @@ -123,9 +123,9 @@ } }, "node_modules/@napi-rs/canvas-linux-arm64-gnu": { - "version": "1.0.9", - "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-gnu/-/canvas-linux-arm64-gnu-1.0.9.tgz", - "integrity": "sha512-012oiYtKaE7i9oxc8q7nraT7kDOpLcaCmFLzVe9Ty34RHDdoDzbWLrVh827CNxYh/EADX1eSikA3ymLjo/nNuw==", + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-gnu/-/canvas-linux-arm64-gnu-1.0.10.tgz", + "integrity": "sha512-0/Hj7IwM9pmzdwxiqkp4YLJ8dKqASBrWQ3Y7dWUSU/4dgvCL9O6YNvX7xVUJjonQvdD3IE9ESS/nXa5srQqoVw==", "cpu": [ "arm64" ], @@ -146,9 +146,9 @@ } }, "node_modules/@napi-rs/canvas-linux-arm64-musl": { - "version": "1.0.9", - "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-musl/-/canvas-linux-arm64-musl-1.0.9.tgz", - "integrity": "sha512-Ls5UWYFFn63casTZEczbeyEg3vRDRkv9lscuGwfchtY5yLLQhgOB8SN4YGxmfJ5vTaBwZ2YUxBS3NtmjJmFXdA==", + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-musl/-/canvas-linux-arm64-musl-1.0.10.tgz", + "integrity": "sha512-fmjKM0P2MLF9O6XzCQsRjhB+jlLzaf6tFQU36hMPMsYup+VThYvpjjhqNLOc/Ee3tHUE3r13H2BSJx/yXz4pFA==", "cpu": [ "arm64" ], @@ -169,9 +169,9 @@ } }, "node_modules/@napi-rs/canvas-linux-riscv64-gnu": { - "version": "1.0.9", - "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-riscv64-gnu/-/canvas-linux-riscv64-gnu-1.0.9.tgz", - "integrity": "sha512-hLKEGxV7ZiRHqndePTokgDMdBlo/rDfzg7P4p4QIv9pUhuYobnu3R2NIFLCRghG0nwfo+s2sw+c1xZFeCmEAsw==", + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-riscv64-gnu/-/canvas-linux-riscv64-gnu-1.0.10.tgz", + "integrity": "sha512-O/BUrGwrs3pVP9PpD/3++ZiOpjHMJlmiB7aBgswP8rbe4egPfiFkFxmcCczoLvZa72TaQX5lJ8Udedb6rc/Hvg==", "cpu": [ "riscv64" ], @@ -192,9 +192,9 @@ } }, "node_modules/@napi-rs/canvas-linux-x64-gnu": { - "version": "1.0.9", - "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-gnu/-/canvas-linux-x64-gnu-1.0.9.tgz", - "integrity": "sha512-6kaz3w0QMy77PDWk6rJ1ksIihdad3qzEyX2o2oGT8GwCaypfT5mhjr8buOO5hstyLxcWXDScuz56RsINLtBPIQ==", + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-gnu/-/canvas-linux-x64-gnu-1.0.10.tgz", + "integrity": "sha512-48HkZPQeAN/R+9NPpY64tceoyCUW5xYYtHKZnC+BG11qiihXJCbH+xfbgGU+OdYp1Q4s84HDl9ILU0KBK6SBOQ==", "cpu": [ "x64" ], @@ -215,9 +215,9 @@ } }, "node_modules/@napi-rs/canvas-linux-x64-musl": { - "version": "1.0.9", - "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-musl/-/canvas-linux-x64-musl-1.0.9.tgz", - "integrity": "sha512-xrGvmS3v55hmZ86ls/kBLVNMUTYio3f6Ik0DireemG994VfPAwiA3ZXA0Uf1bByctkB3NQ1Sfb+H5bkdUnnzfQ==", + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-musl/-/canvas-linux-x64-musl-1.0.10.tgz", + "integrity": "sha512-QYHVi7WZ9v+Wm9OQyHLMTnJ8QiBPVX94BgbCHFJC/4bCFMfvX7fRzo4FJPgkRP0ISz3ulYNow/FCbQOPjzhYjg==", "cpu": [ "x64" ], @@ -238,9 +238,9 @@ } }, "node_modules/@napi-rs/canvas-win32-arm64-msvc": { - "version": "1.0.9", - "resolved": "https://registry.npmjs.org/@napi-rs/canvas-win32-arm64-msvc/-/canvas-win32-arm64-msvc-1.0.9.tgz", - "integrity": "sha512-yjmVS3ArZeRVCP7jqbPq4rpZa/BhTeI7ELE2XqJg3snICQBDevLZyArxswHkiTnT34KRic33/4fLirrHI+SY8A==", + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-win32-arm64-msvc/-/canvas-win32-arm64-msvc-1.0.10.tgz", + "integrity": "sha512-bWK+YajM/8sL0mMgLtug2s063FVBPzFV4QAn987CQomMKlmzrBY6Trz43KY+24hq7IOa3qNhUAgP/XMEF50ZEQ==", "cpu": [ "arm64" ], @@ -258,9 +258,9 @@ } }, "node_modules/@napi-rs/canvas-win32-x64-msvc": { - "version": "1.0.9", - "resolved": "https://registry.npmjs.org/@napi-rs/canvas-win32-x64-msvc/-/canvas-win32-x64-msvc-1.0.9.tgz", - "integrity": "sha512-QlSYQdMQslB81nlABo9wNfQ6npFhE7/O+saCZdqVGueGanyRk4jCogD5EwQenfP3kIq9e+mm6GreQBjX5MrA8g==", + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-win32-x64-msvc/-/canvas-win32-x64-msvc-1.0.10.tgz", + "integrity": "sha512-9WtzW85PaIhtB27iXaR/9RrLCqHLbqD3Z8Q+kh17NjFxwQXFXU5vWbYH4vEH+drM307TTyK3hidZgpt09G5T3g==", "cpu": [ "x64" ], From 49c24a7075011ffb60cdd1802cf4b29159e395f5 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Sat, 3 Oct 2026 22:20:50 +0200 Subject: [PATCH 04/28] baseline: say how the twenty-three records are re-taken MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit §30 names the records as the next step, and nothing in the repository said how to take them. The top-level README shows one invocation of "images -json"; neither file said how the SET is taken, what it is named, or what it costs. So: one invocation per population, which is where the filenames in baseline/ come from, and omitting -only gives one record carrying every population of that corpus -- not the shape baseline/ holds. With the cost, measured one at a time on 2026-10-01: fr-cerfa (450 forms) about two minutes, ia-biodiversity (250 scans) about fifty. And with the reason to take them one at a time, which is §22's: the judge is held to a wall-clock bound, that bound is a property of the machine rather than of the corpus, and two runs beside each other can turn a document into a hung entry neither would produce alone. --- baseline/README.md | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/baseline/README.md b/baseline/README.md index 145b2d8..099713f 100644 --- a/baseline/README.md +++ b/baseline/README.md @@ -2671,8 +2671,9 @@ The entry above that reads *"No aggregate bound is applied … choosing one from these records is a job for a later run, and the records carry the terms"* cannot be done from these records. Their twenty-one buckets with `terms` put `mse.worst` at 0.654 and `|mean.worst|` at 0.579, with `peak.worst` at 4 in -every bucket but one (`gh-openpdf DCTDecode converted`, at 20) -- a tight, -inviting distribution, and **a bound fitted to it would be fitted to v0.35.0.** +every bucket but one (`gh-openpdf DCTDecode converted`, at 20, which §17 +already accounts for) -- a tight, inviting distribution, and **a bound fitted to +it would be fitted to v0.35.0.** A gate borrowed from a measurement of code that no longer exists goes red on the first legitimate change and is then deleted, which costs more than never having had it. @@ -2691,3 +2692,31 @@ removes. What is wanted is the move §28 and conformance#92 both make: make the number **visible**, and let a reader see for themselves that a record is six days and thirty-two releases behind the code it is cited for. This table is that, until the records are re-taken. + +### How the twenty-three are re-taken + +§30 says the records are the next step. The top-level README shows a single +invocation of `images -json`; neither file said how the **set** is taken, what +it is named, or what it costs. One invocation per population, named after the corpus directory and the +population, which is where the filenames in `baseline/` come from: + +```sh +for pop in ia-americana ia-biodiversity ia-medical ia-texts ia-uscourts; do + images -dir /Users/Shared/pdfscans -only "$pop" -json > "baseline/pdfscans-$pop.json" +done +``` + +and the eighteen of `/Users/Shared/pdfforms` the same way. Omitting `-only` +writes **one** record carrying every population of that corpus, which is not +the shape `baseline/` holds. + +**They are not interchangeable in cost.** Measured 2026-10-01 on an M4 Max, one +at a time: `fr-cerfa` (450 forms) takes about two minutes, `ia-biodiversity` +(250 scans) about **fifty**. The scans dominate, and the whole set is an +afternoon. + +**Take them one at a time.** The judge is held to a wall-clock bound +(`-timeout`, 2 minutes by default) and §22 is what it cost to learn that the +bound is a property of the machine, not of the corpus: two runs in parallel can +turn a document into a `hung` entry that neither would produce alone. A count +measured beside another copy of itself is not a count of the corpus. From 8ffddedfb0853aacbbd772e7d474f765134fa8d8 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Sat, 3 Oct 2026 23:07:51 +0200 Subject: [PATCH 05/28] =?UTF-8?q?images:=20ask=20the=20judge=20before=20ca?= =?UTF-8?q?lling=20a=20page=20refusal=20a=20defect,=20and=20=C2=A731?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit §28 is titled "The one cell in this document that said «defect», and closing it", and it is not closed. Re-taken 2026-10-03 at the versions §28 names -- render v0.67.0, go-images/jpeg2000 v0.13.2, one population at a time, with a v0.60.0 run of the same population as control -- four of its five cells reproduce and the fifth does not: §28 before §28 after re-measured pictures 754 755 755 ok direct 705 706 706 ok compared 650 651 651 ok exact 650 651 651 ok refused 1 0 1 NOT bulletinno38tasm.pdf is still refused, byte-identically at v0.60.0 and v0.67.0, so the seven releases §28 credits changed nothing about it. It is not §26's ceiling either. Page 1 names THREE pictures of 9449 by 13701 -- two JPEG 2000 greys and a JBIG2 soft mask, all three of which pdfimages takes out -- each 129 460 749 pixels. Two spend 258 921 498 of the 268 435 456 budget and leave 9 513 958, which is the number in the message; the third cannot be afforded. The charge is honest for the path: decodeBase says r.bounded is true exactly on the Images path, which hands pictures out through raster.Image and so spends four bytes a pixel. Three pictures of 129.5 Mpx is 1.447 GiB at four bytes against a 1.000 GiB page budget, and 0.362 GiB as the single-component greys they are. Tracked as go-pdfkit/render#100. Looking for the document turned up a defect in the instrument instead. Ours is documented as "ours would not draw it AND THE JUDGE WOULD", and it is set at two places: reader.Open failing, which goes through blame(), and render.Images failing on a page, which asked nobody. A page refusal went into the defect column whatever poppler did with the same page. Here it happens to be right, but by hand, not by instrument. judgePage now asks, of the same tool on the same page, the way the branch below it already does: pictures back is Ours, a tool that does not finish is Hung, and a refusal -- or a run that takes nothing out, which judgeShots reports the same way -- is Neither. The vocabulary now says that the last two are not told apart instead of implying a distinction the code cannot make. It changed an answer at once: TestAPageThatIsNotThereSaysSo asks for page nine of a one-page document and asserted Ours, so the instrument called a page that does not exist a defect of ours. Asked, poppler cannot give page nine either. Checked by mutation -- never asking the judge, and swapping Hung for Neither, each make the tests fail. 100.0% of statements in every package. And §30 is why this sat for nine days: §28's figures were taken in a tree with the modules upgraded by hand and never written to baseline/, so the only copy was prose, which cannot be re-run. The committed record said refused: 1 the whole time. The repository contained its own refutation. --- baseline/README.md | 140 ++++++++++++++++++++++++++++++++++++++++-- images/images.go | 38 ++++++++++-- images/images_test.go | 48 ++++++++++++++- 3 files changed, 216 insertions(+), 10 deletions(-) diff --git a/baseline/README.md b/baseline/README.md index 099713f..0fdb7e9 100644 --- a/baseline/README.md +++ b/baseline/README.md @@ -1964,12 +1964,23 @@ it is the only part where the answer is ground the references do not cover. **Measured before acting on it, and not acted on.** `Ours` is set at two places: a file that will not read, and a page that will not be got. It is the - `refused` column, and §28 shows that column is **zero in all twenty-three + `refused` column, and ~~§28 shows that column is **zero in all twenty-three populations** — a document ours refuses that poppler refuses too goes through `blame()` and comes out `Neither`, not `Ours`, which is why §27's sixty-six are not here. Splitting the label today would divide two empty buckets, and - the split changes a JSON field and every table that carries it. It is left - named, which is what this list is for. + the split changes a JSON field and every table that carries it.~~ **§31 + measures that column at 1, not 0, and the one document in it is the SECOND + kind** — `bulletinno38tasm.pdf`, a page that will not be got. So the two + buckets are not both empty and the reason given for not splitting them does + not hold. + + Worse, the two paths do not test the same thing. The `reader.Open` path calls + `blame()`, so a file poppler cannot open either comes out `Neither`. The + page path at `judgePage` sets `Missing: Ours` **without asking poppler at + all** — so for a page-level refusal the column's own definition, *"and the + judge would"*, is asserted rather than measured. Here it happens to be true: + `pdfimages -list` takes out all three pictures of that page (§31). It was not + the instrument that established that. - ~~**The three `DCTDecode` rows that did not move are not diagnosed.** `fr-impots` at a median peak of 255, `gh-pypdf` at 233 and `gh-qpdf` at 171 are not chroma reconstruction and are not rounding.~~ **All three are at a peak of @@ -2576,8 +2587,15 @@ bytes each — and `render` v0.66.0 fixed it. Re-measured against v0.67.0 and | agreement | 100.0% | 100.0% | | **refused** | **1** | **0** | -The extra picture is the one that was refused, and it comes out exact. **The -column is now zero in every population.** +~~The extra picture is the one that was refused, and it comes out exact. **The +column is now zero in every population.**~~ + +> **Both sentences are wrong, and §31 has the measurement.** Re-taken +> 2026-10-03 at the same versions, `refused` is **1**: `bulletinno38tasm.pdf` is +> still refused, byte-identically to v0.60.0, because its page names three +> 9 449 × 13 701 pictures and the `Images` path spends four bytes on every pixel +> of each. The extra picture is real and is a different document's. Four cells +> of this table reproduce; this one does not. `ia-americana` was re-measured too, since it holds the other two pages §26 names, and it is **unchanged in every column**. Its two pages were refused by @@ -2720,3 +2738,115 @@ afternoon. bound is a property of the machine, not of the corpus: two runs in parallel can turn a document into a `hung` entry that neither would produce alone. A count measured beside another copy of itself is not a count of the corpus. + +## §31 — §28 closed a cell that is still open, and §30 is why nobody saw + +§28 is titled *"The one cell in this document that said «defect», and closing +it"*. Its table reports `ia-biodiversity` re-measured against `render` v0.67.0 +and `go-images/jpeg2000` v0.13.2: + +| `ia-biodiversity` | §28's before | §28's after | **re-measured 2026-10-03** | +|---|---:|---:|---:| +| pictures | 754 | 755 | **755** ✓ | +| direct | 705 | 706 | **706** ✓ | +| compared | 650 | 651 | **651** ✓ | +| exact | 650 | 651 | **651** ✓ | +| **refused** | **1** | **0** | **1** ✗ | + +**Four cells of five reproduce and the fifth does not — and the fifth is the +one the section is about.** Taken with `images -only ia-biodiversity -json` +against render v0.67.0 and jpeg2000 v0.13.2, one at a time, beside a v0.60.0 run +of the same population for a control. The control reproduces §28's *before* +column exactly, which is what makes the *after* column's single disagreement +readable rather than arguable. + +### What is actually refused, and why it is not §26's ceiling + +The document is the one §28 names, `bulletinno38tasm.pdf`, and the refusal is +byte-identical at v0.60.0 and at v0.67.0 — so the seven releases §28 credits +changed nothing about it: + +``` +render: the page names more picture than may be decoded at once: +a picture of 9449 by 13701 pixels, with 9513958 of the 268435456 pixels left +``` + +That is not one picture being too big. **It is the third of three.** Page 1 +names three pictures of the same 9 449 × 13 701 — two JPEG 2000 greys and a +JBIG2 soft mask — and `pdfimages -list` takes out all three: + +``` +page num type width height color comp bpc enc object size + 1 0 image 9449 13701 gray 1 8 jpx 32 884K + 1 1 image 9449 13701 gray 1 8 jpx 34 167K + 1 2 smask 9449 13701 gray 1 1 jbig2 34 254K +``` + +Each is 129 460 749 pixels. Two of them are 258 921 498, and 268 435 456 − +258 921 498 = **9 513 958**, which is the number the message prints. The budget +is spent by the first two and the third cannot be afforded. + +**The cause is the hand-off, not the codec.** `afford` charges declared pixels, +and `decodeBase` says why that is honest: `r.bounded` is true exactly on the +`Images` path, which hands pictures out through `raster.Image` and therefore +**expands every one of them to four bytes a pixel**. Three pictures of 129.5 +megapixels is **1.447 GiB** at four bytes, against a page budget of 268 435 456 +pixels — exactly 1.000 GiB. Handed out as the single-component greys they are, +the same page is **0.362 GiB** and well inside it. + +So this is the mirror image of §26. §26 was a ceiling that *refused* pages by +counting pixels at four bytes each when the decoder no longer spent that; this +is a page refused because the `Images` path really does spend it, on a corpus +whose pictures are all one component. §25 took the decoder from 19.8 to 8.6 +bytes a pixel and this last factor of four is downstream of it, in the type the +pictures come back in. + +Tracked as [go-pdfkit/render#100](https://github.com/go-pdfkit/render/issues/100). +**The `refused` column is not zero, and this document should stop saying it is.** + +### Why it took nine days + +§30 is the answer. §28's figures were taken in a tree with the modules upgraded +by hand and were never written to `baseline/`, so the only copy of them was +prose — and prose cannot be re-run. The committed record still said `refused: 1` +the whole time, which is to say **the repository contained the refutation +already** and nothing compared the two halves of its own document. + +That is the argument for re-taking the records rather than for writing another +section. + +### And the column was asserting half of its own definition + +Looking for the document turned up a defect in the instrument rather than in +the renderer. `Ours` is documented as *"ours would not open the document or +draw the page **and the judge would**. That is a defect and the only one of +these that is"* — and `Ours` is set at two places: + +| where | asked the judge? | +|---|---| +| `reader.Open` fails | yes — `blame()`, which runs `pdfinfo` | +| `render.Images` fails on a page | **no** | + +So a page refusal went into the defect column whatever poppler did with the +same page. Here it happens to be right — `pdfimages` takes out all three +pictures — but it was not the instrument that established that; it was this +section, by hand. + +`judgePage` now asks, of the same tool on the same page, the way the branch +below it already asks when the judge takes nothing out of a page ours drew for: + +| what comes back | column | +|---|---| +| the judge hands pictures back | `Ours` — a defect, and `bulletinno38tasm.pdf` is the corpus's only one | +| the judge does not finish | `Hung`, naming `pdfimages` | +| the judge refuses, or runs and takes nothing out | `Neither` | + +The last row folds two cases that `judgeShots` does not tell apart — it reports +"no pictures came out" as an error — and the vocabulary now says so instead of +implying a distinction the code cannot make. + +**It changed an answer immediately.** `TestAPageThatIsNotThereSaysSo` asks for +page nine of a one-page document and asserted `Ours`: the instrument called a +page that does not exist a defect of ours. Asked, poppler cannot give page nine +either, and it is now `Neither`. Checked by mutation — never asking the judge, +and swapping `Hung` for `Neither`, each make the tests fail. diff --git a/images/images.go b/images/images.go index 26caf00..467a819 100644 --- a/images/images.go +++ b/images/images.go @@ -325,8 +325,11 @@ const ( // judge would. That is a defect and the only one of these that is: a // document the field can read and we cannot. Ours Missing = "ours" - // Neither means no implementation would open it — ours refused, and so - // did the judge, asked separately about the same file. + // Neither means no implementation produced a picture — ours refused, and + // so did the judge, asked separately about the same file, or for a page, + // about the same page. A judge that answers and takes nothing out of a + // page ours refused is here too: judgeShots reports that as an error, and + // the two are not told apart. // // This has to be told apart from Ours or the count misleads in the // direction of comfort in one direction and panic in the other. Seven of @@ -414,8 +417,35 @@ func Judge(path string, opt Options) []Result { func judgePage(d *reader.Document, path string, p int) []Result { ours, err := render.Images(d, p) if err != nil { - return []Result{{Path: path, Page: p, Difference: unjudged(), - Missing: Ours, Note: "no page: " + err.Error()}} + // Ours says "ours would not draw it AND THE JUDGE WOULD. That is a + // defect and the only one of these that is." This path used to assert + // the second half rather than ask it: the open path above goes through + // blame(), and a page refusal went straight into the defect column + // whatever poppler did with the same page. + // + // The question here is narrower than blame()'s -- not whether poppler + // opens the FILE but whether it gets this PAGE -- so it is asked the + // way the branch below asks it, of the same tool on the same page. + // + // judgeShots reports "no pictures came out" as an ERROR, so a judge + // that refuses the page and a judge that answers and takes nothing out + // of it arrive here the same way. Both become Neither, which is what + // that word means for a page: no implementation produced a picture for + // it. It is also symmetric with the branch below, where the judge + // taking nothing out of a page OURS drew for is Theirs rather than a + // disagreement. + r := Result{Path: path, Page: p, Difference: unjudged(), + Missing: Ours, Note: "no page: " + err.Error()} + if _, tool, jerr := judgeShots(path, p); jerr != nil { + if tool != "" { + r.Missing, r.Tool = Hung, tool + r.Note += "; " + tool + " hung, so whose refusal this is is not known" + } else { + r.Missing = Neither + r.Note += "; and the judge drew nothing for it either: " + jerr.Error() + } + } + return []Result{r} } if len(ours) == 0 { return nil diff --git a/images/images_test.go b/images/images_test.go index 154a45d..1740470 100644 --- a/images/images_test.go +++ b/images/images_test.go @@ -827,9 +827,55 @@ func TestAPageThatIsNotThereSaysSo(t *testing.T) { if len(got) != 1 || got[0].Share != -1 || !strings.Contains(got[0].Note, "no page") { t.Fatalf("page nine of a one-page document came back as %+v", got) } - if got[0].Missing != Ours { + // NEITHER, not ours. This said Ours while the branch asserted the judge + // would have drawn it; asked, poppler cannot give page nine of a one-page + // document either, and a page that is not there is not a defect of ours. + if got[0].Missing != Neither { t.Errorf("blamed %q", got[0].Missing) } + if !strings.Contains(got[0].Note, "the judge drew nothing for it either") { + t.Errorf("the note does not say the judge was asked: %q", got[0].Note) + } +} + +func TestAPageRefusalIsNotADefectUntilTheJudgeIsAsked(t *testing.T) { + // The three ways the question can come back, over the same refusal, since + // the column that carries the answer is the one this document calls the + // count that is a defect. + path := pageOfPictures(t, func(w *reader.Writer) reader.Dict { + return reader.Dict{"I": grey(w)} + }) + b, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + d, err := reader.Open(b) + if err != nil { + t.Fatal(err) + } + was := popplerCommand + t.Cleanup(func() { popplerCommand = was }) + + popplerCommand = func(...string) (bool, error) { return true, os.ErrDeadlineExceeded } + got := judgePage(d, path, 9) + if len(got) != 1 || got[0].Missing != Hung || got[0].Tool != "pdfimages" { + t.Errorf("a judge that did not finish came back as %+v", got) + } + + // A judge that runs and takes nothing out is NOT a defect of ours, and + // judgeShots reports that as an error, so it arrives as Neither like a + // refusal does. The two are not told apart and the vocabulary says so. + popplerCommand = func(...string) (bool, error) { return false, nil } + got = judgePage(d, path, 9) + if len(got) != 1 || got[0].Missing != Neither { + t.Errorf("a judge that took nothing out came back as %+v", got) + } + // What is left of Ours is the case that matters: ours refuses and the + // judge hands pictures back. That is the one bulletinno38tasm.pdf is in, + // and the whole corpus's only instance of it -- see baseline §31. + if got[0].Tool != "" { + t.Errorf("a judge that did not hang named a tool: %q", got[0].Tool) + } } func TestTheRealJudgeIsTheOneAskedWhoRefused(t *testing.T) { From 2a02ddc62234cac36205e5c2f25f07d5197bf968 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Sat, 3 Oct 2026 23:36:47 +0200 Subject: [PATCH 06/28] baseline: check the refusal fix against the corpus, not only its tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ia-biodiversity re-taken a third time with the asking in place: refused is still 1, unopenable and declined still 0, every filter row identical to the run before it. A classification that quietly moved a real defect into Neither would be indistinguishable from a fix when read off the test suite alone; asked of the document it was written for, it leaves it where it belongs, because pdfimages does hand three pictures back. Says explicitly that none of the three runs is committed to baseline/: they were taken at render v0.67.0 while go.mod holds v0.60.0, which is what §30 is about. --- baseline/README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/baseline/README.md b/baseline/README.md index 0fdb7e9..e39e9fa 100644 --- a/baseline/README.md +++ b/baseline/README.md @@ -2850,3 +2850,16 @@ page nine of a one-page document and asserted `Ours`: the instrument called a page that does not exist a defect of ours. Asked, poppler cannot give page nine either, and it is now `Neither`. Checked by mutation — never asking the judge, and swapping `Hung` for `Neither`, each make the tests fail. + +**The fix was then checked against the corpus, not only against its tests.** +`ia-biodiversity` re-taken a third time, with the asking in place: `refused` +is still **1**, `unopenable` and `declined` still 0, and every filter row +identical to the run before it. A classification that quietly moved a real +defect into `Neither` would look exactly like a fix from the test suite alone; +asked of the document it was written for, it leaves it where it belongs, +because `pdfimages` does hand three pictures back. + +None of these three runs is committed to `baseline/`. They were taken at render +v0.67.0 while `go.mod` holds v0.60.0, and a record whose modules disagree with +the tree that reads it is the whole subject of §30. The bump is open as +[conformance#55](https://github.com/go-pdfkit/conformance/pull/55). From 6dbb82e72062916027379dde44ef54db392f3d13 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Sun, 4 Oct 2026 10:48:40 +0200 Subject: [PATCH 07/28] images: the page draws, only the extraction refuses -- and the census is one MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A correction to §31, found the same day, by asking the renderer the question the column's own words name. afford() and r.budget are reached from images.go ALONE, and bounded:true is set at exactly one place in the whole module -- inside Images. So: Images REFUSED: ... with 9513958 of the 268435456 pixels left Page ok: 411x596, 23659 dark pixels, 216735 light, 4562 between Not a blank page with a clean exit either. Through this repository's own compare, against pdftoppm at 72 dpi: 7.30% of pixels differ, mean 14.875 levels, and we take 795ms against poppler's 2.992s. A 9449x13701 scan reduced to 411x596 is a 23x downscale, which is the resampling disagreement §1 describes and the reason this baseline measures codecs through images rather than through a rasteriser. So Ours, whose comment said "would not open the document or draw the page", is here NEITHER: ours draws the page. The comment now says what the column measures -- no picture came out of the extraction -- and names the instance, because a vocabulary that overstates is how §28 came to be written. render#100 drops from a gap against poppler to a limit of one API, and says so. Then the census, so that the limit is sized rather than guessed. Every document of both corpora, page 1, our side only, never poppler: 3281 documents 67 our side produced nothing for 66 of them refused at reader.Open 1 of them refused by the page budget One document in 3 281. And the sixty-six are §27's sixty-six, recovered by an instrument that asks neither build nor poppler: 39 unsupported security handler, 23 wrong password, 2 no indirect objects, 1 /Brotli, 1 no startxref, in the same populations (ia-americana 30, gh-openpdf 14, gh-pdfbox 8, ia-texts 7, gh-safedocs 5, gh-pypdf 1, ia-medical 1). Two instruments that share no dependency agreeing on a census is a control this file has been short of -- three agreed once before and all three read the same layer. --- baseline/README.md | 91 ++++++++++++++++++++++++++++++++++++++++++++++ images/images.go | 16 ++++++-- 2 files changed, 104 insertions(+), 3 deletions(-) diff --git a/baseline/README.md b/baseline/README.md index e39e9fa..ac91c62 100644 --- a/baseline/README.md +++ b/baseline/README.md @@ -2015,6 +2015,18 @@ it is the only part where the answer is ground the references do not cover. and the same sample came back with **every** picture paired by size (7 of 7 `DCTDecode`, 16 of 16 `JPXDecode`, and so on down the rows). The counter is live; the zero is a property of the corpus. +- **`refused` does not ask the renderer, and now that the difference is known + it could.** §31 found that the corpus's one entry in that column is a document + `render.Page` draws and `render.Images` refuses. Asking `Page` as well, on the + few documents that reach this branch, would let the column separate "no + picture came out of the extraction" from "the field can read this and we + cannot" — which is what the column was described as meaning. + + **Measured, named, not acted on**, because it is a question about what the + column should MEAN and not a defect: a third outcome changes a JSON field and + every table that carries it, and whether an extraction harness should hold an + opinion about the renderer is not ours to settle in a commit. The cost is one + `render.Page` per refusal, and refusals are one document in 3 281. - **No aggregate bound is applied.** `mse` and `mean` are recorded in FFmpeg's and pdfium's units and bounded by nothing, because no bound has been measured for pictures that were *extracted* rather than rendered. Choosing one from @@ -2863,3 +2875,82 @@ None of these three runs is committed to `baseline/`. They were taken at render v0.67.0 while `go.mod` holds v0.60.0, and a record whose modules disagree with the tree that reads it is the whole subject of §30. The bump is open as [conformance#55](https://github.com/go-pdfkit/conformance/pull/55). + +### Correction, the same day: the page DRAWS. Only the extraction refuses. + +The two paragraphs above say the refusal is in the hand-off to `raster.Image`, +and they are right about the arithmetic and wrong about the reach. `afford` and +`r.budget` are reached from **`images.go` only**, and `bounded: true` is set at +exactly one place in the whole module — inside `Images`. The page renderer +neither budgets this way nor refuses for this reason. + +Asked directly, at render v0.67.0: + +``` +Images REFUSED: ... with 9513958 of the 268435456 pixels left +Page ok: 411x596, 23659 dark pixels, 216735 light, 4562 between +``` + +It is not a blank page with a clean exit, either. Run through this repository's +own `compare` on a one-document corpus, against `pdftoppm` at 72 dpi: + +| `bulletinno38tasm.pdf` page 1 | | +|---|---| +| pixels differing | 7.30% | +| mean abs difference | 14.875 levels | +| worst square mean | 55 levels | +| ours | **795 ms** | +| poppler | 2.992 s — we are 0.266× of it | + +7.3% on a 9 449 × 13 701 scan reduced to 411 × 596 is a **23× downscale**, which +is the resampling disagreement §1 describes and the reason this baseline measures +codecs through `images` rather than through a rasteriser at all. It is not a +reading of the page being wrong; it is the reason the page is not the unit. + +**So `Ours`, whose comment says *"would not open the document or draw the page"*, +is here neither.** Ours draws the page. What refuses is the extraction API the +harness uses, under a budget the renderer does not share — and the one non-zero +cell in the corpus's defect column is that, not a document the field can read +and we cannot. `images.go` now says so where `Ours` is defined. + +That lowers [render#100](https://github.com/go-pdfkit/render/issues/100) from a +gap against poppler to a limit of one API: no caller of `Page` is affected, and +the issue says so now. **The measurement that found it was the renderer being +asked the question the column's own words name**, which nothing in the harness +had ever done — it only ever calls `Images`. + +### How many pages the budget refuses, over both corpora + +Before deciding what the budget is worth fixing, it was worth asking how often +it fires. Every document of both corpora, page 1, at render v0.67.0, asking our +side only — `reader.Open`, `PageCount`, `render.Images` — and never poppler, so +this is a census and not a comparison: + +| | | +|---|---:| +| documents | **3 281** | +| our side produced nothing for page 1 | **67** | +| of those, `reader.Open` refused the file | **66** | +| of those, the page budget refused | **1** | + +**One document in 3 281.** `bulletinno38tasm.pdf` and nothing else, which is +what makes [render#100](https://github.com/go-pdfkit/render/issues/100) a limit +worth recording rather than a campaign. + +And the sixty-six are §27's sixty-six, **recovered by a different instrument**: + +| count | `reader.Open` says | §27/§28's table | +|---:|---|---:| +| 39 | `unsupported security handler` | 39 | +| 23 | `the password does not open this file` | 23 | +| 2 | no indirect objects, and no `startxref` either | 2 | +| 1 | no document catalogue, and an unsupported `/Brotli` filter | 1 | +| 1 | no document catalogue, and no `startxref` | 1 | + +Their populations agree too — `ia-americana` 30, `gh-openpdf` 14, `gh-pdfbox` 8, +`ia-texts` 7, `gh-safedocs` 5, `gh-pypdf` 1, `ia-medical` 1. §27's figures came +out of a sweep that compares two builds and asks poppler separately; these came +out of a program that asks neither. **Two instruments sharing no dependency +agreeing on a census of sixty-six is the kind of control this file has been +short of** — three of them agreed once before and all three were reading the +same layer. diff --git a/images/images.go b/images/images.go index 467a819..ddaa661 100644 --- a/images/images.go +++ b/images/images.go @@ -321,9 +321,19 @@ type Missing string const ( // Judged means there was a picture on both sides and they were compared. Judged Missing = "" - // Ours means ours would not open the document or draw the page and the - // judge would. That is a defect and the only one of these that is: a - // document the field can read and we cannot. + // Ours means ours produced no picture for a page the judge did produce + // pictures for. That is the only one of these that can be a defect. + // + // It is NOT "would not draw the page", which this said for months. The + // two are different questions and the corpus has an instance of the + // difference: bulletinno38tasm.pdf is counted here, and render.Page draws + // it. What refuses it is render.Images, under a per-page pixel budget that + // the page renderer does not share -- afford() is reached from images.go + // alone. See baseline/README.md §31 and go-pdfkit/render#100. + // + // So a document in this column is worth reading and is not by itself a + // document the field can read and we cannot. Asking the renderer is a + // separate step, and this harness never takes it: it only calls Images. Ours Missing = "ours" // Neither means no implementation produced a picture — ours refused, and // so did the judge, asked separately about the same file, or for a page, From 472956cd55ae18c95bf041b935f180abbdfcf1b3 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Wed, 30 Sep 2026 18:03:06 +0200 Subject: [PATCH 08/28] images: count the pictures paired by size, and say so PairedBy's own comment states the requirement: A run whose size share is large is a run whose numbers are worth less, and that has to be visible. It was not visible anywhere. The pairing already records how each picture was matched -- by object number, or by falling back to size -- but Tally threw that away, so neither the report nor the JSON record carried it. A reader of either had no way to tell a run where every picture was matched by its object number from one where the matching was guesswork. Counts gains SizePaired, Tally increments it on PairedBySize only, the filter line ends with "N paired by size", and FilterCounts carries it into the record (omitempty, so existing records that have no such number stay as they are). The test asserts both directions. A counter that increments on every picture reads exactly like one that works, until the run that is all object-paired reports every picture as suspect -- so the JPXDecode row, which has no size-paired picture in it, must come out zero. Checked by mutation: counting unconditionally, dropping the report column, and dropping the record field each make it fail. --- images/images.go | 24 +++++++++++++++++++++--- images/images_test.go | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 3 deletions(-) diff --git a/images/images.go b/images/images.go index 5bd0484..26caf00 100644 --- a/images/images.go +++ b/images/images.go @@ -1157,6 +1157,18 @@ type Bucket struct { type Counts struct { // Pictures is how many were judged. Pictures int + // SizePaired is how many were matched to the judge's picture by SIZE + // rather than by object number. + // + // PairedBy's own note says a run whose size share is large is a run whose + // numbers are worth less, "and that has to be visible". It was not: the + // field was recorded per picture and no report or record carried it, so + // the share could not be read from a run at all. conformance#13 found 144 + // of 173 apparent inversions were this fallback pairing pictures that had + // nothing to do with each other, which is what makes the share worth + // printing beside the counts it qualifies rather than kept for whoever + // thinks to ask. + SizePaired int // Unmatched is how many the other implementation had no picture for. Unmatched int // Remapped is how many carried a /Decode array, which we apply and @@ -1199,6 +1211,9 @@ func Tally(rs []Result) map[string]*Counts { by[key] = c } c.Pictures++ + if r.PairedBy == PairedBySize { + c.SizePaired++ + } switch { case r.RawBits: c.RawBits++ @@ -1251,8 +1266,8 @@ func Report(by map[string]*Counts) string { var sb strings.Builder for _, key := range order(by) { c := by[key] - fmt.Fprintf(&sb, "%-22s %5d pictures %5d unmatched %5d remapped\n", - key, c.Pictures, c.Unmatched, c.Remapped) + fmt.Fprintf(&sb, "%-22s %5d pictures %5d unmatched %5d remapped %5d paired by size\n", + key, c.Pictures, c.Unmatched, c.Remapped, c.SizePaired) reportBucket(&sb, "direct", &c.Direct) reportBucket(&sb, "converted", &c.Converted) } @@ -1337,6 +1352,9 @@ type FilterCounts struct { // Unmatched and Remapped are the pictures no comparison was made of. Unmatched int `json:"unmatched"` Remapped int `json:"remapped"` + // SizePaired is how many were paired by size rather than by object. A + // large share here devalues every figure on the line beside it. + SizePaired int `json:"sizePaired,omitempty"` // RawBits is the pictures the judge wrote as samples rather than as // colour, which is a third way of not being asked the same question. RawBits int `json:"rawBits,omitempty"` @@ -1401,7 +1419,7 @@ func Summarize(population string, documents int, rs []Result) Summary { by := Tally(rs) for _, key := range order(by) { c := by[key] - s.Filters = append(s.Filters, FilterCounts{Filter: key, + s.Filters = append(s.Filters, FilterCounts{Filter: key, SizePaired: c.SizePaired, Pictures: c.Pictures, Unmatched: c.Unmatched, Remapped: c.Remapped, RawBits: c.RawBits, Direct: bucketCounts(&c.Direct), Converted: bucketCounts(&c.Converted)}) diff --git a/images/images_test.go b/images/images_test.go index 8bff692..154a45d 100644 --- a/images/images_test.go +++ b/images/images_test.go @@ -1757,3 +1757,43 @@ func TestAPaletteInAStreamIsReadToo(t *testing.T) { t.Errorf("a black-and-white palette in a stream was counted apart: %v", got) } } + +func TestTheSizePairedShareIsCounted(t *testing.T) { + // PairedBy's own note says a run whose size share is large is a run whose + // numbers are worth less, "and that has to be visible". Until this counter + // it was in no report and no record. + // + // Both sides are asserted. A counter that increments on every picture reads + // exactly like one that works, until the run that is all object-paired says + // every picture is suspect. + rows := []Result{ + {Name: "a", Filter: "DCTDecode", PairedBy: PairedBySize}, + {Name: "b", Filter: "DCTDecode", PairedBy: PairedByObject}, + {Name: "c", Filter: "DCTDecode", PairedBy: PairedBySize}, + {Name: "d", Filter: "JPXDecode", PairedBy: PairedByObject}, + } + by := Tally(rows) + if got := by["DCTDecode"].SizePaired; got != 2 { + t.Errorf("DCTDecode counted %d paired by size, want 2", got) + } + if got := by["JPXDecode"].SizePaired; got != 0 { + t.Errorf("JPXDecode counted %d paired by size, want 0 -- the counter "+ + "fires on pictures it should not", got) + } + // And it reaches the two places anyone reads: the report and the record. + if rep := Report(by); !strings.Contains(rep, "2 paired by size") { + t.Errorf("the report does not carry the share:\n%s", rep) + } + var found bool + for _, f := range Summarize("p", 1, rows).Filters { + if f.Filter == "DCTDecode" { + found = true + if f.SizePaired != 2 { + t.Errorf("the record says %d paired by size, want 2", f.SizePaired) + } + } + } + if !found { + t.Error("the record has no DCTDecode line to carry it") + } +} From ab95894b22c0138853d656de560d3c16c5e0bad7 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Thu, 1 Oct 2026 15:31:30 +0200 Subject: [PATCH 09/28] baseline: answer the pairing-audit entry, and date the records MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two entries in "What is not measured, and why" are addressed, one of them by closing it and one by naming what blocks it. The pairing entry asked whether a per-picture pairing audit had been run since conformance#13 closed. It is now counted in every run rather than audited once, and the first reading is ZERO over 1 606 pictures -- 330 across the five pdfscans populations, 1 276 across all eighteen of pdfforms, 53 filter rows, none paired by size. A zero needs a control, because a counter that never fires reads exactly like a fallback that never fires. Ablating the object test -- object forced to 0 inside match, the only input its first loop reads -- pairs EVERY picture of the same sample by size. The counter is live and the zero belongs to the corpus. §30 is the other half, and it is the uncomfortable one. §29 says this file must be read backwards because a section is true of the run that wrote it; that applies to the records too, and there a strikethrough will not do. All twenty-three are dated 2026-09-24 at render v0.35.0 and jpeg2000 v0.1.0, while go.mod holds v0.60.0/v0.9.1 and §29 measures v0.67.0/v0.13.2 -- four subjects in one document, with the machine-readable half the oldest of them. Nineteen jpeg2000 releases have landed since the version the records name, including the decoder of §25 and the three blank pages of §26. That is what blocks the aggregate bound. The twenty-one buckets carrying terms do invite one -- mse.worst 0.654, |mean.worst| 0.579, peak.worst 4 everywhere but gh-openpdf's converted bucket at 20 -- but a bound fitted to them is fitted to v0.35.0, and a gate borrowed from code that no longer exists reddens on the first honest change and is then deleted. The next step is the records, not the bound. No gate is added. A test asserting the records name go.mod's version would redden every dependency bump; the move here is §28's and conformance#92's -- make the number visible and let the reader see that a record is six days and thirty-two releases behind what it is cited for. --- baseline/README.md | 76 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 74 insertions(+), 2 deletions(-) diff --git a/baseline/README.md b/baseline/README.md index b2a9f03..145b2d8 100644 --- a/baseline/README.md +++ b/baseline/README.md @@ -1979,9 +1979,31 @@ it is the only part where the answer is ground the references do not cover. see that an entry here can outlive its subject. [conformance#13](https://github.com/go-pdfkit/conformance/issues/13), cited beside it, is **closed**: the disagreements it found were this repository's - size matcher, and `match` now pairs by OBJECT first. Whether a per-picture + size matcher, and `match` now pairs by OBJECT first. ~~Whether a per-picture pairing audit has been run since is still not recorded, and that part of the - entry stands. + entry stands.~~ **It is recorded now, in every run.** + + `PairedBy` already said what follows from itself -- *"a run whose size share + is large is a run whose numbers are worth less, and that has to be visible"* + -- and `Tally` dropped it, so the share reached no report line and no record. + It is counted as of + [conformance#92](https://github.com/go-pdfkit/conformance/pull/92): + `SizePaired` on the filter line and in the JSON. An audit is a one-off that + ages; a counter does not. + + **First reading: ZERO, over 1 606 pictures.** Measured 2026-10-01 at the + `go.mod` of this run (`render` v0.60.0, `go-images/jpeg2000` v0.9.1): 330 + pictures over the five `pdfscans` populations (`-limit 40`) and 1 276 over all + eighteen of `pdfforms` (`-limit 60`), 53 filter rows, not one paired by size. + Every picture both sides took out was matched by the one identity both + publish. + + **A zero needs a control, because a counter that never fires reads exactly + like a fallback that never fires.** The object test was ablated -- `object` + forced to 0 inside `match`, which is the only input the first loop reads -- + and the same sample came back with **every** picture paired by size (7 of 7 + `DCTDecode`, 16 of 16 `JPXDecode`, and so on down the rows). The counter is + live; the zero is a property of the corpus. - **No aggregate bound is applied.** `mse` and `mean` are recorded in FFmpeg's and pdfium's units and bounded by nothing, because no bound has been measured for pictures that were *extracted* rather than rendered. Choosing one from @@ -2619,3 +2641,53 @@ forwards, as I did, produces a defect hunt for something fixed two months ago. The same hazard, in a smaller form, is in §27: a sweep's "drawn by neither" line counts the two builds it compares, not us and the judge. + +## §30 — §29's hazard, in the half of this document a program reads + +§29 says this file has to be read backwards, because a section is true of the +run that produced it. That is about the prose. The same thing has happened to +the **records**, and there it is worse: a paragraph can be struck through, a +`.json` either describes the code or it does not. + +Read off the files themselves, 2026-10-01: + +| what | render | `go-images/jpeg2000` | dated | +|---|---|---|---| +| the twenty-three `baseline/*.json` | **v0.35.0** | **v0.1.0** | 2026-09-24 | +| `go.mod`, which is what the tool in this repository builds against | v0.60.0 | v0.9.1 | — | +| §24, the timings | v0.60.0 | v0.9.1 | 2026-09-28 | +| §29, measured in a tree with the modules upgraded | **v0.67.0** | **v0.13.2** | 2026-09-30 | + +**The document has four subjects at once and the machine-readable half is the +oldest of them.** Every `taken` in `baseline/` is 2026-09-24T17:08Z..20:07Z, and +`go-images/jpeg2000` has had **nineteen** releases since the version those +records name (v0.2.0 through v0.13.2, counted off the tag list) — among them the whole 8.6-bytes-per-pixel decoder of §25 and the +three pages of §26 that came back blank and now draw. Nothing in the records +says they predate that work, because when they were written there was no later. + +### What this blocks, by name + +The entry above that reads *"No aggregate bound is applied … choosing one from +these records is a job for a later run, and the records carry the terms"* cannot +be done from these records. Their twenty-one buckets with `terms` put +`mse.worst` at 0.654 and `|mean.worst|` at 0.579, with `peak.worst` at 4 in +every bucket but one (`gh-openpdf DCTDecode converted`, at 20) -- a tight, +inviting distribution, and **a bound fitted to it would be fitted to v0.35.0.** +A gate borrowed from a measurement of code that no longer exists goes red on +the first legitimate change and is then deleted, which costs more than never +having had it. + +So the order is: land the `render` v0.67.0 bump that is open as +[conformance#55](https://github.com/go-pdfkit/conformance/pull/55), re-take the +twenty-three records at that `go.mod`, and read the distribution again. **The +bound is not the next step. The records are.** + +### Why this is not a gate + +The obvious fix -- a test asserting every record names the `render` version in +`go.mod` -- would redden CI on every dependency bump, including the ones that +change no pixel, and a gate that reddens on routine upgrades is a gate someone +removes. What is wanted is the move §28 and conformance#92 both make: make the +number **visible**, and let a reader see for themselves that a record is six +days and thirty-two releases behind the code it is cited for. This table is +that, until the records are re-taken. From febbcc5131538c6435add2ae9d843229b806bffe Mon Sep 17 00:00:00 2001 From: tannevaled Date: Sat, 3 Oct 2026 22:20:50 +0200 Subject: [PATCH 10/28] baseline: say how the twenty-three records are re-taken MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit §30 names the records as the next step, and nothing in the repository said how to take them. The top-level README shows one invocation of "images -json"; neither file said how the SET is taken, what it is named, or what it costs. So: one invocation per population, which is where the filenames in baseline/ come from, and omitting -only gives one record carrying every population of that corpus -- not the shape baseline/ holds. With the cost, measured one at a time on 2026-10-01: fr-cerfa (450 forms) about two minutes, ia-biodiversity (250 scans) about fifty. And with the reason to take them one at a time, which is §22's: the judge is held to a wall-clock bound, that bound is a property of the machine rather than of the corpus, and two runs beside each other can turn a document into a hung entry neither would produce alone. --- baseline/README.md | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/baseline/README.md b/baseline/README.md index 145b2d8..099713f 100644 --- a/baseline/README.md +++ b/baseline/README.md @@ -2671,8 +2671,9 @@ The entry above that reads *"No aggregate bound is applied … choosing one from these records is a job for a later run, and the records carry the terms"* cannot be done from these records. Their twenty-one buckets with `terms` put `mse.worst` at 0.654 and `|mean.worst|` at 0.579, with `peak.worst` at 4 in -every bucket but one (`gh-openpdf DCTDecode converted`, at 20) -- a tight, -inviting distribution, and **a bound fitted to it would be fitted to v0.35.0.** +every bucket but one (`gh-openpdf DCTDecode converted`, at 20, which §17 +already accounts for) -- a tight, inviting distribution, and **a bound fitted to +it would be fitted to v0.35.0.** A gate borrowed from a measurement of code that no longer exists goes red on the first legitimate change and is then deleted, which costs more than never having had it. @@ -2691,3 +2692,31 @@ removes. What is wanted is the move §28 and conformance#92 both make: make the number **visible**, and let a reader see for themselves that a record is six days and thirty-two releases behind the code it is cited for. This table is that, until the records are re-taken. + +### How the twenty-three are re-taken + +§30 says the records are the next step. The top-level README shows a single +invocation of `images -json`; neither file said how the **set** is taken, what +it is named, or what it costs. One invocation per population, named after the corpus directory and the +population, which is where the filenames in `baseline/` come from: + +```sh +for pop in ia-americana ia-biodiversity ia-medical ia-texts ia-uscourts; do + images -dir /Users/Shared/pdfscans -only "$pop" -json > "baseline/pdfscans-$pop.json" +done +``` + +and the eighteen of `/Users/Shared/pdfforms` the same way. Omitting `-only` +writes **one** record carrying every population of that corpus, which is not +the shape `baseline/` holds. + +**They are not interchangeable in cost.** Measured 2026-10-01 on an M4 Max, one +at a time: `fr-cerfa` (450 forms) takes about two minutes, `ia-biodiversity` +(250 scans) about **fifty**. The scans dominate, and the whole set is an +afternoon. + +**Take them one at a time.** The judge is held to a wall-clock bound +(`-timeout`, 2 minutes by default) and §22 is what it cost to learn that the +bound is a property of the machine, not of the corpus: two runs in parallel can +turn a document into a `hung` entry that neither would produce alone. A count +measured beside another copy of itself is not a count of the corpus. From d6dcfa0aaf1138816ba0f646e391a6195c7d26b3 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Sat, 3 Oct 2026 23:07:51 +0200 Subject: [PATCH 11/28] =?UTF-8?q?images:=20ask=20the=20judge=20before=20ca?= =?UTF-8?q?lling=20a=20page=20refusal=20a=20defect,=20and=20=C2=A731?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit §28 is titled "The one cell in this document that said «defect», and closing it", and it is not closed. Re-taken 2026-10-03 at the versions §28 names -- render v0.67.0, go-images/jpeg2000 v0.13.2, one population at a time, with a v0.60.0 run of the same population as control -- four of its five cells reproduce and the fifth does not: §28 before §28 after re-measured pictures 754 755 755 ok direct 705 706 706 ok compared 650 651 651 ok exact 650 651 651 ok refused 1 0 1 NOT bulletinno38tasm.pdf is still refused, byte-identically at v0.60.0 and v0.67.0, so the seven releases §28 credits changed nothing about it. It is not §26's ceiling either. Page 1 names THREE pictures of 9449 by 13701 -- two JPEG 2000 greys and a JBIG2 soft mask, all three of which pdfimages takes out -- each 129 460 749 pixels. Two spend 258 921 498 of the 268 435 456 budget and leave 9 513 958, which is the number in the message; the third cannot be afforded. The charge is honest for the path: decodeBase says r.bounded is true exactly on the Images path, which hands pictures out through raster.Image and so spends four bytes a pixel. Three pictures of 129.5 Mpx is 1.447 GiB at four bytes against a 1.000 GiB page budget, and 0.362 GiB as the single-component greys they are. Tracked as go-pdfkit/render#100. Looking for the document turned up a defect in the instrument instead. Ours is documented as "ours would not draw it AND THE JUDGE WOULD", and it is set at two places: reader.Open failing, which goes through blame(), and render.Images failing on a page, which asked nobody. A page refusal went into the defect column whatever poppler did with the same page. Here it happens to be right, but by hand, not by instrument. judgePage now asks, of the same tool on the same page, the way the branch below it already does: pictures back is Ours, a tool that does not finish is Hung, and a refusal -- or a run that takes nothing out, which judgeShots reports the same way -- is Neither. The vocabulary now says that the last two are not told apart instead of implying a distinction the code cannot make. It changed an answer at once: TestAPageThatIsNotThereSaysSo asks for page nine of a one-page document and asserted Ours, so the instrument called a page that does not exist a defect of ours. Asked, poppler cannot give page nine either. Checked by mutation -- never asking the judge, and swapping Hung for Neither, each make the tests fail. 100.0% of statements in every package. And §30 is why this sat for nine days: §28's figures were taken in a tree with the modules upgraded by hand and never written to baseline/, so the only copy was prose, which cannot be re-run. The committed record said refused: 1 the whole time. The repository contained its own refutation. --- baseline/README.md | 140 ++++++++++++++++++++++++++++++++++++++++-- images/images.go | 38 ++++++++++-- images/images_test.go | 48 ++++++++++++++- 3 files changed, 216 insertions(+), 10 deletions(-) diff --git a/baseline/README.md b/baseline/README.md index 099713f..0fdb7e9 100644 --- a/baseline/README.md +++ b/baseline/README.md @@ -1964,12 +1964,23 @@ it is the only part where the answer is ground the references do not cover. **Measured before acting on it, and not acted on.** `Ours` is set at two places: a file that will not read, and a page that will not be got. It is the - `refused` column, and §28 shows that column is **zero in all twenty-three + `refused` column, and ~~§28 shows that column is **zero in all twenty-three populations** — a document ours refuses that poppler refuses too goes through `blame()` and comes out `Neither`, not `Ours`, which is why §27's sixty-six are not here. Splitting the label today would divide two empty buckets, and - the split changes a JSON field and every table that carries it. It is left - named, which is what this list is for. + the split changes a JSON field and every table that carries it.~~ **§31 + measures that column at 1, not 0, and the one document in it is the SECOND + kind** — `bulletinno38tasm.pdf`, a page that will not be got. So the two + buckets are not both empty and the reason given for not splitting them does + not hold. + + Worse, the two paths do not test the same thing. The `reader.Open` path calls + `blame()`, so a file poppler cannot open either comes out `Neither`. The + page path at `judgePage` sets `Missing: Ours` **without asking poppler at + all** — so for a page-level refusal the column's own definition, *"and the + judge would"*, is asserted rather than measured. Here it happens to be true: + `pdfimages -list` takes out all three pictures of that page (§31). It was not + the instrument that established that. - ~~**The three `DCTDecode` rows that did not move are not diagnosed.** `fr-impots` at a median peak of 255, `gh-pypdf` at 233 and `gh-qpdf` at 171 are not chroma reconstruction and are not rounding.~~ **All three are at a peak of @@ -2576,8 +2587,15 @@ bytes each — and `render` v0.66.0 fixed it. Re-measured against v0.67.0 and | agreement | 100.0% | 100.0% | | **refused** | **1** | **0** | -The extra picture is the one that was refused, and it comes out exact. **The -column is now zero in every population.** +~~The extra picture is the one that was refused, and it comes out exact. **The +column is now zero in every population.**~~ + +> **Both sentences are wrong, and §31 has the measurement.** Re-taken +> 2026-10-03 at the same versions, `refused` is **1**: `bulletinno38tasm.pdf` is +> still refused, byte-identically to v0.60.0, because its page names three +> 9 449 × 13 701 pictures and the `Images` path spends four bytes on every pixel +> of each. The extra picture is real and is a different document's. Four cells +> of this table reproduce; this one does not. `ia-americana` was re-measured too, since it holds the other two pages §26 names, and it is **unchanged in every column**. Its two pages were refused by @@ -2720,3 +2738,115 @@ afternoon. bound is a property of the machine, not of the corpus: two runs in parallel can turn a document into a `hung` entry that neither would produce alone. A count measured beside another copy of itself is not a count of the corpus. + +## §31 — §28 closed a cell that is still open, and §30 is why nobody saw + +§28 is titled *"The one cell in this document that said «defect», and closing +it"*. Its table reports `ia-biodiversity` re-measured against `render` v0.67.0 +and `go-images/jpeg2000` v0.13.2: + +| `ia-biodiversity` | §28's before | §28's after | **re-measured 2026-10-03** | +|---|---:|---:|---:| +| pictures | 754 | 755 | **755** ✓ | +| direct | 705 | 706 | **706** ✓ | +| compared | 650 | 651 | **651** ✓ | +| exact | 650 | 651 | **651** ✓ | +| **refused** | **1** | **0** | **1** ✗ | + +**Four cells of five reproduce and the fifth does not — and the fifth is the +one the section is about.** Taken with `images -only ia-biodiversity -json` +against render v0.67.0 and jpeg2000 v0.13.2, one at a time, beside a v0.60.0 run +of the same population for a control. The control reproduces §28's *before* +column exactly, which is what makes the *after* column's single disagreement +readable rather than arguable. + +### What is actually refused, and why it is not §26's ceiling + +The document is the one §28 names, `bulletinno38tasm.pdf`, and the refusal is +byte-identical at v0.60.0 and at v0.67.0 — so the seven releases §28 credits +changed nothing about it: + +``` +render: the page names more picture than may be decoded at once: +a picture of 9449 by 13701 pixels, with 9513958 of the 268435456 pixels left +``` + +That is not one picture being too big. **It is the third of three.** Page 1 +names three pictures of the same 9 449 × 13 701 — two JPEG 2000 greys and a +JBIG2 soft mask — and `pdfimages -list` takes out all three: + +``` +page num type width height color comp bpc enc object size + 1 0 image 9449 13701 gray 1 8 jpx 32 884K + 1 1 image 9449 13701 gray 1 8 jpx 34 167K + 1 2 smask 9449 13701 gray 1 1 jbig2 34 254K +``` + +Each is 129 460 749 pixels. Two of them are 258 921 498, and 268 435 456 − +258 921 498 = **9 513 958**, which is the number the message prints. The budget +is spent by the first two and the third cannot be afforded. + +**The cause is the hand-off, not the codec.** `afford` charges declared pixels, +and `decodeBase` says why that is honest: `r.bounded` is true exactly on the +`Images` path, which hands pictures out through `raster.Image` and therefore +**expands every one of them to four bytes a pixel**. Three pictures of 129.5 +megapixels is **1.447 GiB** at four bytes, against a page budget of 268 435 456 +pixels — exactly 1.000 GiB. Handed out as the single-component greys they are, +the same page is **0.362 GiB** and well inside it. + +So this is the mirror image of §26. §26 was a ceiling that *refused* pages by +counting pixels at four bytes each when the decoder no longer spent that; this +is a page refused because the `Images` path really does spend it, on a corpus +whose pictures are all one component. §25 took the decoder from 19.8 to 8.6 +bytes a pixel and this last factor of four is downstream of it, in the type the +pictures come back in. + +Tracked as [go-pdfkit/render#100](https://github.com/go-pdfkit/render/issues/100). +**The `refused` column is not zero, and this document should stop saying it is.** + +### Why it took nine days + +§30 is the answer. §28's figures were taken in a tree with the modules upgraded +by hand and were never written to `baseline/`, so the only copy of them was +prose — and prose cannot be re-run. The committed record still said `refused: 1` +the whole time, which is to say **the repository contained the refutation +already** and nothing compared the two halves of its own document. + +That is the argument for re-taking the records rather than for writing another +section. + +### And the column was asserting half of its own definition + +Looking for the document turned up a defect in the instrument rather than in +the renderer. `Ours` is documented as *"ours would not open the document or +draw the page **and the judge would**. That is a defect and the only one of +these that is"* — and `Ours` is set at two places: + +| where | asked the judge? | +|---|---| +| `reader.Open` fails | yes — `blame()`, which runs `pdfinfo` | +| `render.Images` fails on a page | **no** | + +So a page refusal went into the defect column whatever poppler did with the +same page. Here it happens to be right — `pdfimages` takes out all three +pictures — but it was not the instrument that established that; it was this +section, by hand. + +`judgePage` now asks, of the same tool on the same page, the way the branch +below it already asks when the judge takes nothing out of a page ours drew for: + +| what comes back | column | +|---|---| +| the judge hands pictures back | `Ours` — a defect, and `bulletinno38tasm.pdf` is the corpus's only one | +| the judge does not finish | `Hung`, naming `pdfimages` | +| the judge refuses, or runs and takes nothing out | `Neither` | + +The last row folds two cases that `judgeShots` does not tell apart — it reports +"no pictures came out" as an error — and the vocabulary now says so instead of +implying a distinction the code cannot make. + +**It changed an answer immediately.** `TestAPageThatIsNotThereSaysSo` asks for +page nine of a one-page document and asserted `Ours`: the instrument called a +page that does not exist a defect of ours. Asked, poppler cannot give page nine +either, and it is now `Neither`. Checked by mutation — never asking the judge, +and swapping `Hung` for `Neither`, each make the tests fail. diff --git a/images/images.go b/images/images.go index 26caf00..467a819 100644 --- a/images/images.go +++ b/images/images.go @@ -325,8 +325,11 @@ const ( // judge would. That is a defect and the only one of these that is: a // document the field can read and we cannot. Ours Missing = "ours" - // Neither means no implementation would open it — ours refused, and so - // did the judge, asked separately about the same file. + // Neither means no implementation produced a picture — ours refused, and + // so did the judge, asked separately about the same file, or for a page, + // about the same page. A judge that answers and takes nothing out of a + // page ours refused is here too: judgeShots reports that as an error, and + // the two are not told apart. // // This has to be told apart from Ours or the count misleads in the // direction of comfort in one direction and panic in the other. Seven of @@ -414,8 +417,35 @@ func Judge(path string, opt Options) []Result { func judgePage(d *reader.Document, path string, p int) []Result { ours, err := render.Images(d, p) if err != nil { - return []Result{{Path: path, Page: p, Difference: unjudged(), - Missing: Ours, Note: "no page: " + err.Error()}} + // Ours says "ours would not draw it AND THE JUDGE WOULD. That is a + // defect and the only one of these that is." This path used to assert + // the second half rather than ask it: the open path above goes through + // blame(), and a page refusal went straight into the defect column + // whatever poppler did with the same page. + // + // The question here is narrower than blame()'s -- not whether poppler + // opens the FILE but whether it gets this PAGE -- so it is asked the + // way the branch below asks it, of the same tool on the same page. + // + // judgeShots reports "no pictures came out" as an ERROR, so a judge + // that refuses the page and a judge that answers and takes nothing out + // of it arrive here the same way. Both become Neither, which is what + // that word means for a page: no implementation produced a picture for + // it. It is also symmetric with the branch below, where the judge + // taking nothing out of a page OURS drew for is Theirs rather than a + // disagreement. + r := Result{Path: path, Page: p, Difference: unjudged(), + Missing: Ours, Note: "no page: " + err.Error()} + if _, tool, jerr := judgeShots(path, p); jerr != nil { + if tool != "" { + r.Missing, r.Tool = Hung, tool + r.Note += "; " + tool + " hung, so whose refusal this is is not known" + } else { + r.Missing = Neither + r.Note += "; and the judge drew nothing for it either: " + jerr.Error() + } + } + return []Result{r} } if len(ours) == 0 { return nil diff --git a/images/images_test.go b/images/images_test.go index 154a45d..1740470 100644 --- a/images/images_test.go +++ b/images/images_test.go @@ -827,9 +827,55 @@ func TestAPageThatIsNotThereSaysSo(t *testing.T) { if len(got) != 1 || got[0].Share != -1 || !strings.Contains(got[0].Note, "no page") { t.Fatalf("page nine of a one-page document came back as %+v", got) } - if got[0].Missing != Ours { + // NEITHER, not ours. This said Ours while the branch asserted the judge + // would have drawn it; asked, poppler cannot give page nine of a one-page + // document either, and a page that is not there is not a defect of ours. + if got[0].Missing != Neither { t.Errorf("blamed %q", got[0].Missing) } + if !strings.Contains(got[0].Note, "the judge drew nothing for it either") { + t.Errorf("the note does not say the judge was asked: %q", got[0].Note) + } +} + +func TestAPageRefusalIsNotADefectUntilTheJudgeIsAsked(t *testing.T) { + // The three ways the question can come back, over the same refusal, since + // the column that carries the answer is the one this document calls the + // count that is a defect. + path := pageOfPictures(t, func(w *reader.Writer) reader.Dict { + return reader.Dict{"I": grey(w)} + }) + b, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + d, err := reader.Open(b) + if err != nil { + t.Fatal(err) + } + was := popplerCommand + t.Cleanup(func() { popplerCommand = was }) + + popplerCommand = func(...string) (bool, error) { return true, os.ErrDeadlineExceeded } + got := judgePage(d, path, 9) + if len(got) != 1 || got[0].Missing != Hung || got[0].Tool != "pdfimages" { + t.Errorf("a judge that did not finish came back as %+v", got) + } + + // A judge that runs and takes nothing out is NOT a defect of ours, and + // judgeShots reports that as an error, so it arrives as Neither like a + // refusal does. The two are not told apart and the vocabulary says so. + popplerCommand = func(...string) (bool, error) { return false, nil } + got = judgePage(d, path, 9) + if len(got) != 1 || got[0].Missing != Neither { + t.Errorf("a judge that took nothing out came back as %+v", got) + } + // What is left of Ours is the case that matters: ours refuses and the + // judge hands pictures back. That is the one bulletinno38tasm.pdf is in, + // and the whole corpus's only instance of it -- see baseline §31. + if got[0].Tool != "" { + t.Errorf("a judge that did not hang named a tool: %q", got[0].Tool) + } } func TestTheRealJudgeIsTheOneAskedWhoRefused(t *testing.T) { From 6eaed6fdcfb4ccd8765a4f38bf09888e231bcac7 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Sat, 3 Oct 2026 23:36:47 +0200 Subject: [PATCH 12/28] baseline: check the refusal fix against the corpus, not only its tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ia-biodiversity re-taken a third time with the asking in place: refused is still 1, unopenable and declined still 0, every filter row identical to the run before it. A classification that quietly moved a real defect into Neither would be indistinguishable from a fix when read off the test suite alone; asked of the document it was written for, it leaves it where it belongs, because pdfimages does hand three pictures back. Says explicitly that none of the three runs is committed to baseline/: they were taken at render v0.67.0 while go.mod holds v0.60.0, which is what §30 is about. --- baseline/README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/baseline/README.md b/baseline/README.md index 0fdb7e9..e39e9fa 100644 --- a/baseline/README.md +++ b/baseline/README.md @@ -2850,3 +2850,16 @@ page nine of a one-page document and asserted `Ours`: the instrument called a page that does not exist a defect of ours. Asked, poppler cannot give page nine either, and it is now `Neither`. Checked by mutation — never asking the judge, and swapping `Hung` for `Neither`, each make the tests fail. + +**The fix was then checked against the corpus, not only against its tests.** +`ia-biodiversity` re-taken a third time, with the asking in place: `refused` +is still **1**, `unopenable` and `declined` still 0, and every filter row +identical to the run before it. A classification that quietly moved a real +defect into `Neither` would look exactly like a fix from the test suite alone; +asked of the document it was written for, it leaves it where it belongs, +because `pdfimages` does hand three pictures back. + +None of these three runs is committed to `baseline/`. They were taken at render +v0.67.0 while `go.mod` holds v0.60.0, and a record whose modules disagree with +the tree that reads it is the whole subject of §30. The bump is open as +[conformance#55](https://github.com/go-pdfkit/conformance/pull/55). From 4240adfb7cb692423d6a57e3ba3e794cd6a2954f Mon Sep 17 00:00:00 2001 From: tannevaled Date: Sun, 4 Oct 2026 10:48:40 +0200 Subject: [PATCH 13/28] images: the page draws, only the extraction refuses -- and the census is one MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A correction to §31, found the same day, by asking the renderer the question the column's own words name. afford() and r.budget are reached from images.go ALONE, and bounded:true is set at exactly one place in the whole module -- inside Images. So: Images REFUSED: ... with 9513958 of the 268435456 pixels left Page ok: 411x596, 23659 dark pixels, 216735 light, 4562 between Not a blank page with a clean exit either. Through this repository's own compare, against pdftoppm at 72 dpi: 7.30% of pixels differ, mean 14.875 levels, and we take 795ms against poppler's 2.992s. A 9449x13701 scan reduced to 411x596 is a 23x downscale, which is the resampling disagreement §1 describes and the reason this baseline measures codecs through images rather than through a rasteriser. So Ours, whose comment said "would not open the document or draw the page", is here NEITHER: ours draws the page. The comment now says what the column measures -- no picture came out of the extraction -- and names the instance, because a vocabulary that overstates is how §28 came to be written. render#100 drops from a gap against poppler to a limit of one API, and says so. Then the census, so that the limit is sized rather than guessed. Every document of both corpora, page 1, our side only, never poppler: 3281 documents 67 our side produced nothing for 66 of them refused at reader.Open 1 of them refused by the page budget One document in 3 281. And the sixty-six are §27's sixty-six, recovered by an instrument that asks neither build nor poppler: 39 unsupported security handler, 23 wrong password, 2 no indirect objects, 1 /Brotli, 1 no startxref, in the same populations (ia-americana 30, gh-openpdf 14, gh-pdfbox 8, ia-texts 7, gh-safedocs 5, gh-pypdf 1, ia-medical 1). Two instruments that share no dependency agreeing on a census is a control this file has been short of -- three agreed once before and all three read the same layer. --- baseline/README.md | 91 ++++++++++++++++++++++++++++++++++++++++++++++ images/images.go | 16 ++++++-- 2 files changed, 104 insertions(+), 3 deletions(-) diff --git a/baseline/README.md b/baseline/README.md index e39e9fa..ac91c62 100644 --- a/baseline/README.md +++ b/baseline/README.md @@ -2015,6 +2015,18 @@ it is the only part where the answer is ground the references do not cover. and the same sample came back with **every** picture paired by size (7 of 7 `DCTDecode`, 16 of 16 `JPXDecode`, and so on down the rows). The counter is live; the zero is a property of the corpus. +- **`refused` does not ask the renderer, and now that the difference is known + it could.** §31 found that the corpus's one entry in that column is a document + `render.Page` draws and `render.Images` refuses. Asking `Page` as well, on the + few documents that reach this branch, would let the column separate "no + picture came out of the extraction" from "the field can read this and we + cannot" — which is what the column was described as meaning. + + **Measured, named, not acted on**, because it is a question about what the + column should MEAN and not a defect: a third outcome changes a JSON field and + every table that carries it, and whether an extraction harness should hold an + opinion about the renderer is not ours to settle in a commit. The cost is one + `render.Page` per refusal, and refusals are one document in 3 281. - **No aggregate bound is applied.** `mse` and `mean` are recorded in FFmpeg's and pdfium's units and bounded by nothing, because no bound has been measured for pictures that were *extracted* rather than rendered. Choosing one from @@ -2863,3 +2875,82 @@ None of these three runs is committed to `baseline/`. They were taken at render v0.67.0 while `go.mod` holds v0.60.0, and a record whose modules disagree with the tree that reads it is the whole subject of §30. The bump is open as [conformance#55](https://github.com/go-pdfkit/conformance/pull/55). + +### Correction, the same day: the page DRAWS. Only the extraction refuses. + +The two paragraphs above say the refusal is in the hand-off to `raster.Image`, +and they are right about the arithmetic and wrong about the reach. `afford` and +`r.budget` are reached from **`images.go` only**, and `bounded: true` is set at +exactly one place in the whole module — inside `Images`. The page renderer +neither budgets this way nor refuses for this reason. + +Asked directly, at render v0.67.0: + +``` +Images REFUSED: ... with 9513958 of the 268435456 pixels left +Page ok: 411x596, 23659 dark pixels, 216735 light, 4562 between +``` + +It is not a blank page with a clean exit, either. Run through this repository's +own `compare` on a one-document corpus, against `pdftoppm` at 72 dpi: + +| `bulletinno38tasm.pdf` page 1 | | +|---|---| +| pixels differing | 7.30% | +| mean abs difference | 14.875 levels | +| worst square mean | 55 levels | +| ours | **795 ms** | +| poppler | 2.992 s — we are 0.266× of it | + +7.3% on a 9 449 × 13 701 scan reduced to 411 × 596 is a **23× downscale**, which +is the resampling disagreement §1 describes and the reason this baseline measures +codecs through `images` rather than through a rasteriser at all. It is not a +reading of the page being wrong; it is the reason the page is not the unit. + +**So `Ours`, whose comment says *"would not open the document or draw the page"*, +is here neither.** Ours draws the page. What refuses is the extraction API the +harness uses, under a budget the renderer does not share — and the one non-zero +cell in the corpus's defect column is that, not a document the field can read +and we cannot. `images.go` now says so where `Ours` is defined. + +That lowers [render#100](https://github.com/go-pdfkit/render/issues/100) from a +gap against poppler to a limit of one API: no caller of `Page` is affected, and +the issue says so now. **The measurement that found it was the renderer being +asked the question the column's own words name**, which nothing in the harness +had ever done — it only ever calls `Images`. + +### How many pages the budget refuses, over both corpora + +Before deciding what the budget is worth fixing, it was worth asking how often +it fires. Every document of both corpora, page 1, at render v0.67.0, asking our +side only — `reader.Open`, `PageCount`, `render.Images` — and never poppler, so +this is a census and not a comparison: + +| | | +|---|---:| +| documents | **3 281** | +| our side produced nothing for page 1 | **67** | +| of those, `reader.Open` refused the file | **66** | +| of those, the page budget refused | **1** | + +**One document in 3 281.** `bulletinno38tasm.pdf` and nothing else, which is +what makes [render#100](https://github.com/go-pdfkit/render/issues/100) a limit +worth recording rather than a campaign. + +And the sixty-six are §27's sixty-six, **recovered by a different instrument**: + +| count | `reader.Open` says | §27/§28's table | +|---:|---|---:| +| 39 | `unsupported security handler` | 39 | +| 23 | `the password does not open this file` | 23 | +| 2 | no indirect objects, and no `startxref` either | 2 | +| 1 | no document catalogue, and an unsupported `/Brotli` filter | 1 | +| 1 | no document catalogue, and no `startxref` | 1 | + +Their populations agree too — `ia-americana` 30, `gh-openpdf` 14, `gh-pdfbox` 8, +`ia-texts` 7, `gh-safedocs` 5, `gh-pypdf` 1, `ia-medical` 1. §27's figures came +out of a sweep that compares two builds and asks poppler separately; these came +out of a program that asks neither. **Two instruments sharing no dependency +agreeing on a census of sixty-six is the kind of control this file has been +short of** — three of them agreed once before and all three were reading the +same layer. diff --git a/images/images.go b/images/images.go index 467a819..ddaa661 100644 --- a/images/images.go +++ b/images/images.go @@ -321,9 +321,19 @@ type Missing string const ( // Judged means there was a picture on both sides and they were compared. Judged Missing = "" - // Ours means ours would not open the document or draw the page and the - // judge would. That is a defect and the only one of these that is: a - // document the field can read and we cannot. + // Ours means ours produced no picture for a page the judge did produce + // pictures for. That is the only one of these that can be a defect. + // + // It is NOT "would not draw the page", which this said for months. The + // two are different questions and the corpus has an instance of the + // difference: bulletinno38tasm.pdf is counted here, and render.Page draws + // it. What refuses it is render.Images, under a per-page pixel budget that + // the page renderer does not share -- afford() is reached from images.go + // alone. See baseline/README.md §31 and go-pdfkit/render#100. + // + // So a document in this column is worth reading and is not by itself a + // document the field can read and we cannot. Asking the renderer is a + // separate step, and this harness never takes it: it only calls Images. Ours Missing = "ours" // Neither means no implementation produced a picture — ours refused, and // so did the judge, asked separately about the same file, or for a page, From 9c560bb86237f602a70e7c087f1bd9b2637312bf Mon Sep 17 00:00:00 2001 From: tannevaled Date: Sun, 4 Oct 2026 11:13:32 +0200 Subject: [PATCH 14/28] images: count the pictures the judge got out and we did not MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pairing walks OUR pictures and appends one result for each, so a row of the judge's that nothing claimed left no trace in any count. The one direction this harness could not see was the direction that matters most: a picture the field gets out of a file and we do not. Its own test suite had already set the case up and recorded the blind spot as the expected answer. TestAPictureTheOtherSideDoesNotHave stands in a judge with a 9x9 against a page holding a 2x1: ours is unmatched AND theirs is, the first has been reported since the day it was written, the second was dropped, and the test asserted len == 1. So: Missing.Unseen, one result per unclaimed row with Name left empty so Tally skips it -- there is no filter of OURS to group it under, because there is no picture of ours -- Summary.Unseen (omitempty), and a report line, for the reason the hangs have one: a report that leaves it out cannot be told from one that has none. It is known to be non-empty by construction for at least one class. render.Images does not return inline images, and says why: they are the objects of nothing. pdfimages lists them with an object of 0. Whether this corpus holds any is what the count is for. Two mutations, and the SECOND needed a test written for it: dropping the loop fails TestAPictureTheOtherSideDoesNotHave, but counting the CLAIMED rows as well passed everything, because no test had a judge with one row paired and one not. TestOnlyTheJudgePicturesNothingClaimedAreCounted is that case, and it is the same hole SizePaired was guarded against -- a counter that fires on everything reads exactly like one that works. 100.0% of statements in every package. §31's closing note is amended: #55 landed on 2026-10-03, so a record taken at v0.67.0 now agrees with the tree that reads it. --- baseline/README.md | 22 ++++++++++++------ cmd/images/run.go | 7 ++++++ cmd/images/run_test.go | 21 +++++++++++++++++ images/images.go | 34 ++++++++++++++++++++++++++++ images/images_test.go | 51 ++++++++++++++++++++++++++++++++++++++++-- 5 files changed, 126 insertions(+), 9 deletions(-) diff --git a/baseline/README.md b/baseline/README.md index ac91c62..1cd4fbc 100644 --- a/baseline/README.md +++ b/baseline/README.md @@ -2003,7 +2003,9 @@ it is the only part where the answer is ground the references do not cover. ages; a counter does not. **First reading: ZERO, over 1 606 pictures.** Measured 2026-10-01 at the - `go.mod` of this run (`render` v0.60.0, `go-images/jpeg2000` v0.9.1): 330 + `go.mod` of this run (`render` v0.60.0, `go-images/jpeg2000` v0.9.1 -- what + it held before #55; the count is a pairing property and does not depend on + either version): 330 pictures over the five `pdfscans` populations (`-limit 40`) and 1 276 over all eighteen of `pdfforms` (`-limit 60`), 53 filter rows, not one paired by size. Every picture both sides took out was matched by the one identity both @@ -2054,7 +2056,7 @@ instrument that produced them. They can now. |---|---| | taken | 2026-09-28T14:49Z .. 2026-09-28T15:16Z (UTC) *(first: 2026-09-26T16:59Z)* | | judge | pdftoppm version 26.04.0 | -| `go-pdfkit/render` | **v0.60.0**, which is what `go.mod` holds *(first taken at v0.47.0; nine releases later, each re-measured -- see the end of §24)* | +| `go-pdfkit/render` | **v0.60.0**, which is what `go.mod` held when this was taken -- [#55](https://github.com/go-pdfkit/conformance/pull/55) has since moved it to **v0.67.0**, and these timings have NOT been re-taken against it *(first taken at v0.47.0; nine releases later, each re-measured -- see the end of §24)* | | `go-images/jpeg2000` | **v0.9.1** | | **cores** | **we use up to eight on a JPEG 2000 page; the judge uses one.** See *What this section does not say*. | | every named row | **drawn again three times** and the minimum kept (`compare -confirm`), because a list selected on the maximum of one sample per page is biased upward by the act of selecting | @@ -2684,7 +2686,7 @@ Read off the files themselves, 2026-10-01: | what | render | `go-images/jpeg2000` | dated | |---|---|---|---| | the twenty-three `baseline/*.json` | **v0.35.0** | **v0.1.0** | 2026-09-24 | -| `go.mod`, which is what the tool in this repository builds against | v0.60.0 | v0.9.1 | — | +| `go.mod`, which is what the tool in this repository builds against | ~~v0.60.0~~ **v0.67.0** | ~~v0.9.1~~ **v0.13.2** | since #55, 2026-10-03 | | §24, the timings | v0.60.0 | v0.9.1 | 2026-09-28 | | §29, measured in a tree with the modules upgraded | **v0.67.0** | **v0.13.2** | 2026-09-30 | @@ -2708,10 +2710,16 @@ A gate borrowed from a measurement of code that no longer exists goes red on the first legitimate change and is then deleted, which costs more than never having had it. -So the order is: land the `render` v0.67.0 bump that is open as -[conformance#55](https://github.com/go-pdfkit/conformance/pull/55), re-take the -twenty-three records at that `go.mod`, and read the distribution again. **The -bound is not the next step. The records are.** +So the order was: land the `render` v0.67.0 bump, re-take the twenty-three +records at that `go.mod`, and read the distribution again. **The bound is not the +next step. The records are.** + +**[#55](https://github.com/go-pdfkit/conformance/pull/55) landed on +2026-10-03**, so the first step is done and the table above has been amended -- +which is this section happening to itself, four days after it was written. What +is left of §30 is the row it exists for: the twenty-three records are still at +v0.35.0, and now they are **thirty-two** `render` releases behind the tree that +reads them rather than twenty-five. ### Why this is not a gate diff --git a/cmd/images/run.go b/cmd/images/run.go index c49e282..dde3d2c 100644 --- a/cmd/images/run.go +++ b/cmd/images/run.go @@ -129,6 +129,13 @@ func run(args []string, out, errOut io.Writer) int { for _, h := range summary.Hung { fmt.Fprintf(out, " hung %s %s page %d\n", h.Tool, h.Path, h.Page) } + // And the pictures the judge took out that we produced nothing for, + // by the same argument: the one direction the pairing cannot see is + // the one that says the field gets something out of a file and we do + // not. + if summary.Unseen > 0 { + fmt.Fprintf(out, " %5d the judge took out and we did not\n", summary.Unseen) + } } if *asJSON { // Marshal fails only on a value it cannot encode, and this one is diff --git a/cmd/images/run_test.go b/cmd/images/run_test.go index b3513ef..a3dd7d2 100644 --- a/cmd/images/run_test.go +++ b/cmd/images/run_test.go @@ -218,6 +218,7 @@ func TestWhatCouldNotBeComparedReachesTheRecord(t *testing.T) { images.Result{Difference: images.Difference{Share: -1}, Missing: images.Ours, Note: "refused: x"}, images.Result{Difference: images.Difference{Share: -1}, Missing: images.Neither, Note: "refused: y"}, images.Result{Difference: images.Difference{Share: -1}, Missing: images.Theirs, Note: "they took nothing out"}, + images.Result{Difference: images.Difference{Share: -1}, Missing: images.Unseen, Note: "the judge took out a image of 9x9 (row 0, object 0) we have no picture for"}, ) atTime(t, "2026-08-30T15:04:05Z") var out, errOut bytes.Buffer @@ -234,6 +235,9 @@ func TestWhatCouldNotBeComparedReachesTheRecord(t *testing.T) { t.Errorf("refused %d, unopenable %d, declined %d; want one of each", p.Refused, p.Unopenable, p.Declined) } + if p.Unseen != 1 { + t.Errorf("unseen %d, want 1 -- the one direction the pairing cannot see", p.Unseen) + } // Nothing was comparable, so nothing may be reported as a filter that // agreed or disagreed. if len(p.Filters) != 0 { @@ -241,6 +245,23 @@ func TestWhatCouldNotBeComparedReachesTheRecord(t *testing.T) { } } +func TestTheReportNamesWhatTheJudgeGotAndWeDidNot(t *testing.T) { + // For the reason the hangs are named: a report that leaves it out cannot + // be told from one that has none, and this is the only direction of + // disagreement that says the field reads something we do not. + judge(t, + images.Result{Difference: images.Difference{Share: -1}, Missing: images.Unseen, + Note: "the judge took out a image of 9x9 (row 0, object 0) we have no picture for"}, + ) + var out, errOut bytes.Buffer + if code := run([]string{"-dir", tinyCorpus(t), "-only", "alpha"}, &out, &errOut); code != 0 { + t.Fatalf("exit %d: %s", code, errOut.String()) + } + if !strings.Contains(out.String(), "1 the judge took out and we did not") { + t.Errorf("the report is silent about it:\n%s", out.String()) + } +} + func TestABaselineWithNoBuildInfoStillRecordsTheCounts(t *testing.T) { // Nothing here is worth losing a run's counts over. was := buildInfo diff --git a/images/images.go b/images/images.go index ddaa661..9155b15 100644 --- a/images/images.go +++ b/images/images.go @@ -354,6 +354,19 @@ const ( // this cannot get an answer about, and one that must be counted so that a // corpus getting harder is not read as a decoder getting worse. Theirs Missing = "theirs" + // Unseen means the JUDGE took a picture out that ours produced nothing + // for. It is the mirror of Unmatched and it had no name, because the + // pairing walks OUR pictures and appends one result for each: a row of + // the judge's that nothing claimed left no trace in any count, so the one + // direction this harness could not see was the direction that matters + // most -- a picture the field gets out of a file and we do not. + // + // It is known to be non-empty by construction for one class: inline + // images. render.Images does not return the pictures written into a + // content stream with BI, because they are the objects of nothing, and + // pdfimages lists them with an object of 0. Whether this corpus holds any + // is what the count is for. + Unseen Missing = "unseen" // Hung means a poppler tool did not finish within Timeout. // // It is its own value rather than a kind of Theirs because a judge that @@ -518,6 +531,21 @@ func judgePage(d *reader.Document, path string, p int) []Result { } out = append(out, r) } + // The judge's rows nothing claimed. Name is left empty so that Tally + // skips these -- there is no filter of OURS to group them under, because + // there is no picture of ours -- and Summarize counts them from Missing + // like the other asymmetries. They are reported for the reason the hangs + // are: a report that leaves them out cannot be told from one that has + // none. + for j, t := range theirs { + if claimed[j] { + continue + } + out = append(out, Result{Path: path, Page: p, Difference: unjudged(), + Missing: Unseen, + Note: fmt.Sprintf("the judge took out a %s of %dx%d (row %d, object %d) we have no picture for", + t.kind, t.pic.W, t.pic.H, t.num, t.object)}) + } return out } @@ -1369,6 +1397,10 @@ type Summary struct { // Declined is how many pages ours drew pictures for and the judge took // none out of, so there was nothing to compare them with. Declined int `json:"declined"` + // Unseen is how many pictures the judge took out that ours produced + // nothing for -- the direction the pairing could not see at all until it + // was counted. omitempty, so records written before it keep their shape. + Unseen int `json:"unseen,omitempty"` // Hung names the documents a poppler tool would not finish on, with the // tool. A document that was skipped because the judge hung is not a // document that scored badly, and the two are indistinguishable in a @@ -1452,6 +1484,8 @@ func Summarize(population string, documents int, rs []Result) Summary { s.Unopenable++ case Theirs: s.Declined++ + case Unseen: + s.Unseen++ case Hung: s.Hung = append(s.Hung, Hang{Path: r.Path, Page: r.Page, Tool: r.Tool}) } diff --git a/images/images_test.go b/images/images_test.go index 1740470..71daad0 100644 --- a/images/images_test.go +++ b/images/images_test.go @@ -500,14 +500,36 @@ func TestPicturesAreOrderedByTheirNumberAndNotTheirName(t *testing.T) { func TestAPictureTheOtherSideDoesNotHave(t *testing.T) { // Nothing exact to pair on, and nothing of our size to fall back to. + // + // This case is SYMMETRIC and the test only ever asserted one half of it. + // Ours has a 2x1 nothing of theirs matches; theirs has a 9x9 nothing of + // ours matches. The first was reported from the day this was written; the + // second left no trace in any count, because the pairing walks our + // pictures and appends one result for each. The test asserted len == 1 + // and so recorded the blind spot as the expected answer. standInWithoutObjects(t, image.NewRGBA(image.Rect(0, 0, 9, 9))) got := Judge(pageOfPictures(t, func(w *reader.Writer) reader.Dict { return reader.Dict{"I": grey(w)} }), Options{}) - if len(got) != 1 || got[0].Share != -1 || got[0].PairedBy != "" || - got[0].Note != "they took out nothing this size" { + if len(got) != 2 { t.Fatalf("got %+v", got) } + if got[0].Share != -1 || got[0].PairedBy != "" || + got[0].Note != "they took out nothing this size" { + t.Errorf("our unmatched picture came back as %+v", got[0]) + } + if got[1].Missing != Unseen || got[1].Name != "" || + got[1].Note != "the judge took out a image of 9x9 (row 0, object 0) we have no picture for" { + t.Errorf("their unmatched picture came back as %+v", got[1]) + } + // Name is empty on purpose: there is no filter of OURS to group it under, + // so Tally must skip it and Summarize must still count it. + if by := Tally(got); len(by) != 1 { + t.Errorf("the tally grouped the judge's own picture under a filter: %v", by) + } + if s := Summarize("p", 1, got); s.Unseen != 1 { + t.Errorf("the record says %d unseen, want 1", s.Unseen) + } } // TestAPairWhoseSizesDisagreeSaysSo is the other way a comparison does not @@ -1843,3 +1865,28 @@ func TestTheSizePairedShareIsCounted(t *testing.T) { t.Error("the record has no DCTDecode line to carry it") } } + +func TestOnlyTheJudgePicturesNothingClaimedAreCounted(t *testing.T) { + // The other side of the counter, and the side a test is easy to leave + // out: a judge that took out TWO pictures, one of which ours matched. + // Counting both would read exactly like counting the right one, until a + // run where every picture paired reported every picture as missing from + // our side. + standIn(t, image.NewRGBA(image.Rect(0, 0, 2, 1)), image.NewRGBA(image.Rect(0, 0, 2, 1))) + got := Judge(pageOfPictures(t, func(w *reader.Writer) reader.Dict { + return reader.Dict{"I": grey(w)} + }), Options{}) + if len(got) != 2 { + t.Fatalf("got %+v", got) + } + if got[0].Missing != Judged || got[0].PairedBy != PairedByObject { + t.Errorf("our picture did not pair: %+v", got[0]) + } + if got[1].Missing != Unseen { + t.Errorf("the second judge row came back as %+v", got[1]) + } + if s := Summarize("p", 1, got); s.Unseen != 1 { + t.Errorf("the record says %d unseen, want 1 -- the claimed row is "+ + "being counted as well", s.Unseen) + } +} From 11415bc7ff32163a6086d41c3bc63aeed21c6e73 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Sun, 4 Oct 2026 11:37:45 +0200 Subject: [PATCH 15/28] images: tell a repeated draw apart from a picture we failed to get MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The counter added in the commit before this one said 7 614 over the eighteen forms populations, 7 480 of them in fr-cerfa, and that number was wrong in the way a number is wrong when one document carries 304 of the 316 in a sixty-document sample. The document is one this file already names, cerfa_10074.pdf, the page of 211 uniform 2x2 swatches under /SMasks. Asked directly: rows pdfimages -list prints 729 distinct objects among them 214 rows for object 147 alone 62 pictures render.Images returns 425 729 - 425 304 = the unseen count pdfimages lists one row PER DRAW. The judgePage comment said a repeated draw came back "against pdfimages's one"; that was never measured and is wrong, and it is corrected where it stands. Ours is one entry per object on purpose -- decoding the same stream twice says nothing about a codec -- so the asymmetry is expected and harmless to every fidelity figure here. Counting it as pictures we failed to get is not. Missing.Repeated is now its own count, in the record and on a report line printed beside Unseen so the small number is read as small. What is left in Unseen is a scope difference and it is real: six rows in those sixty documents, one of which is cerfa_10011.pdf's 106x56 DCTDecode reached as /Annots -> /Widget /Btn -> /MK /I -> Form /FRM -> /Im0, a push button's ICON. Images walks the content stream and cannot see it; pdfimages renders the page with its annotations. A tool written to count exactly those agreed DOCUMENT BY DOCUMENT on 56 of 60, and the four it missed are the repeat-draw case. render.Page draws annotations, so this is not a decoder gap -- it is the extraction API's scope, which is §31's finding in a different place. Both classifications are mutation-checked in both directions: never repeated gives "repeated 0, want 1" and "unseen 2, want 1"; always repeated breaks the two tests that need an unseen. 100.0% of statements. §32 has the measurement. Whether Images should walk /AP and /MK is named there and not decided: it changes every count in the file. --- baseline/README.md | 63 ++++++++++++++++++++++++++++++ cmd/images/run.go | 6 +++ cmd/images/run_test.go | 21 ++++++++++ images/images.go | 89 +++++++++++++++++++++++++++++++++--------- images/images_test.go | 63 ++++++++++++++++++++++++++++++ 5 files changed, 223 insertions(+), 19 deletions(-) diff --git a/baseline/README.md b/baseline/README.md index 1cd4fbc..063c2b3 100644 --- a/baseline/README.md +++ b/baseline/README.md @@ -2962,3 +2962,66 @@ out of a program that asks neither. **Two instruments sharing no dependency agreeing on a census of sixty-six is the kind of control this file has been short of** — three of them agreed once before and all three were reading the same layer. + +## §32 — The direction the pairing could not see, and why its first number was wrong + +The pairing walks OUR pictures and appends one result for each. A row of the +judge's that nothing claimed left no trace in any count, so **the one direction +this harness could not see was the direction that matters most**: a picture the +field gets out of a file and we do not. + +Its own test suite had already set the case up and recorded the blind spot as +the expected answer. `TestAPictureTheOtherSideDoesNotHave` stands a judge with +a 9×9 against a page holding a 2×1: ours is unmatched **and** theirs is, the +first has been reported since the day it was written, the second was dropped, +and the test asserted `len == 1`. + +### The first number was 7 614, and it was not a gap + +Counted over the eighteen `pdfforms` populations at render v0.67.0: +**7 614**, of which `fr-cerfa` alone had 7 480. A number that size would mean +poppler extracts thousands of pictures we cannot. It does not, and **one +document carried 304 of the 316 in the first sixty of `fr-cerfa`** — which is +the shape a dominating artefact has, not a population. + +That document is one this file already names: `cerfa_10074.pdf`, the page of +211 uniform 2×2 swatches under `/SMask`s. Asked directly: + +| page 1 of `cerfa_10074.pdf` | | +|---|---:| +| rows `pdfimages -list` prints | **729** | +| distinct objects among them | **214** | +| rows for object 147 alone | **62** | +| pictures `render.Images` returns | **425** | +| 729 − 425 | **304**, the unseen count exactly | + +**`pdfimages` lists one row PER DRAW.** The `judgePage` comment in +`images/images.go` said a repeated draw came back "against pdfimages's one"; +that was never measured and is wrong. It is corrected there, citing this. + +Ours is one entry per object on purpose — decoding the same stream a second +time says nothing about a codec — so the asymmetry is expected and harmless to +every fidelity figure in this file. What is NOT harmless is counting it as +pictures we failed to get. `Repeated` is now a count of its own, and the report +prints it beside `Unseen` so that the small number is read as small. + +### What is left in `Unseen` is a scope difference, and it is real + +In the same sixty documents, six rows remain, and they are a different thing. +`cerfa_10011.pdf` is one: a 106×56 `DCTDecode` image, object 1865, reached as + + /Annots → /Widget /FT /Btn → /MK /I → 1864 (Form /FRM) → /XObject /Im0 → 1865 + +a push button's **icon**. `render.Images` walks the content stream, so it cannot +see it; `pdfimages` renders the page with its annotations and takes it out. +A tool written for the purpose agreed with the count **document by document on +56 of 60**, and the four it missed are the repeat-draw case above. + +**This is not a decoder gap.** `render.Page` draws annotations by default +(`Options.NoAnnotations` turns them off), so our renderer shows these icons; it +is the extraction API whose scope is narrower — the same shape as §31's finding +about the page budget, in a different place. + +Whether `Images` should also walk `/AP` and `/MK` is a question about what an +extraction API is for, and it changes every count in this file. It is named +here and not decided. diff --git a/cmd/images/run.go b/cmd/images/run.go index dde3d2c..efcf424 100644 --- a/cmd/images/run.go +++ b/cmd/images/run.go @@ -136,6 +136,12 @@ func run(args []string, out, errOut io.Writer) int { if summary.Unseen > 0 { fmt.Fprintf(out, " %5d the judge took out and we did not\n", summary.Unseen) } + // And beside it the rows that are only a difference of unit, so that + // the one above is read as the small number it is. + if summary.Repeated > 0 { + fmt.Fprintf(out, " %5d rows the judge listed again for a picture we return once\n", + summary.Repeated) + } } if *asJSON { // Marshal fails only on a value it cannot encode, and this one is diff --git a/cmd/images/run_test.go b/cmd/images/run_test.go index a3dd7d2..a1a4c56 100644 --- a/cmd/images/run_test.go +++ b/cmd/images/run_test.go @@ -219,6 +219,7 @@ func TestWhatCouldNotBeComparedReachesTheRecord(t *testing.T) { images.Result{Difference: images.Difference{Share: -1}, Missing: images.Neither, Note: "refused: y"}, images.Result{Difference: images.Difference{Share: -1}, Missing: images.Theirs, Note: "they took nothing out"}, images.Result{Difference: images.Difference{Share: -1}, Missing: images.Unseen, Note: "the judge took out a image of 9x9 (row 0, object 0) we have no picture for"}, + images.Result{Difference: images.Difference{Share: -1}, Missing: images.Repeated, Note: "the judge listed object 7 again as a image of 2x1 (row 1); we return it once"}, ) atTime(t, "2026-08-30T15:04:05Z") var out, errOut bytes.Buffer @@ -238,6 +239,10 @@ func TestWhatCouldNotBeComparedReachesTheRecord(t *testing.T) { if p.Unseen != 1 { t.Errorf("unseen %d, want 1 -- the one direction the pairing cannot see", p.Unseen) } + // Apart, because one is a fidelity question and the other is a unit. + if p.Repeated != 1 { + t.Errorf("repeated %d, want 1", p.Repeated) + } // Nothing was comparable, so nothing may be reported as a filter that // agreed or disagreed. if len(p.Filters) != 0 { @@ -262,6 +267,22 @@ func TestTheReportNamesWhatTheJudgeGotAndWeDidNot(t *testing.T) { } } +func TestTheReportSeparatesARepeatedRowFromAMissingPicture(t *testing.T) { + // Printed beside the line above so that the line above is read as the + // small number it is: on one real page the two are 6 and 304. + judge(t, + images.Result{Difference: images.Difference{Share: -1}, Missing: images.Repeated, + Note: "the judge listed object 7 again as a image of 2x1 (row 1); we return it once"}, + ) + var out, errOut bytes.Buffer + if code := run([]string{"-dir", tinyCorpus(t), "-only", "alpha"}, &out, &errOut); code != 0 { + t.Fatalf("exit %d: %s", code, errOut.String()) + } + if !strings.Contains(out.String(), "1 rows the judge listed again") { + t.Errorf("the report is silent about it:\n%s", out.String()) + } +} + func TestABaselineWithNoBuildInfoStillRecordsTheCounts(t *testing.T) { // Nothing here is worth losing a run's counts over. was := buildInfo diff --git a/images/images.go b/images/images.go index 9155b15..45f86b4 100644 --- a/images/images.go +++ b/images/images.go @@ -354,19 +354,39 @@ const ( // this cannot get an answer about, and one that must be counted so that a // corpus getting harder is not read as a decoder getting worse. Theirs Missing = "theirs" - // Unseen means the JUDGE took a picture out that ours produced nothing - // for. It is the mirror of Unmatched and it had no name, because the - // pairing walks OUR pictures and appends one result for each: a row of - // the judge's that nothing claimed left no trace in any count, so the one - // direction this harness could not see was the direction that matters + // Unseen means the judge took a picture out whose OBJECT ours produced + // nothing for. It is the mirror of Unmatched and it had no name, because + // the pairing walks OUR pictures and appends one result for each: a row + // of the judge's that nothing claimed left no trace in any count, so the + // one direction this harness could not see was the direction that matters // most -- a picture the field gets out of a file and we do not. // - // It is known to be non-empty by construction for one class: inline - // images. render.Images does not return the pictures written into a - // content stream with BI, because they are the objects of nothing, and - // pdfimages lists them with an object of 0. Whether this corpus holds any - // is what the count is for. + // Two classes are known to land here by construction. Inline images: + // render.Images does not return the pictures written into a content + // stream with BI, because they are the objects of nothing, and pdfimages + // lists them with an object of 0. And pictures only an ANNOTATION + // reaches: Images walks the content stream, so a push button's icon + // (/MK /I) or a widget's appearance (/AP /N) is invisible to it, while + // pdfimages renders the page with its annotations. cerfa_10011.pdf is one + // picture of that second kind, and it is NOT a decoder gap -- render.Page + // draws annotations. See baseline/README.md §32. Unseen Missing = "unseen" + // Repeated means the judge listed a row for a picture ours produced + // ONCE, under an object we did compare -- the same stream drawn again. + // + // It has to be apart from Unseen or the number is worthless. pdfimages + // lists one row PER DRAW: page 1 of cerfa_10074.pdf gives it 729 rows + // over 214 distinct objects, object 147 sixty-two times, against the 425 + // pictures Images returns. Counting those 304 leftovers as pictures the + // field gets and we do not would be false, and large enough to drown the + // six real ones in the same sixty documents. + // + // Ours is one entry per object on purpose: decoding the same stream a + // second time says nothing about a codec. The asymmetry is therefore + // expected, and it is counted so that 729 against 425 has somewhere to + // show. The package comment's claim that pdfimages gives "one" for a + // repeated draw was never measured and is wrong; §32 measures it. + Repeated Missing = "repeated" // Hung means a poppler tool did not finish within Timeout. // // It is its own value rather than a kind of Theirs because a judge that @@ -423,9 +443,17 @@ func Judge(path string, opt Options) []Result { // Nothing deduplicates the pictures here, and that is a decision about which // version of render this is built against. Under v0.19.0 render.Images // answered per DRAW — a page that stamped the same logo five hundred times -// yielded five hundred entries decoded from the same bytes, against -// pdfimages's one, and every repeat after the first landed in the unmatched -// column. This package compensated by keeping one entry per resource name. +// yielded five hundred entries decoded from the same bytes, and every repeat +// after the first landed in the unmatched column. This package compensated by +// keeping one entry per resource name. +// +// That sentence said "against pdfimages's one", which was never measured and +// is WRONG. pdfimages lists one row per draw as well: page 1 of +// cerfa_10074.pdf gives it 729 rows over 214 distinct objects, object 147 +// sixty-two times, against the 425 pictures Images returns. The two sides +// disagree about the UNIT, and the 304 rows left over on the judge's side are +// counted as Repeated rather than as pictures we failed to get. See +// baseline/README.md §32. // // v0.20.0 removed the cause: a page's resources are a graph and were being // walked as a tree, so a picture reached through two forms was decoded twice. @@ -537,14 +565,29 @@ func judgePage(d *reader.Document, path string, p int) []Result { // like the other asymmetries. They are reported for the reason the hangs // are: a report that leaves them out cannot be told from one that has // none. + // + // A row whose OBJECT is one we compared is the same stream drawn again, + // which is not a picture we failed to get. Those two must not share a + // count: see Repeated. + mine := make(map[int]bool, len(ours)) + for _, im := range ours { + if im.Object > 0 { + mine[im.Object] = true + } + } for j, t := range theirs { if claimed[j] { continue } - out = append(out, Result{Path: path, Page: p, Difference: unjudged(), - Missing: Unseen, + r := Result{Path: path, Page: p, Difference: unjudged(), Missing: Unseen, Note: fmt.Sprintf("the judge took out a %s of %dx%d (row %d, object %d) we have no picture for", - t.kind, t.pic.W, t.pic.H, t.num, t.object)}) + t.kind, t.pic.W, t.pic.H, t.num, t.object)} + if t.object > 0 && mine[t.object] { + r.Missing = Repeated + r.Note = fmt.Sprintf("the judge listed object %d again as a %s of %dx%d (row %d); we return it once", + t.object, t.kind, t.pic.W, t.pic.H, t.num) + } + out = append(out, r) } return out } @@ -1397,10 +1440,16 @@ type Summary struct { // Declined is how many pages ours drew pictures for and the judge took // none out of, so there was nothing to compare them with. Declined int `json:"declined"` - // Unseen is how many pictures the judge took out that ours produced - // nothing for -- the direction the pairing could not see at all until it - // was counted. omitempty, so records written before it keep their shape. + // Unseen is how many pictures the judge took out, under an object ours + // produced nothing for -- the direction the pairing could not see at all + // until it was counted. omitempty, so records written before it keep + // their shape. Unseen int `json:"unseen,omitempty"` + // Repeated is how many of the judge's rows name an object we compared + // once. It is the scale of a difference in UNIT, not in fidelity: + // pdfimages lists one row per draw and Images returns one entry per + // object. + Repeated int `json:"repeated,omitempty"` // Hung names the documents a poppler tool would not finish on, with the // tool. A document that was skipped because the judge hung is not a // document that scored badly, and the two are indistinguishable in a @@ -1486,6 +1535,8 @@ func Summarize(population string, documents int, rs []Result) Summary { s.Declined++ case Unseen: s.Unseen++ + case Repeated: + s.Repeated++ case Hung: s.Hung = append(s.Hung, Hang{Path: r.Path, Page: r.Page, Tool: r.Tool}) } diff --git a/images/images_test.go b/images/images_test.go index 71daad0..6ccc8d8 100644 --- a/images/images_test.go +++ b/images/images_test.go @@ -1890,3 +1890,66 @@ func TestOnlyTheJudgePicturesNothingClaimedAreCounted(t *testing.T) { "being counted as well", s.Unseen) } } + +// TestARowTheJudgeListedTwiceIsNotAPictureWeMissed is the measurement that +// made Repeated necessary. pdfimages lists one row PER DRAW: page 1 of +// cerfa_10074.pdf gives it 729 rows over 214 distinct objects -- object 147 +// sixty-two times -- against the 425 pictures Images returns, and all 304 +// leftovers were being counted as pictures the field gets out and we do not. +func TestARowTheJudgeListedTwiceIsNotAPictureWeMissed(t *testing.T) { + path := pageOfPictures(t, func(w *reader.Writer) reader.Dict { + return reader.Dict{"I": grey(w)} + }) + obj := imageObjects(path) + if len(obj) != 1 { + t.Fatalf("the fixture has %d image objects, want 1", len(obj)) + } + // A judge that drew the one picture twice, and one row for a picture that + // is not ours at all, so both branches are exercised at once. + wasPictures, wasList := popplerCommand, listCommand + t.Cleanup(func() { popplerCommand, listCommand = wasPictures, wasList }) + popplerCommand = func(args ...string) (bool, error) { + stem := args[len(args)-1] + for i := 0; i < 3; i++ { + f, err := os.Create(fmt.Sprintf("%s-%03d.png", stem, i)) + if err != nil { + return false, err + } + err = png.Encode(f, image.NewRGBA(image.Rect(0, 0, 2, 1))) + f.Close() + if err != nil { + return false, err + } + } + return false, nil + } + listCommand = func(...string) ([]byte, bool, error) { + var sb strings.Builder + sb.WriteString("page num type width height color comp bpc enc interp object ID x-ppi y-ppi size ratio\n") + sb.WriteString("------\n") + for i, o := range []int{obj[0], obj[0], 9999} { + fmt.Fprintf(&sb, " 1 %4d image 2 1 gray 1 8 image no %2d 0 72 72 9B 50%%\n", i, o) + } + return []byte(sb.String()), false, nil + } + got := Judge(path, Options{}) + s := Summarize("p", 1, got) + if s.Repeated != 1 { + t.Errorf("repeated %d, want 1 -- the second row of our own object", s.Repeated) + } + if s.Unseen != 1 { + t.Errorf("unseen %d, want 1 -- object 9999 is nobody's picture of ours", s.Unseen) + } + var sawRepeat bool + for _, r := range got { + if r.Missing == Repeated { + sawRepeat = true + if !strings.Contains(r.Note, "we return it once") { + t.Errorf("the note does not say why: %q", r.Note) + } + } + } + if !sawRepeat { + t.Error("no result carried the repeat") + } +} From 6eea8b4452e4ba8ca51ddab00e6fbb7394bfddee Mon Sep 17 00:00:00 2001 From: tannevaled Date: Sun, 4 Oct 2026 11:47:58 +0200 Subject: [PATCH 16/28] images: name the real cause of Unseen -- inline images, not annotations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit §32 offered annotations as the cause and that was fitted to a sample where the thing being counted was almost always nought. Fifty-six of sixty documents "agreeing" was fifty-four pairs of noughts. Over the whole fr-cerfa population instead -- 450 forms, page 1 each -- there are 4 153 unseen rows after the repeat split, and a program that counts the pictures only an annotation reaches finds 25 of them in all 450. Grouped by what the judge's own listing says: 4 147 object 0 2 614 stencil type 1 538 image type 2 592 16x16 13 documents of 450 carrying any 2 592 cerfa_12481.pdf alone An object of nought is what pdfimages prints for an INLINE image, and render.Images says plainly that it does not return those. Read at the byte level, the first three of that document's eight content streams hold 751 BI...ID images, the first being /IM true /W 16 /H 16 /BPC 1 /D[1 0] /F /CCF /DP<> a 16x16 CCITT-fax image mask, which is exactly the stencil 16x16 row poppler lists. So a class of 4 147 pictures in one population is drawn by render.Page and compared against the reference by nothing: a defect in inline-image decoding would be invisible to every figure in this baseline. Filed as go-pdfkit/render#101. It also explains a nought from two sections earlier. SizePaired is zero in every population because the pairing falls back to size exactly when an object number is missing, and the one class whose object number is missing never reaches the pairing. The fallback was written for a case the API cannot deliver. The annotation case stays, measured and small, with cerfa_10011.pdf's 106x56 DCTDecode reached through /MK /I. Neither case is a decoder gap -- render.Page draws both -- and §32 keeps the method note, because this is the third total in one day dominated by a single document. --- baseline/README.md | 81 ++++++++++++++++++++++++++++++++++++++-------- images/images.go | 25 ++++++++------ 2 files changed, 83 insertions(+), 23 deletions(-) diff --git a/baseline/README.md b/baseline/README.md index 063c2b3..6c5a64c 100644 --- a/baseline/README.md +++ b/baseline/README.md @@ -3005,23 +3005,76 @@ every fidelity figure in this file. What is NOT harmless is counting it as pictures we failed to get. `Repeated` is now a count of its own, and the report prints it beside `Unseen` so that the small number is read as small. -### What is left in `Unseen` is a scope difference, and it is real +### What is left in `Unseen` is INLINE IMAGES, and the first answer here was wrong -In the same sixty documents, six rows remain, and they are a different thing. -`cerfa_10011.pdf` is one: a 106×56 `DCTDecode` image, object 1865, reached as +~~In the same sixty documents, six rows remain … a push button's **icon** … +A tool written for the purpose agreed with the count **document by document on +56 of 60**.~~ + +**That was measured on a sample where the thing being counted was almost always +zero, and it is wrong.** Taken over the whole population instead: `fr-cerfa` has +**4 153** unseen rows after the repeat split, and a program that counts the +pictures only an annotation reaches finds **25** of them in all 450 documents. +Fifty-six of sixty agreeing was fifty-four pairs of noughts. + +Grouped by what the judge's own listing says: + +| of `fr-cerfa`'s 4 153 unseen rows | | +|---|---:| +| object **0** | **4 147** | +| `stencil` type | 2 614 | +| `image` type | 1 538 | +| 16×16 | 2 592 | +| documents carrying any | **13** of 450 | +| `cerfa_12481.pdf` alone | **2 592** | + +An object of nought is what `pdfimages` prints for an **inline image**, and +`render.Images` says plainly that it does not return those: *"Inline images -- +the ones written into the content stream with BI -- are not returned: they are +named by no resource and are objects of nothing."* Read at the byte level, the +first three of `cerfa_12481.pdf`'s eight content streams hold **751** `BI`…`ID` +images, the first of them + + /IM true /W 16 /H 16 /BPC 1 /D[1 0] /F /CCF /DP<> + +a 16×16 CCITT-fax image mask -- which is exactly the `stencil 16x16` row +poppler lists. + +**So a class of 4 147 pictures in one population is decoded by `render.Page` +and compared against the reference by nothing.** `drawInlineImage` draws them; +no figure in this file touches them. A defect in inline-image handling would be +invisible to every number here. + +And it explains a nought that stood unexplained two sections ago. +**`SizePaired` is zero in every population** because the pairing falls back to +size exactly when an object number is missing, and the one class whose object +number is missing never reaches the pairing at all. The fallback was written for +a case the harness cannot see. + +Tracked as [go-pdfkit/render#101](https://github.com/go-pdfkit/render/issues/101). + +The annotation case is real and small: 25 in 450, one of them +`cerfa_10011.pdf`'s 106×56 `DCTDecode`, object 1865, reached as /Annots → /Widget /FT /Btn → /MK /I → 1864 (Form /FRM) → /XObject /Im0 → 1865 -a push button's **icon**. `render.Images` walks the content stream, so it cannot -see it; `pdfimages` renders the page with its annotations and takes it out. -A tool written for the purpose agreed with the count **document by document on -56 of 60**, and the four it missed are the repeat-draw case above. +a push button's **icon**. Neither case is a decoder gap: `render.Page` draws +annotations by default and draws inline images. Both are the extraction API's +scope being narrower than the renderer's -- which is §31's finding, a third +time, in a third place. + +### The method, which is the part worth keeping -**This is not a decoder gap.** `render.Page` draws annotations by default -(`Options.NoAnnotations` turns them off), so our renderer shows these icons; it -is the extraction API whose scope is narrower — the same shape as §31's finding -about the page budget, in a different place. +Three times in one day a total was dominated by one document, and twice the +first explanation fitted the sample and not the population: -Whether `Images` should also walk `/AP` and `/MK` is a question about what an -extraction API is for, and it changes every count in this file. It is named -here and not decided. +| | sample | population | +|---|---|---| +| first count | 316 in 60 documents | 7 614 in 18 populations | +| after the repeat split | 12 | 4 243 | +| cause offered | annotations, 56/60 agreeing | annotations explain **25 of 4 153** | +| cause measured | — | inline images, **4 147 of 4 153** | + +**An agreement where both sides are mostly nought is not an agreement.** The +control that would have caught it is the one this file keeps asking for: count +the positives, and refuse to quote a rate whose denominator is six. diff --git a/images/images.go b/images/images.go index 45f86b4..cdeb738 100644 --- a/images/images.go +++ b/images/images.go @@ -361,15 +361,22 @@ const ( // one direction this harness could not see was the direction that matters // most -- a picture the field gets out of a file and we do not. // - // Two classes are known to land here by construction. Inline images: - // render.Images does not return the pictures written into a content - // stream with BI, because they are the objects of nothing, and pdfimages - // lists them with an object of 0. And pictures only an ANNOTATION - // reaches: Images walks the content stream, so a push button's icon - // (/MK /I) or a widget's appearance (/AP /N) is invisible to it, while - // pdfimages renders the page with its annotations. cerfa_10011.pdf is one - // picture of that second kind, and it is NOT a decoder gap -- render.Page - // draws annotations. See baseline/README.md §32. + // Measured over fr-cerfa's 450 forms, it is 4 153 rows and two classes, + // in proportions worth knowing before anyone reads the number as a gap: + // + // 4 147 object 0 -- INLINE IMAGES, which render.Images does not + // return at all. 2 614 of them are stencils, 2 592 of them + // 16x16, and ONE document holds 2 592. go-pdfkit/render#101. + // 25 pictures only an ANNOTATION reaches, counted over the whole + // population: a push button's icon (/MK /I) or a widget's + // appearance (/AP /N), which Images cannot see because it + // walks the content stream. cerfa_10011.pdf is one. + // + // Neither is a decoder gap: render.Page draws annotations and draws + // inline images. Both are the extraction API's scope being narrower than + // the renderer's. See baseline/README.md §32, which also records that + // the FIRST cause offered here was the small one, fitted to a sample + // where both sides were almost always nought. Unseen Missing = "unseen" // Repeated means the judge listed a row for a picture ours produced // ONCE, under an object we did compare -- the same stream drawn again. From 0772fc7587683b4045a9a381967953b6f5fd3fdf Mon Sep 17 00:00:00 2001 From: tannevaled Date: Sun, 4 Oct 2026 12:05:30 +0200 Subject: [PATCH 17/28] baseline: the third cause of Unseen was a rendering defect, and it is fixed gh-qpdf's 13 unseen rows are 3 inline images, 0 annotation pictures, and TEN that were a defect. The fixture names itself: form-xobjects-some-resources2.pdf. Both of render's paths had half a rule. drawForm and imagesDrawn each fell back to the parent when a form XObject's /Resources was ABSENT, and a PRESENT one replaced it wholesale -- so a name the form did not list resolved to nothing and the picture was silently not drawn. A form's resource dictionary is not required to be complete. Six images against our two on that page, and it reached the page and not only the extraction: through this repository's own compare against pdftoppm, 8.54% of pixels away at a worst square mean of 130 levels. After the fix, 0.03% and 8.80. go-pdfkit/render#103 fixes it, filed as #102, checked in both directions over the whole forms corpus: exactly one population moves, gh-qpdf's pictures 63 -> 73, and the ten recovered land in (samples) direct as 51 of 51 exact and 51 of 51 byte-identical with poppler. Which is what the counter was for. Unseen was added as a field documenting a requirement, its first number was wrong twice, and what survived the two corrections found a rendering defect every fidelity figure in this file had been blind to -- because a picture nobody returns is a picture nobody compares. --- baseline/README.md | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/baseline/README.md b/baseline/README.md index 6c5a64c..59f280b 100644 --- a/baseline/README.md +++ b/baseline/README.md @@ -3063,6 +3063,41 @@ annotations by default and draws inline images. Both are the extraction API's scope being narrower than the renderer's -- which is §31's finding, a third time, in a third place. +### A third cause, and it was a real defect + +`gh-qpdf` has 13 unseen rows, of which 3 are inline images and **0** are +annotation pictures. The remaining **ten** were a defect, and the fixture names +itself: `form-xobjects-some-resources2.pdf`. + +Both of `render`'s paths had half a rule. `drawForm` and `imagesDrawn` each fell +back to the parent when a form XObject's `/Resources` was **absent**, and a +**present** one replaced it wholesale — so a name the form did not list resolved +to nothing and the picture was silently not drawn. A form's resource dictionary +is not required to be complete; a name it does not provide is resolved in the +resources in force where the form was painted, which is what poppler does. + +| that page | | +|---|---:| +| images `pdfimages -list` takes out | 6 | +| images `render.Images` returned | **2** | + +And it reached the page, not only the extraction: through this repository's own +`compare` against `pdftoppm`, those fixtures were **8.54%** of pixels away at a +worst square mean of **130 levels**. After the fix, **0.03%** and 8.80. + +Fixed in [go-pdfkit/render#103](https://github.com/go-pdfkit/render/pull/103), +filed as [#102](https://github.com/go-pdfkit/render/issues/102). Checked in both +directions over the whole forms corpus — 2 268 documents, eighteen populations — +where **exactly one** population moves: `gh-qpdf`, pictures 63 → 73, the ten +recovered landing in `(samples) direct` as **51 of 51 exact and 51 of 51 +byte-identical**. They are not merely present; they are right. + +**That is what the counter was for.** `Unseen` was added two sections ago as a +field that documented a requirement, its first number was wrong twice, and what +survived the two corrections found a rendering defect that every fidelity figure +in this file had been blind to — because a picture nobody returns is a picture +nobody compares. + ### The method, which is the part worth keeping Three times in one day a total was dominated by one document, and twice the @@ -3074,6 +3109,7 @@ first explanation fitted the sample and not the population: | after the repeat split | 12 | 4 243 | | cause offered | annotations, 56/60 agreeing | annotations explain **25 of 4 153** | | cause measured | — | inline images, **4 147 of 4 153** | +| what the remainder held | — | a real defect: render#102, ten rows of `gh-qpdf` | **An agreement where both sides are mostly nought is not an agreement.** The control that would have caught it is the one this file keeps asking for: count From 1f1be1f67cd54a460eb97bd1e432f83e41886845 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Sun, 4 Oct 2026 12:09:26 +0200 Subject: [PATCH 18/28] baseline: the annotation case dominates another population, so neither ranks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit §32 demoted annotations to "real and small: 25 in 450", which is true of fr-cerfa and false as a general claim -- the same mistake as the one the section is about, in miniature. population documents unseen object 0 annotation-reached fr-cerfa 450 4153 4147 25 us-uscis 88 73 0 74 In us-uscis it is ONE picture per document in 73 of 88, and not one is inline: g-1041.pdf's is a 1125x75 one-bit strip, which is a barcode. On a population of government forms the barcode or logo that identifies the form is the one picture per page nothing ever compares. Filed as go-pdfkit/render#104. Neither cause ranks above the other -- each dominates a different population -- and a single ordering of them would have been wrong whichever way round it was written. --- baseline/README.md | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/baseline/README.md b/baseline/README.md index 59f280b..fed6ebb 100644 --- a/baseline/README.md +++ b/baseline/README.md @@ -3053,7 +3053,25 @@ a case the harness cannot see. Tracked as [go-pdfkit/render#101](https://github.com/go-pdfkit/render/issues/101). -The annotation case is real and small: 25 in 450, one of them +### The annotation case is small HERE and dominates another population + +Which is the same mistake in miniature, so it is worth the table. Counted over +two whole populations rather than one sample: + +| population | documents | unseen rows | of which object 0 | pictures an annotation reaches | +|---|---:|---:|---:|---:| +| `fr-cerfa` | 450 | 4 153 | **4 147** | 25 | +| `us-uscis` | 88 | **73** | **0** | **74** | + +In `us-uscis` it is **one picture per document, in 73 of 88**, and not one of +them is inline: `g-1041.pdf`'s is a 1125×75 one-bit strip, which is a barcode. +On a population of government forms, **the barcode or logo that identifies the +form is the one picture per page nothing ever compares.** Filed as +[go-pdfkit/render#104](https://github.com/go-pdfkit/render/issues/104). + +So neither cause ranks above the other; each dominates a different population, +and a single ordering of them would have been wrong whichever way round it was +written. In `fr-cerfa` the annotation case is 25 in 450, one of them `cerfa_10011.pdf`'s 106×56 `DCTDecode`, object 1865, reached as /Annots → /Widget /FT /Btn → /MK /I → 1864 (Form /FRM) → /XObject /Im0 → 1865 From 8b1276f3b2cddcdc7ee0366299b53b115316ee00 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Sun, 4 Oct 2026 15:03:02 +0200 Subject: [PATCH 19/28] =?UTF-8?q?baseline:=20re-take=20the=20twenty-three?= =?UTF-8?q?=20records,=20and=20=C2=A733=20on=20what=20moved?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit §30 said the records were thirty-two render releases and nineteen jpeg2000 releases behind the tree that reads them, and that the next step was the records and not a bound. Re-taken 2026-10-04 at render v0.67.0 and go-images/jpeg2000 v0.13.2, one population at a time for §22's reason, against the same pdfimages 26.04.0. TWO CELLS MOVED in thirty-two releases: ia-biodiversity JPXDecode direct pictures 496 -> 497 exact 496 -> 497 gh-pdfbox JPXDecode conv. mean -0.0020404 -> -0.0020666 and nothing else, in twenty-three populations: every other bucket is identical cell for cell, to every digit recorded. The first is §26's page, and the picture it gained comes out exact -- the one claim of §28's table that §31 did not refute, now standing on a record instead of on prose. The second is a signed mean moving by 0.000026 of a level on one picture. So §30 was right about the fact and wrong about the risk it implied: a record can be thirty-two releases stale in its metadata and current in its numbers, and re-taking it is the only way to know which. What it buys is that the prose and the records now describe the same code, which is what §31 found had failed. The fresh records also carry what the old ones could not: sizePaired 0, unseen 4 277, repeated 3 371 over 3 280 documents and 8 067 pictures. And §33 reconciles 66 with 63. §27 counts 66 documents neither draws; the records say unopenable 63. Four scans files are on disk and in no manifest row, and three of them are refusals -- two in ia-americana, one in ia-medical. images and compare walk the MANIFEST; the sweeps behind §27 and §31 walk the directory. Neither is wrong and no count is transferable between them without saying which it walked, which is why §24 reads 3 208, §27 reads 3 281 and §33 reads 3 280. One row has a fix waiting: render#103 takes gh-qpdf from 63 pictures to 73. These records are at the go.mod as it stands, so that row is expected to move when it lands, and §33 says so before it does. --- baseline/README.md | 86 ++++++++++++++++++++++++-- baseline/pdfforms-ca-cra.json | 15 ++--- baseline/pdfforms-fr-cerfa.json | 16 ++--- baseline/pdfforms-fr-impots.json | 14 ++--- baseline/pdfforms-gh-openpdf.json | 15 ++--- baseline/pdfforms-gh-pdfbox.json | 20 +++--- baseline/pdfforms-gh-pdfcpu.json | 14 ++--- baseline/pdfforms-gh-pypdf.json | 15 ++--- baseline/pdfforms-gh-qpdf.json | 15 ++--- baseline/pdfforms-gh-safedocs.json | 14 ++--- baseline/pdfforms-gh-verapdf.json | 14 ++--- baseline/pdfforms-int-wipo.json | 14 ++--- baseline/pdfforms-uk-govuk.json | 14 ++--- baseline/pdfforms-us-dol.json | 15 ++--- baseline/pdfforms-us-irs.json | 14 ++--- baseline/pdfforms-us-opm.json | 14 ++--- baseline/pdfforms-us-ssa.json | 14 ++--- baseline/pdfforms-us-uscis.json | 15 ++--- baseline/pdfforms-us-uscourts.json | 15 ++--- baseline/pdfscans-ia-americana.json | 15 ++--- baseline/pdfscans-ia-biodiversity.json | 20 +++--- baseline/pdfscans-ia-medical.json | 15 ++--- baseline/pdfscans-ia-texts.json | 14 ++--- baseline/pdfscans-ia-uscourts.json | 15 ++--- 24 files changed, 262 insertions(+), 170 deletions(-) diff --git a/baseline/README.md b/baseline/README.md index fed6ebb..8b0672f 100644 --- a/baseline/README.md +++ b/baseline/README.md @@ -2716,10 +2716,11 @@ next step. The records are.** **[#55](https://github.com/go-pdfkit/conformance/pull/55) landed on 2026-10-03**, so the first step is done and the table above has been amended -- -which is this section happening to itself, four days after it was written. What -is left of §30 is the row it exists for: the twenty-three records are still at -v0.35.0, and now they are **thirty-two** `render` releases behind the tree that -reads them rather than twenty-five. +which is this section happening to itself, four days after it was written. +~~What is left of §30 is the row it exists for: the twenty-three records are +still at v0.35.0.~~ **They were re-taken on 2026-10-04 and §33 says what +moved: two cells of twenty-three populations, one of them a signed mean in its +fifth decimal place.** §30's fact was right and the risk it implied was not. ### Why this is not a gate @@ -3132,3 +3133,80 @@ first explanation fitted the sample and not the population: **An agreement where both sides are mostly nought is not an agreement.** The control that would have caught it is the one this file keeps asking for: count the positives, and refuse to quote a rate whose denominator is six. + +## §33 — The records re-taken, and the staleness was almost entirely immaterial + +§30 said the twenty-three records were thirty-two `render` releases and nineteen +`go-images/jpeg2000` releases behind the tree that reads them, and that the next +step was the records and not a bound. They are re-taken: **2026-10-04, render +v0.67.0, `go-images/jpeg2000` v0.13.2**, one population at a time for §22's +reason, against the same `pdfimages` 26.04.0. + +### What moved in thirty-two releases: two cells + +| population | cell | 2026-09-24, v0.35.0 | 2026-10-04, v0.67.0 | +|---|---|---:|---:| +| `ia-biodiversity` | `JPXDecode direct` pictures | 496 | **497** | +| | …and `exact` | 496 | **497** | +| `gh-pdfbox` | `JPXDecode converted` signed `mean` | −0.0020404 | **−0.0020666** | + +**And nothing else, in twenty-three populations.** Every other bucket of every +other population is identical cell for cell: the same pictures, the same exact, +identical, inverted and differing, the same `share`, `peak` and `mse` to every +digit recorded. + +The first row is §26's page, and the picture it gained comes out exact — which +is the one claim of §28's table that §31 did NOT refute, now standing on a +record rather than on prose. The second is a signed mean moving by **0.000026 of +a level** on one picture, with its share, peak and mse unchanged. + +So §30 was right about the fact and wrong about the risk it implied. **A record +can be thirty-two releases stale in its metadata and current in its numbers**, +and the only way to know which is to re-take it. What this does buy is that the +prose and the records now describe the same code, which is the thing §31 found +had failed. + +### What the fresh records carry that the old ones could not + +| over 23 populations, 3 280 documents, 8 067 pictures | | +|---|---:| +| `refused` | **1** — §31's, still open as [render#100](https://github.com/go-pdfkit/render/issues/100) | +| `unopenable` | 63 | +| `declined` | 0 | +| `hung` | 0 | +| **`sizePaired`** | **0** | +| **`unseen`** | **4 277** | +| **`repeated`** | **3 371** | + +`sizePaired` is zero for the reason §32 found: the pairing falls back to size +when an object number is missing, and the one class whose number is missing — +inline images — never reaches it. + +### The four files that are not in the corpus, and why 66 was not 63 + +§27 counts **66** documents neither implementation draws. The records say +`unopenable` **63**. Both are right, and the difference is enumeration: + + /Users/Shared/pdfscans MANIFEST.tsv 1012 rows, 1013 files on disk + /Users/Shared/pdfforms MANIFEST.tsv 2268 rows, 2268 files on disk + +Four scans files are on disk and in no manifest row — +`bidragtillknne38suom.pdf`, `calcflh_000254.pdf`, +`epn11-1968countclip2restricted.pdf` and `b22346703.pdf` — and **three of those +four are refusals**: two in `ia-americana`, one in `ia-medical`. 66 − 3 = 63, +exactly. + +`images` and `compare` walk the MANIFEST; the sweep behind §27 and the census +behind §31 walk the directory. **Neither is wrong and no count is transferable +between them without saying which it walked**, which is why §24's totals read +3 208 and §27's read 3 281 and this section's read 3 280. The fourth file opens +normally, which is why it never showed up in a refusal count to be asked about. + +### One row of these records has a fix waiting + +[go-pdfkit/render#103](https://github.com/go-pdfkit/render/pull/103) is open and +takes `gh-qpdf` from 63 pictures to 73 — ten pictures a form XObject's +incomplete `/Resources` was hiding, all ten exact and byte-identical. These +records are taken at the `go.mod` as it stands, so that row is **expected to +move** the moment that lands, and this is the sentence that says so before it +does. diff --git a/baseline/pdfforms-ca-cra.json b/baseline/pdfforms-ca-cra.json index a238ffd..0a229f4 100644 --- a/baseline/pdfforms-ca-cra.json +++ b/baseline/pdfforms-ca-cra.json @@ -1,6 +1,6 @@ { "corpus": "/Users/Shared/pdfforms", - "taken": "2026-09-24T17:08:38Z", + "taken": "2026-10-04T09:37:56Z", "pages": 1, "gate": 2, "timeout": "2m0s", @@ -8,11 +8,11 @@ "modules": [ { "path": "github.com/ajroetker/go-highway", - "version": "v0.0.4" + "version": "v0.0.12" }, { "path": "github.com/go-gfx/gfx", - "version": "v0.31.0" + "version": "v0.34.0" }, { "path": "github.com/go-images/jpeg", @@ -20,15 +20,15 @@ }, { "path": "github.com/go-images/jpeg2000", - "version": "v0.1.0" + "version": "v0.13.2" }, { "path": "github.com/go-opentype/fonts", - "version": "v0.9.0" + "version": "v0.10.0" }, { "path": "github.com/go-opentype/opentype", - "version": "v0.12.0" + "version": "v0.13.0" }, { "path": "github.com/go-pdfkit/pdffont", @@ -40,7 +40,7 @@ }, { "path": "github.com/go-pdfkit/render", - "version": "v0.35.0" + "version": "v0.67.0" }, { "path": "github.com/sergeymakinen/go-bmp", @@ -70,6 +70,7 @@ "refused": 0, "unopenable": 0, "declined": 0, + "repeated": 5, "filters": [ { "filter": "(samples)", diff --git a/baseline/pdfforms-fr-cerfa.json b/baseline/pdfforms-fr-cerfa.json index 144c1af..fc43470 100644 --- a/baseline/pdfforms-fr-cerfa.json +++ b/baseline/pdfforms-fr-cerfa.json @@ -1,6 +1,6 @@ { "corpus": "/Users/Shared/pdfforms", - "taken": "2026-09-24T17:08:51Z", + "taken": "2026-10-04T09:38:07Z", "pages": 1, "gate": 2, "timeout": "2m0s", @@ -8,11 +8,11 @@ "modules": [ { "path": "github.com/ajroetker/go-highway", - "version": "v0.0.4" + "version": "v0.0.12" }, { "path": "github.com/go-gfx/gfx", - "version": "v0.31.0" + "version": "v0.34.0" }, { "path": "github.com/go-images/jpeg", @@ -20,15 +20,15 @@ }, { "path": "github.com/go-images/jpeg2000", - "version": "v0.1.0" + "version": "v0.13.2" }, { "path": "github.com/go-opentype/fonts", - "version": "v0.9.0" + "version": "v0.10.0" }, { "path": "github.com/go-opentype/opentype", - "version": "v0.12.0" + "version": "v0.13.0" }, { "path": "github.com/go-pdfkit/pdffont", @@ -40,7 +40,7 @@ }, { "path": "github.com/go-pdfkit/render", - "version": "v0.35.0" + "version": "v0.67.0" }, { "path": "github.com/sergeymakinen/go-bmp", @@ -70,6 +70,8 @@ "refused": 0, "unopenable": 0, "declined": 0, + "unseen": 4153, + "repeated": 3327, "filters": [ { "filter": "DCTDecode", diff --git a/baseline/pdfforms-fr-impots.json b/baseline/pdfforms-fr-impots.json index d0757f9..a32247b 100644 --- a/baseline/pdfforms-fr-impots.json +++ b/baseline/pdfforms-fr-impots.json @@ -1,6 +1,6 @@ { "corpus": "/Users/Shared/pdfforms", - "taken": "2026-09-24T17:09:32Z", + "taken": "2026-10-04T09:38:44Z", "pages": 1, "gate": 2, "timeout": "2m0s", @@ -8,11 +8,11 @@ "modules": [ { "path": "github.com/ajroetker/go-highway", - "version": "v0.0.4" + "version": "v0.0.12" }, { "path": "github.com/go-gfx/gfx", - "version": "v0.31.0" + "version": "v0.34.0" }, { "path": "github.com/go-images/jpeg", @@ -20,15 +20,15 @@ }, { "path": "github.com/go-images/jpeg2000", - "version": "v0.1.0" + "version": "v0.13.2" }, { "path": "github.com/go-opentype/fonts", - "version": "v0.9.0" + "version": "v0.10.0" }, { "path": "github.com/go-opentype/opentype", - "version": "v0.12.0" + "version": "v0.13.0" }, { "path": "github.com/go-pdfkit/pdffont", @@ -40,7 +40,7 @@ }, { "path": "github.com/go-pdfkit/render", - "version": "v0.35.0" + "version": "v0.67.0" }, { "path": "github.com/sergeymakinen/go-bmp", diff --git a/baseline/pdfforms-gh-openpdf.json b/baseline/pdfforms-gh-openpdf.json index 70f589f..e7b1d4b 100644 --- a/baseline/pdfforms-gh-openpdf.json +++ b/baseline/pdfforms-gh-openpdf.json @@ -1,6 +1,6 @@ { "corpus": "/Users/Shared/pdfforms", - "taken": "2026-09-24T17:09:35Z", + "taken": "2026-10-04T09:38:46Z", "pages": 1, "gate": 2, "timeout": "2m0s", @@ -8,11 +8,11 @@ "modules": [ { "path": "github.com/ajroetker/go-highway", - "version": "v0.0.4" + "version": "v0.0.12" }, { "path": "github.com/go-gfx/gfx", - "version": "v0.31.0" + "version": "v0.34.0" }, { "path": "github.com/go-images/jpeg", @@ -20,15 +20,15 @@ }, { "path": "github.com/go-images/jpeg2000", - "version": "v0.1.0" + "version": "v0.13.2" }, { "path": "github.com/go-opentype/fonts", - "version": "v0.9.0" + "version": "v0.10.0" }, { "path": "github.com/go-opentype/opentype", - "version": "v0.12.0" + "version": "v0.13.0" }, { "path": "github.com/go-pdfkit/pdffont", @@ -40,7 +40,7 @@ }, { "path": "github.com/go-pdfkit/render", - "version": "v0.35.0" + "version": "v0.67.0" }, { "path": "github.com/sergeymakinen/go-bmp", @@ -70,6 +70,7 @@ "refused": 0, "unopenable": 14, "declined": 0, + "repeated": 4, "filters": [ { "filter": "DCTDecode", diff --git a/baseline/pdfforms-gh-pdfbox.json b/baseline/pdfforms-gh-pdfbox.json index d1bbcdb..818560b 100644 --- a/baseline/pdfforms-gh-pdfbox.json +++ b/baseline/pdfforms-gh-pdfbox.json @@ -1,6 +1,6 @@ { "corpus": "/Users/Shared/pdfforms", - "taken": "2026-09-24T17:09:35Z", + "taken": "2026-10-04T09:38:47Z", "pages": 1, "gate": 2, "timeout": "2m0s", @@ -8,11 +8,11 @@ "modules": [ { "path": "github.com/ajroetker/go-highway", - "version": "v0.0.4" + "version": "v0.0.12" }, { "path": "github.com/go-gfx/gfx", - "version": "v0.31.0" + "version": "v0.34.0" }, { "path": "github.com/go-images/jpeg", @@ -20,15 +20,15 @@ }, { "path": "github.com/go-images/jpeg2000", - "version": "v0.1.0" + "version": "v0.13.2" }, { "path": "github.com/go-opentype/fonts", - "version": "v0.9.0" + "version": "v0.10.0" }, { "path": "github.com/go-opentype/opentype", - "version": "v0.12.0" + "version": "v0.13.0" }, { "path": "github.com/go-pdfkit/pdffont", @@ -40,7 +40,7 @@ }, { "path": "github.com/go-pdfkit/render", - "version": "v0.35.0" + "version": "v0.67.0" }, { "path": "github.com/sergeymakinen/go-bmp", @@ -70,6 +70,8 @@ "refused": 0, "unopenable": 8, "declined": 0, + "unseen": 4, + "repeated": 2, "filters": [ { "filter": "DCTDecode", @@ -200,8 +202,8 @@ "worst": 0.20572634482547583 }, "mean": { - "median": -0.0020404003643650142, - "worst": -0.0020404003643650142 + "median": -0.002066629399579133, + "worst": -0.002066629399579133 } } } diff --git a/baseline/pdfforms-gh-pdfcpu.json b/baseline/pdfforms-gh-pdfcpu.json index 66b49ca..54ae229 100644 --- a/baseline/pdfforms-gh-pdfcpu.json +++ b/baseline/pdfforms-gh-pdfcpu.json @@ -1,6 +1,6 @@ { "corpus": "/Users/Shared/pdfforms", - "taken": "2026-09-24T17:09:38Z", + "taken": "2026-10-04T09:38:49Z", "pages": 1, "gate": 2, "timeout": "2m0s", @@ -8,11 +8,11 @@ "modules": [ { "path": "github.com/ajroetker/go-highway", - "version": "v0.0.4" + "version": "v0.0.12" }, { "path": "github.com/go-gfx/gfx", - "version": "v0.31.0" + "version": "v0.34.0" }, { "path": "github.com/go-images/jpeg", @@ -20,15 +20,15 @@ }, { "path": "github.com/go-images/jpeg2000", - "version": "v0.1.0" + "version": "v0.13.2" }, { "path": "github.com/go-opentype/fonts", - "version": "v0.9.0" + "version": "v0.10.0" }, { "path": "github.com/go-opentype/opentype", - "version": "v0.12.0" + "version": "v0.13.0" }, { "path": "github.com/go-pdfkit/pdffont", @@ -40,7 +40,7 @@ }, { "path": "github.com/go-pdfkit/render", - "version": "v0.35.0" + "version": "v0.67.0" }, { "path": "github.com/sergeymakinen/go-bmp", diff --git a/baseline/pdfforms-gh-pypdf.json b/baseline/pdfforms-gh-pypdf.json index 96aec3f..06e5665 100644 --- a/baseline/pdfforms-gh-pypdf.json +++ b/baseline/pdfforms-gh-pypdf.json @@ -1,6 +1,6 @@ { "corpus": "/Users/Shared/pdfforms", - "taken": "2026-09-24T17:09:51Z", + "taken": "2026-10-04T09:39:03Z", "pages": 1, "gate": 2, "timeout": "2m0s", @@ -8,11 +8,11 @@ "modules": [ { "path": "github.com/ajroetker/go-highway", - "version": "v0.0.4" + "version": "v0.0.12" }, { "path": "github.com/go-gfx/gfx", - "version": "v0.31.0" + "version": "v0.34.0" }, { "path": "github.com/go-images/jpeg", @@ -20,15 +20,15 @@ }, { "path": "github.com/go-images/jpeg2000", - "version": "v0.1.0" + "version": "v0.13.2" }, { "path": "github.com/go-opentype/fonts", - "version": "v0.9.0" + "version": "v0.10.0" }, { "path": "github.com/go-opentype/opentype", - "version": "v0.12.0" + "version": "v0.13.0" }, { "path": "github.com/go-pdfkit/pdffont", @@ -40,7 +40,7 @@ }, { "path": "github.com/go-pdfkit/render", - "version": "v0.35.0" + "version": "v0.67.0" }, { "path": "github.com/sergeymakinen/go-bmp", @@ -70,6 +70,7 @@ "refused": 0, "unopenable": 1, "declined": 0, + "repeated": 30, "filters": [ { "filter": "DCTDecode", diff --git a/baseline/pdfforms-gh-qpdf.json b/baseline/pdfforms-gh-qpdf.json index b66a1a1..51ec454 100644 --- a/baseline/pdfforms-gh-qpdf.json +++ b/baseline/pdfforms-gh-qpdf.json @@ -1,6 +1,6 @@ { "corpus": "/Users/Shared/pdfforms", - "taken": "2026-09-24T17:09:52Z", + "taken": "2026-10-04T09:39:04Z", "pages": 1, "gate": 2, "timeout": "2m0s", @@ -8,11 +8,11 @@ "modules": [ { "path": "github.com/ajroetker/go-highway", - "version": "v0.0.4" + "version": "v0.0.12" }, { "path": "github.com/go-gfx/gfx", - "version": "v0.31.0" + "version": "v0.34.0" }, { "path": "github.com/go-images/jpeg", @@ -20,15 +20,15 @@ }, { "path": "github.com/go-images/jpeg2000", - "version": "v0.1.0" + "version": "v0.13.2" }, { "path": "github.com/go-opentype/fonts", - "version": "v0.9.0" + "version": "v0.10.0" }, { "path": "github.com/go-opentype/opentype", - "version": "v0.12.0" + "version": "v0.13.0" }, { "path": "github.com/go-pdfkit/pdffont", @@ -40,7 +40,7 @@ }, { "path": "github.com/go-pdfkit/render", - "version": "v0.35.0" + "version": "v0.67.0" }, { "path": "github.com/sergeymakinen/go-bmp", @@ -70,6 +70,7 @@ "refused": 0, "unopenable": 0, "declined": 0, + "unseen": 13, "filters": [ { "filter": "DCTDecode", diff --git a/baseline/pdfforms-gh-safedocs.json b/baseline/pdfforms-gh-safedocs.json index 0f047e3..7c0fc98 100644 --- a/baseline/pdfforms-gh-safedocs.json +++ b/baseline/pdfforms-gh-safedocs.json @@ -1,6 +1,6 @@ { "corpus": "/Users/Shared/pdfforms", - "taken": "2026-09-24T17:09:53Z", + "taken": "2026-10-04T09:39:04Z", "pages": 1, "gate": 2, "timeout": "2m0s", @@ -8,11 +8,11 @@ "modules": [ { "path": "github.com/ajroetker/go-highway", - "version": "v0.0.4" + "version": "v0.0.12" }, { "path": "github.com/go-gfx/gfx", - "version": "v0.31.0" + "version": "v0.34.0" }, { "path": "github.com/go-images/jpeg", @@ -20,15 +20,15 @@ }, { "path": "github.com/go-images/jpeg2000", - "version": "v0.1.0" + "version": "v0.13.2" }, { "path": "github.com/go-opentype/fonts", - "version": "v0.9.0" + "version": "v0.10.0" }, { "path": "github.com/go-opentype/opentype", - "version": "v0.12.0" + "version": "v0.13.0" }, { "path": "github.com/go-pdfkit/pdffont", @@ -40,7 +40,7 @@ }, { "path": "github.com/go-pdfkit/render", - "version": "v0.35.0" + "version": "v0.67.0" }, { "path": "github.com/sergeymakinen/go-bmp", diff --git a/baseline/pdfforms-gh-verapdf.json b/baseline/pdfforms-gh-verapdf.json index e627771..003afae 100644 --- a/baseline/pdfforms-gh-verapdf.json +++ b/baseline/pdfforms-gh-verapdf.json @@ -1,6 +1,6 @@ { "corpus": "/Users/Shared/pdfforms", - "taken": "2026-09-24T17:09:53Z", + "taken": "2026-10-04T09:39:05Z", "pages": 1, "gate": 2, "timeout": "2m0s", @@ -8,11 +8,11 @@ "modules": [ { "path": "github.com/ajroetker/go-highway", - "version": "v0.0.4" + "version": "v0.0.12" }, { "path": "github.com/go-gfx/gfx", - "version": "v0.31.0" + "version": "v0.34.0" }, { "path": "github.com/go-images/jpeg", @@ -20,15 +20,15 @@ }, { "path": "github.com/go-images/jpeg2000", - "version": "v0.1.0" + "version": "v0.13.2" }, { "path": "github.com/go-opentype/fonts", - "version": "v0.9.0" + "version": "v0.10.0" }, { "path": "github.com/go-opentype/opentype", - "version": "v0.12.0" + "version": "v0.13.0" }, { "path": "github.com/go-pdfkit/pdffont", @@ -40,7 +40,7 @@ }, { "path": "github.com/go-pdfkit/render", - "version": "v0.35.0" + "version": "v0.67.0" }, { "path": "github.com/sergeymakinen/go-bmp", diff --git a/baseline/pdfforms-int-wipo.json b/baseline/pdfforms-int-wipo.json index 66572e8..331e8d3 100644 --- a/baseline/pdfforms-int-wipo.json +++ b/baseline/pdfforms-int-wipo.json @@ -1,6 +1,6 @@ { "corpus": "/Users/Shared/pdfforms", - "taken": "2026-09-24T17:09:53Z", + "taken": "2026-10-04T09:39:05Z", "pages": 1, "gate": 2, "timeout": "2m0s", @@ -8,11 +8,11 @@ "modules": [ { "path": "github.com/ajroetker/go-highway", - "version": "v0.0.4" + "version": "v0.0.12" }, { "path": "github.com/go-gfx/gfx", - "version": "v0.31.0" + "version": "v0.34.0" }, { "path": "github.com/go-images/jpeg", @@ -20,15 +20,15 @@ }, { "path": "github.com/go-images/jpeg2000", - "version": "v0.1.0" + "version": "v0.13.2" }, { "path": "github.com/go-opentype/fonts", - "version": "v0.9.0" + "version": "v0.10.0" }, { "path": "github.com/go-opentype/opentype", - "version": "v0.12.0" + "version": "v0.13.0" }, { "path": "github.com/go-pdfkit/pdffont", @@ -40,7 +40,7 @@ }, { "path": "github.com/go-pdfkit/render", - "version": "v0.35.0" + "version": "v0.67.0" }, { "path": "github.com/sergeymakinen/go-bmp", diff --git a/baseline/pdfforms-uk-govuk.json b/baseline/pdfforms-uk-govuk.json index 180e7f5..ccb0189 100644 --- a/baseline/pdfforms-uk-govuk.json +++ b/baseline/pdfforms-uk-govuk.json @@ -1,6 +1,6 @@ { "corpus": "/Users/Shared/pdfforms", - "taken": "2026-09-24T17:09:53Z", + "taken": "2026-10-04T09:39:05Z", "pages": 1, "gate": 2, "timeout": "2m0s", @@ -8,11 +8,11 @@ "modules": [ { "path": "github.com/ajroetker/go-highway", - "version": "v0.0.4" + "version": "v0.0.12" }, { "path": "github.com/go-gfx/gfx", - "version": "v0.31.0" + "version": "v0.34.0" }, { "path": "github.com/go-images/jpeg", @@ -20,15 +20,15 @@ }, { "path": "github.com/go-images/jpeg2000", - "version": "v0.1.0" + "version": "v0.13.2" }, { "path": "github.com/go-opentype/fonts", - "version": "v0.9.0" + "version": "v0.10.0" }, { "path": "github.com/go-opentype/opentype", - "version": "v0.12.0" + "version": "v0.13.0" }, { "path": "github.com/go-pdfkit/pdffont", @@ -40,7 +40,7 @@ }, { "path": "github.com/go-pdfkit/render", - "version": "v0.35.0" + "version": "v0.67.0" }, { "path": "github.com/sergeymakinen/go-bmp", diff --git a/baseline/pdfforms-us-dol.json b/baseline/pdfforms-us-dol.json index c0a8907..c64f445 100644 --- a/baseline/pdfforms-us-dol.json +++ b/baseline/pdfforms-us-dol.json @@ -1,6 +1,6 @@ { "corpus": "/Users/Shared/pdfforms", - "taken": "2026-09-24T17:10:44Z", + "taken": "2026-10-04T09:39:46Z", "pages": 1, "gate": 2, "timeout": "2m0s", @@ -8,11 +8,11 @@ "modules": [ { "path": "github.com/ajroetker/go-highway", - "version": "v0.0.4" + "version": "v0.0.12" }, { "path": "github.com/go-gfx/gfx", - "version": "v0.31.0" + "version": "v0.34.0" }, { "path": "github.com/go-images/jpeg", @@ -20,15 +20,15 @@ }, { "path": "github.com/go-images/jpeg2000", - "version": "v0.1.0" + "version": "v0.13.2" }, { "path": "github.com/go-opentype/fonts", - "version": "v0.9.0" + "version": "v0.10.0" }, { "path": "github.com/go-opentype/opentype", - "version": "v0.12.0" + "version": "v0.13.0" }, { "path": "github.com/go-pdfkit/pdffont", @@ -40,7 +40,7 @@ }, { "path": "github.com/go-pdfkit/render", - "version": "v0.35.0" + "version": "v0.67.0" }, { "path": "github.com/sergeymakinen/go-bmp", @@ -70,6 +70,7 @@ "refused": 0, "unopenable": 0, "declined": 0, + "repeated": 1, "filters": [ { "filter": "DCTDecode", diff --git a/baseline/pdfforms-us-irs.json b/baseline/pdfforms-us-irs.json index 5dec769..e57ec09 100644 --- a/baseline/pdfforms-us-irs.json +++ b/baseline/pdfforms-us-irs.json @@ -1,6 +1,6 @@ { "corpus": "/Users/Shared/pdfforms", - "taken": "2026-09-24T17:10:46Z", + "taken": "2026-10-04T09:39:48Z", "pages": 1, "gate": 2, "timeout": "2m0s", @@ -8,11 +8,11 @@ "modules": [ { "path": "github.com/ajroetker/go-highway", - "version": "v0.0.4" + "version": "v0.0.12" }, { "path": "github.com/go-gfx/gfx", - "version": "v0.31.0" + "version": "v0.34.0" }, { "path": "github.com/go-images/jpeg", @@ -20,15 +20,15 @@ }, { "path": "github.com/go-images/jpeg2000", - "version": "v0.1.0" + "version": "v0.13.2" }, { "path": "github.com/go-opentype/fonts", - "version": "v0.9.0" + "version": "v0.10.0" }, { "path": "github.com/go-opentype/opentype", - "version": "v0.12.0" + "version": "v0.13.0" }, { "path": "github.com/go-pdfkit/pdffont", @@ -40,7 +40,7 @@ }, { "path": "github.com/go-pdfkit/render", - "version": "v0.35.0" + "version": "v0.67.0" }, { "path": "github.com/sergeymakinen/go-bmp", diff --git a/baseline/pdfforms-us-opm.json b/baseline/pdfforms-us-opm.json index 67f8fee..fa01bfd 100644 --- a/baseline/pdfforms-us-opm.json +++ b/baseline/pdfforms-us-opm.json @@ -1,6 +1,6 @@ { "corpus": "/Users/Shared/pdfforms", - "taken": "2026-09-24T17:10:47Z", + "taken": "2026-10-04T09:39:50Z", "pages": 1, "gate": 2, "timeout": "2m0s", @@ -8,11 +8,11 @@ "modules": [ { "path": "github.com/ajroetker/go-highway", - "version": "v0.0.4" + "version": "v0.0.12" }, { "path": "github.com/go-gfx/gfx", - "version": "v0.31.0" + "version": "v0.34.0" }, { "path": "github.com/go-images/jpeg", @@ -20,15 +20,15 @@ }, { "path": "github.com/go-images/jpeg2000", - "version": "v0.1.0" + "version": "v0.13.2" }, { "path": "github.com/go-opentype/fonts", - "version": "v0.9.0" + "version": "v0.10.0" }, { "path": "github.com/go-opentype/opentype", - "version": "v0.12.0" + "version": "v0.13.0" }, { "path": "github.com/go-pdfkit/pdffont", @@ -40,7 +40,7 @@ }, { "path": "github.com/go-pdfkit/render", - "version": "v0.35.0" + "version": "v0.67.0" }, { "path": "github.com/sergeymakinen/go-bmp", diff --git a/baseline/pdfforms-us-ssa.json b/baseline/pdfforms-us-ssa.json index db55617..bf7c397 100644 --- a/baseline/pdfforms-us-ssa.json +++ b/baseline/pdfforms-us-ssa.json @@ -1,6 +1,6 @@ { "corpus": "/Users/Shared/pdfforms", - "taken": "2026-09-24T17:10:49Z", + "taken": "2026-10-04T09:39:51Z", "pages": 1, "gate": 2, "timeout": "2m0s", @@ -8,11 +8,11 @@ "modules": [ { "path": "github.com/ajroetker/go-highway", - "version": "v0.0.4" + "version": "v0.0.12" }, { "path": "github.com/go-gfx/gfx", - "version": "v0.31.0" + "version": "v0.34.0" }, { "path": "github.com/go-images/jpeg", @@ -20,15 +20,15 @@ }, { "path": "github.com/go-images/jpeg2000", - "version": "v0.1.0" + "version": "v0.13.2" }, { "path": "github.com/go-opentype/fonts", - "version": "v0.9.0" + "version": "v0.10.0" }, { "path": "github.com/go-opentype/opentype", - "version": "v0.12.0" + "version": "v0.13.0" }, { "path": "github.com/go-pdfkit/pdffont", @@ -40,7 +40,7 @@ }, { "path": "github.com/go-pdfkit/render", - "version": "v0.35.0" + "version": "v0.67.0" }, { "path": "github.com/sergeymakinen/go-bmp", diff --git a/baseline/pdfforms-us-uscis.json b/baseline/pdfforms-us-uscis.json index 45bc0c7..ddde97e 100644 --- a/baseline/pdfforms-us-uscis.json +++ b/baseline/pdfforms-us-uscis.json @@ -1,6 +1,6 @@ { "corpus": "/Users/Shared/pdfforms", - "taken": "2026-09-24T17:10:49Z", + "taken": "2026-10-04T09:39:52Z", "pages": 1, "gate": 2, "timeout": "2m0s", @@ -8,11 +8,11 @@ "modules": [ { "path": "github.com/ajroetker/go-highway", - "version": "v0.0.4" + "version": "v0.0.12" }, { "path": "github.com/go-gfx/gfx", - "version": "v0.31.0" + "version": "v0.34.0" }, { "path": "github.com/go-images/jpeg", @@ -20,15 +20,15 @@ }, { "path": "github.com/go-images/jpeg2000", - "version": "v0.1.0" + "version": "v0.13.2" }, { "path": "github.com/go-opentype/fonts", - "version": "v0.9.0" + "version": "v0.10.0" }, { "path": "github.com/go-opentype/opentype", - "version": "v0.12.0" + "version": "v0.13.0" }, { "path": "github.com/go-pdfkit/pdffont", @@ -40,7 +40,7 @@ }, { "path": "github.com/go-pdfkit/render", - "version": "v0.35.0" + "version": "v0.67.0" }, { "path": "github.com/sergeymakinen/go-bmp", @@ -70,6 +70,7 @@ "refused": 0, "unopenable": 0, "declined": 0, + "unseen": 73, "filters": [ { "filter": "(samples)", diff --git a/baseline/pdfforms-us-uscourts.json b/baseline/pdfforms-us-uscourts.json index f8f51ba..f766490 100644 --- a/baseline/pdfforms-us-uscourts.json +++ b/baseline/pdfforms-us-uscourts.json @@ -1,6 +1,6 @@ { "corpus": "/Users/Shared/pdfforms", - "taken": "2026-09-24T17:11:04Z", + "taken": "2026-10-04T09:40:07Z", "pages": 1, "gate": 2, "timeout": "2m0s", @@ -8,11 +8,11 @@ "modules": [ { "path": "github.com/ajroetker/go-highway", - "version": "v0.0.4" + "version": "v0.0.12" }, { "path": "github.com/go-gfx/gfx", - "version": "v0.31.0" + "version": "v0.34.0" }, { "path": "github.com/go-images/jpeg", @@ -20,15 +20,15 @@ }, { "path": "github.com/go-images/jpeg2000", - "version": "v0.1.0" + "version": "v0.13.2" }, { "path": "github.com/go-opentype/fonts", - "version": "v0.9.0" + "version": "v0.10.0" }, { "path": "github.com/go-opentype/opentype", - "version": "v0.12.0" + "version": "v0.13.0" }, { "path": "github.com/go-pdfkit/pdffont", @@ -40,7 +40,7 @@ }, { "path": "github.com/go-pdfkit/render", - "version": "v0.35.0" + "version": "v0.67.0" }, { "path": "github.com/sergeymakinen/go-bmp", @@ -70,6 +70,7 @@ "refused": 0, "unopenable": 0, "declined": 0, + "repeated": 2, "filters": [ { "filter": "(samples) mask", diff --git a/baseline/pdfscans-ia-americana.json b/baseline/pdfscans-ia-americana.json index 7e524f8..d5aff6d 100644 --- a/baseline/pdfscans-ia-americana.json +++ b/baseline/pdfscans-ia-americana.json @@ -1,6 +1,6 @@ { "corpus": "/Users/Shared/pdfscans", - "taken": "2026-09-24T20:07:32Z", + "taken": "2026-10-04T10:08:26Z", "pages": 1, "gate": 2, "timeout": "2m0s", @@ -8,11 +8,11 @@ "modules": [ { "path": "github.com/ajroetker/go-highway", - "version": "v0.0.4" + "version": "v0.0.12" }, { "path": "github.com/go-gfx/gfx", - "version": "v0.31.0" + "version": "v0.34.0" }, { "path": "github.com/go-images/jpeg", @@ -20,15 +20,15 @@ }, { "path": "github.com/go-images/jpeg2000", - "version": "v0.1.0" + "version": "v0.13.2" }, { "path": "github.com/go-opentype/fonts", - "version": "v0.9.0" + "version": "v0.10.0" }, { "path": "github.com/go-opentype/opentype", - "version": "v0.12.0" + "version": "v0.13.0" }, { "path": "github.com/go-pdfkit/pdffont", @@ -40,7 +40,7 @@ }, { "path": "github.com/go-pdfkit/render", - "version": "v0.35.0" + "version": "v0.67.0" }, { "path": "github.com/sergeymakinen/go-bmp", @@ -70,6 +70,7 @@ "refused": 0, "unopenable": 28, "declined": 0, + "unseen": 24, "filters": [ { "filter": "DCTDecode", diff --git a/baseline/pdfscans-ia-biodiversity.json b/baseline/pdfscans-ia-biodiversity.json index f1731a6..1ffd499 100644 --- a/baseline/pdfscans-ia-biodiversity.json +++ b/baseline/pdfscans-ia-biodiversity.json @@ -1,6 +1,6 @@ { "corpus": "/Users/Shared/pdfscans", - "taken": "2026-09-24T18:52:11Z", + "taken": "2026-10-04T10:52:41Z", "pages": 1, "gate": 2, "timeout": "2m0s", @@ -8,11 +8,11 @@ "modules": [ { "path": "github.com/ajroetker/go-highway", - "version": "v0.0.4" + "version": "v0.0.12" }, { "path": "github.com/go-gfx/gfx", - "version": "v0.31.0" + "version": "v0.34.0" }, { "path": "github.com/go-images/jpeg", @@ -20,15 +20,15 @@ }, { "path": "github.com/go-images/jpeg2000", - "version": "v0.1.0" + "version": "v0.13.2" }, { "path": "github.com/go-opentype/fonts", - "version": "v0.9.0" + "version": "v0.10.0" }, { "path": "github.com/go-opentype/opentype", - "version": "v0.12.0" + "version": "v0.13.0" }, { "path": "github.com/go-pdfkit/pdffont", @@ -40,7 +40,7 @@ }, { "path": "github.com/go-pdfkit/render", - "version": "v0.35.0" + "version": "v0.67.0" }, { "path": "github.com/sergeymakinen/go-bmp", @@ -92,12 +92,12 @@ }, { "filter": "JPXDecode", - "pictures": 496, + "pictures": 497, "unmatched": 0, "remapped": 0, "direct": { - "pictures": 496, - "exact": 496, + "pictures": 497, + "exact": 497, "identical": 3, "inverted": 0, "differing": 0 diff --git a/baseline/pdfscans-ia-medical.json b/baseline/pdfscans-ia-medical.json index 4401c61..e1a7183 100644 --- a/baseline/pdfscans-ia-medical.json +++ b/baseline/pdfscans-ia-medical.json @@ -1,6 +1,6 @@ { "corpus": "/Users/Shared/pdfscans", - "taken": "2026-09-24T19:22:53Z", + "taken": "2026-10-04T11:19:51Z", "pages": 1, "gate": 2, "timeout": "2m0s", @@ -8,11 +8,11 @@ "modules": [ { "path": "github.com/ajroetker/go-highway", - "version": "v0.0.4" + "version": "v0.0.12" }, { "path": "github.com/go-gfx/gfx", - "version": "v0.31.0" + "version": "v0.34.0" }, { "path": "github.com/go-images/jpeg", @@ -20,15 +20,15 @@ }, { "path": "github.com/go-images/jpeg2000", - "version": "v0.1.0" + "version": "v0.13.2" }, { "path": "github.com/go-opentype/fonts", - "version": "v0.9.0" + "version": "v0.10.0" }, { "path": "github.com/go-opentype/opentype", - "version": "v0.12.0" + "version": "v0.13.0" }, { "path": "github.com/go-pdfkit/pdffont", @@ -40,7 +40,7 @@ }, { "path": "github.com/go-pdfkit/render", - "version": "v0.35.0" + "version": "v0.67.0" }, { "path": "github.com/sergeymakinen/go-bmp", @@ -70,6 +70,7 @@ "refused": 0, "unopenable": 0, "declined": 0, + "unseen": 7, "filters": [ { "filter": "DCTDecode", diff --git a/baseline/pdfscans-ia-texts.json b/baseline/pdfscans-ia-texts.json index 38b11d9..af1f5c9 100644 --- a/baseline/pdfscans-ia-texts.json +++ b/baseline/pdfscans-ia-texts.json @@ -1,6 +1,6 @@ { "corpus": "/Users/Shared/pdfscans", - "taken": "2026-09-24T20:05:18Z", + "taken": "2026-10-04T11:55:43Z", "pages": 1, "gate": 2, "timeout": "2m0s", @@ -8,11 +8,11 @@ "modules": [ { "path": "github.com/ajroetker/go-highway", - "version": "v0.0.4" + "version": "v0.0.12" }, { "path": "github.com/go-gfx/gfx", - "version": "v0.31.0" + "version": "v0.34.0" }, { "path": "github.com/go-images/jpeg", @@ -20,15 +20,15 @@ }, { "path": "github.com/go-images/jpeg2000", - "version": "v0.1.0" + "version": "v0.13.2" }, { "path": "github.com/go-opentype/fonts", - "version": "v0.9.0" + "version": "v0.10.0" }, { "path": "github.com/go-opentype/opentype", - "version": "v0.12.0" + "version": "v0.13.0" }, { "path": "github.com/go-pdfkit/pdffont", @@ -40,7 +40,7 @@ }, { "path": "github.com/go-pdfkit/render", - "version": "v0.35.0" + "version": "v0.67.0" }, { "path": "github.com/sergeymakinen/go-bmp", diff --git a/baseline/pdfscans-ia-uscourts.json b/baseline/pdfscans-ia-uscourts.json index 031fbf5..6d54c9d 100644 --- a/baseline/pdfscans-ia-uscourts.json +++ b/baseline/pdfscans-ia-uscourts.json @@ -1,6 +1,6 @@ { "corpus": "/Users/Shared/pdfscans", - "taken": "2026-09-24T20:05:42Z", + "taken": "2026-10-04T11:56:03Z", "pages": 1, "gate": 2, "timeout": "2m0s", @@ -8,11 +8,11 @@ "modules": [ { "path": "github.com/ajroetker/go-highway", - "version": "v0.0.4" + "version": "v0.0.12" }, { "path": "github.com/go-gfx/gfx", - "version": "v0.31.0" + "version": "v0.34.0" }, { "path": "github.com/go-images/jpeg", @@ -20,15 +20,15 @@ }, { "path": "github.com/go-images/jpeg2000", - "version": "v0.1.0" + "version": "v0.13.2" }, { "path": "github.com/go-opentype/fonts", - "version": "v0.9.0" + "version": "v0.10.0" }, { "path": "github.com/go-opentype/opentype", - "version": "v0.12.0" + "version": "v0.13.0" }, { "path": "github.com/go-pdfkit/pdffont", @@ -40,7 +40,7 @@ }, { "path": "github.com/go-pdfkit/render", - "version": "v0.35.0" + "version": "v0.67.0" }, { "path": "github.com/sergeymakinen/go-bmp", @@ -70,6 +70,7 @@ "refused": 0, "unopenable": 0, "declined": 0, + "unseen": 3, "filters": [ { "filter": "DCTDecode", From 4a50fcdb6b780d30fc3802e1f14d41d06ab2a2fe Mon Sep 17 00:00:00 2001 From: tannevaled Date: Sun, 4 Oct 2026 15:04:34 +0200 Subject: [PATCH 20/28] =?UTF-8?q?baseline:=20the=20Conditions=20are=20the?= =?UTF-8?q?=20fresh=20run's,=20and=20=C2=A730=20was=20wrong=20about=20the?= =?UTF-8?q?=20gate?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The re-take needed the Conditions table to follow it, and the repository said so itself: TestTheBaselineReadmeDescribesTheRecordsBesideIt failed on the first run with three rows stale -- render v0.35.0, go-gfx v0.31.0 and jpeg2000 v0.1.0 against records taken at v0.67.0, v0.34.0 and v0.13.2. One of the three is worth naming. The jpeg2000 mention the guard reads is a PROSE sentence, and my first correction put the new version on the next line of the rewrapped paragraph -- so the version was in the file and not on the row, which is exactly the case the guard's own comment says it was narrowed to catch. Reworded so the version sits on the matched line. And §30 is corrected. It argued against adding a gate, and a gate was already there -- a better one. It asserts the README's rows match the RECORDS, not go.mod, so a dependency bump alone cannot redden it; what reddens it is a re-take whose README was not amended, which is what happened here. A guard on agreement between two things this repository holds costs nothing to keep green; a guard on agreement with the outside world is a tax on every upgrade. The "these figures still hold twelve releases later" paragraph is struck: they held for thirty-two, and §33 says which two cells moved. --- baseline/README.md | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/baseline/README.md b/baseline/README.md index 8b0672f..b652d74 100644 --- a/baseline/README.md +++ b/baseline/README.md @@ -17,7 +17,7 @@ instrument. | | | |---|---| -| taken | 2026-09-24T17:08:38Z .. 2026-09-24T20:07:32Z (UTC) | +| taken | **2026-10-04T09:14Z .. 11:56Z (UTC)**, one population at a time (§33) *(the run these replace: 2026-09-24T17:08Z .. 20:07Z)* | | judge | pdfimages version 26.04.0 | | **measure** | **per channel, gate `D` = 2, count budget `N` = 0** ([conformance#16](https://github.com/go-pdfkit/conformance/issues/16)) | | **pairing** | **by object number, falling back to size only when one side published none** ([#30](https://github.com/go-pdfkit/conformance/pull/30)); a MASK by the object of the picture that names it ([#32](https://github.com/go-pdfkit/conformance/pull/32), [#34](https://github.com/go-pdfkit/conformance/pull/34)) | @@ -25,16 +25,19 @@ instrument. | **walk** | **the page's CONTENT STREAM, not its /Resources** ([render#43](https://github.com/go-pdfkit/render/pull/43)) | | **bucketing** | the listing **and** the picture's own `/ColorSpace` ([conformance#20](https://github.com/go-pdfkit/conformance/issues/20)) | | **bound on the judge** | **2m0s per document, per tool** ([conformance#21](https://github.com/go-pdfkit/conformance/issues/21)) | -| `go-pdfkit/render` | **v0.35.0** | +| `go-pdfkit/render` | **v0.67.0** *(§33; these records replace a set taken at v0.35.0, and §33 says which two cells moved)* | | `go-pdfkit/reader` | v0.6.0 | -| `go-gfx/gfx` | **v0.31.0** | +| `go-gfx/gfx` | **v0.34.0** | | `tannevaled/gobig2` | **v0.2.0** | -| `go-images/jpeg2000` | **v0.1.0** *(was `ajroetker/go-jpeg2000` v0.0.2; see §15)* | +| `go-images/jpeg2000` | **v0.13.2** *(was `ajroetker/go-jpeg2000` v0.0.2, then the fork at v0.1.0; see §15 and §25)* | | `go-images/jpeg` | **v0.1.0** *(was the standard library's `image/jpeg`; see §18)* | | pages per document | 1 (the first page of each document) | | corpora | `/Users/Shared/pdfscans` (MANIFEST.tsv), `/Users/Shared/pdfforms` (MANIFEST.tsv) | -**These figures still hold twelve releases later.** `render` has gone from +~~**These figures still hold twelve releases later.**~~ **They held for +thirty-two, and §33 is the re-take that says so: two cells of twenty-three +populations moved, one of them a signed mean in its fifth decimal place.** What +follows was written when `render` had gone from v0.35.0 to v0.47.0 since they were taken -- a real bold face where there was a faked one, and the rest changes made for speed -- and two populations re-measured at v0.43.0 came out **identical**: `ia-texts` and `fr-impots`, every bucket, every @@ -958,7 +961,8 @@ else, so the fourth component was gone before `render` saw it, which left only "components out of a third-party module, or decoding JPEG 2000 here". **There was a third option, and it is the one that was taken: fork the -module.** `github.com/go-images/jpeg2000` v0.1.0 is that fork, under the same +module.** The fork is `github.com/go-images/jpeg2000` v0.13.2 today and was +v0.1.0 when this section was written, under the same Apache-2.0 licence, with its one change stated in a `NOTICE` as section 4(b) requires and also offered upstream. A picture whose own JP2 header declares the enumerated colour space CMYK — EnumCS 12 of ITU-T T.800 Table I.1, which the @@ -2727,7 +2731,19 @@ fifth decimal place.** §30's fact was right and the risk it implied was not. The obvious fix -- a test asserting every record names the `render` version in `go.mod` -- would redden CI on every dependency bump, including the ones that change no pixel, and a gate that reddens on routine upgrades is a gate someone -removes. What is wanted is the move §28 and conformance#92 both make: make the +removes. + +**There is already a gate here, and it is the better one.** +`TestTheBaselineReadmeDescribesTheRecordsBesideIt` asserts that the version rows +of THIS FILE match the versions in the records beside it, on the row's own line. +It does not care what `go.mod` says, so a dependency bump alone cannot redden +it; what reddens it is a re-take whose README was not amended. It caught §33's +on the first run -- three rows, including one where the version had landed on +the next line of a rewrapped sentence and so was not on the row the guard reads. +That is the gate this section said it would not add, already written, and the +distinction is worth keeping: a guard on AGREEMENT BETWEEN TWO THINGS THIS +REPOSITORY HOLDS costs nothing to keep green, while a guard on agreement with +the outside world is a tax on every upgrade. What is wanted is the move §28 and conformance#92 both make: make the number **visible**, and let a reader see for themselves that a record is six days and thirty-two releases behind the code it is cited for. This table is that, until the records are re-taken. From 74fce84a5d2e91e34be099f8d86116168d3401cb Mon Sep 17 00:00:00 2001 From: tannevaled Date: Sun, 4 Oct 2026 16:06:21 +0200 Subject: [PATCH 21/28] README: correct what it says about the record, and document three counts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two paragraphs had outlived their subject, which is the hazard §29 is about, in the file a reader opens first. "refused is 4 across 3280 documents, all in ia-biodiversity, and the same four documents as before" -- it is 1, in the records re-taken 2026-10-04, and the one document is bulletinno38tasm.pdf, whose page render.Page DRAWS: it is the extraction API that refuses it under a per-page budget the Images path spends four bytes a pixel of. §31 has the measurement and render#100 holds the question. "The instrument folds cannot-read and declined-to-decode into one count and should not" -- it no longer asserts the other half either. Ours is documented as "ours produced nothing and the judge did", and the page path set it WITHOUT ASKING poppler. It asks now, and the first thing that changed was a test demanding Ours for page nine of a one-page document. "The bound is unexercised by images on this corpus" -- §22 struck that months ago: the bound fired three times and it is a property of the machine, not of the corpus. The narrower true thing is that none of these 23 records has a hung entry. And the three counts the records now carry are described where the record is described: sizePaired, unseen and repeated, with what each is FOR and what it reads today -- 0, 4 277 and 3 371. All omitempty. --- README.md | 49 +++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 39 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 40f14bf..68a2453 100644 --- a/README.md +++ b/README.md @@ -690,18 +690,31 @@ are `DCTDecode` — the four `/CalRGB` pictures the issue named — and one is `(samples)`. `calibrated`, at 1802, is a different quantity and is not that count. -**And one thing the record says about itself.** `refused` is 4 across 3280 -documents, all in `ia-biodiversity`, and the same four documents as before: -`render`'s own 256-megapixel decode budget declining a page rather than a -document we cannot read. The instrument folds "cannot read" and "declined to -decode" into one count and should not; the four are named in the baseline so -nobody reads them as a coverage gap. +**And one thing the record says about itself.** ~~`refused` is 4 across 3280 +documents, all in `ia-biodiversity`, and the same four documents as before~~ — +**it is 1**, in the records re-taken 2026-10-04. One document of 3 280: +`bulletinno38tasm.pdf`, whose page names three 9 449 × 13 701 pictures against +a per-page budget the `Images` path spends four bytes a pixel of. `render.Page` +**draws** that page; it is the extraction API that refuses it. The baseline's +§31 has the measurement and +[go-pdfkit/render#100](https://github.com/go-pdfkit/render/issues/100) holds the +question. + +~~The instrument folds "cannot read" and "declined to decode" into one count and +should not.~~ It no longer asserts the other half either: `Ours` is documented +as "ours produced nothing and the judge did", and the page path used to set it +**without asking poppler at all**. It asks now — pictures back is `Ours`, a +tool that will not finish is `Hung`, a refusal is `Neither` — and the first +thing that changed was a test demanding `Ours` for page nine of a one-page +document. See §31. **`hung` is 0 in all 23 populations, and that is not "nothing hangs".** The document that hangs draws no picture on its first page, so `images` never asks -poppler about it at all; `compare` does meet it and names it. The bound is -unexercised by `images` on this corpus, and the baseline says so rather than -letting a reader infer otherwise. +poppler about it at all; `compare` does meet it and names it. ~~The bound is +unexercised by `images` on this corpus~~ — **§22 of the baseline strikes that: +the bound fired three times, and it is a property of the MACHINE and not of the +corpus.** What is true of the records beside this file is the narrower thing: +none of these 23 has a `hung` entry. ## What it comes to today @@ -716,7 +729,23 @@ two runs means a regression only if everything else held. images -dir /Users/Shared/pdfscans -only ia-medical -json ``` -[`baseline/README.md`](baseline/README.md) reads it out. +[`baseline/README.md`](baseline/README.md) reads it out, and +[its §33](baseline/README.md) says how the whole set is re-taken and what it +costs — one population at a time, because the judge is held to a wall-clock +bound. + +### Three counts about the measurement rather than about a picture + +Added 2026-10-04 with the records above, and each of them is a number the +instrument needed to say about ITSELF. + +| | | +|---|---| +| `sizePaired` | how many pictures were matched to the judge's by SIZE rather than by object number. The weaker matcher, so a large share is a run whose other numbers are worth less. It is **0** here, and §32 says why that zero was not the good news it looked like. | +| `unseen` | how many pictures the judge took out that ours produced nothing for — **the one direction the pairing could not see at all**, because it walks our pictures and appends one result for each. **4 277** over 23 populations; §32 names the three causes and [render#101](https://github.com/go-pdfkit/render/issues/101) and [#104](https://github.com/go-pdfkit/render/issues/104) hold two of them. | +| `repeated` | how many of the judge's rows name an object we returned once. `pdfimages` lists one row per DRAW and `Images` returns one entry per object, so this is a difference of **unit** and not of fidelity. **3 371**, and counting it apart is what keeps `unseen` meaningful. | + +All three are `omitempty`, so a record written before them keeps its shape. ### Every population runs, and has since v0.20.0 From 025ec7c203095e548aa1f2c52ad0a1c12bd7a2e8 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Sun, 4 Oct 2026 16:19:21 +0200 Subject: [PATCH 22/28] =?UTF-8?q?baseline:=20=C2=A734,=20the=20page=20comp?= =?UTF-8?q?ared=20against=20poppler=20over=20the=20whole=20corpus?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Everything above §34 measures extracted pictures -- images against pdfimages, 23 records, every codec drilled to one picture -- and §24 measures time. The page itself was never recorded: baseline/pages/ held one population, as a control of one binary against itself. compare against pdftoppm at 72 dpi with -cropbox, one population at a time, 2026-10-04, at the go.mod of the records beside it. All 23 raw outputs are in baseline/pages/, unedited. 3214 pages compared, 66 not 3001 of 3214 differ on under 1% of their pixels (93.4%) the MEDIAN is at or under 0.35% in every population, 0.000% in five The 66 not compared are 63 refused, 2 of different pixel sizes and 1 hung, and the 63 are the records' unopenable 63 by an instrument that renders instead of extracting. That this run's total is ALSO 66 is a coincidence of three causes rather than §27's 66, and §34 says so, because the number invites the mistake. The one hung makes concrete the paragraph the top-level README carries: images reports hung 0 in all 23 records because that document draws no picture on its first page and the extraction harness never asks poppler about it. compare meets it. And §34 says not to read a low byte-identical share as a defect. ia-medical is 15.69% byte-identical against gh-verapdf's 99.60% and is not twenty times worse -- its median difference is 0.000% of pixels. A 9000-pixel scan reduced to 600 is a 15x to 23x downscale, where any resampling difference puts a level into most pixels and byte-identity collapses while the picture does not. The worst page of the corpus is ia-americana's 19.23% and is not diagnosed here. Sixth worst is gh-qpdf's 8.54%, which render#103 takes to 0.03%: these records are at the go.mod as it stands, and §33 says the same of the row it moves. Two things these files are not, both named: the timing columns were measured while the load moved between 7 and 59 and are not to be read -- only the fidelity columns are deterministic, which is the sole reason the run is usable -- and they are text, because compare has no -json. That is a real gap against this repository's own "a number that is not written down cannot be regressed against", and §34 records why it was not closed here: the record type and its helpers live in cmd/images, package main, so compare cannot import them, and moving them touches files an unmerged pull request is changing. --- baseline/README.md | 105 ++++++++++++++++++++ baseline/pages/README.md | 12 ++- baseline/pages/pdfforms-ca-cra.txt | 25 +++++ baseline/pages/pdfforms-fr-cerfa.txt | 25 +++++ baseline/pages/pdfforms-fr-impots.txt | 25 +++++ baseline/pages/pdfforms-gh-openpdf.txt | 26 +++++ baseline/pages/pdfforms-gh-pdfbox.txt | 26 +++++ baseline/pages/pdfforms-gh-pdfcpu.txt | 25 +++++ baseline/pages/pdfforms-gh-pypdf.txt | 27 +++++ baseline/pages/pdfforms-gh-qpdf.txt | 27 +++++ baseline/pages/pdfforms-gh-safedocs.txt | 27 +++++ baseline/pages/pdfforms-gh-verapdf.txt | 25 +++++ baseline/pages/pdfforms-int-wipo.txt | 25 +++++ baseline/pages/pdfforms-uk-govuk.txt | 25 +++++ baseline/pages/pdfforms-us-dol.txt | 25 +++++ baseline/pages/pdfforms-us-irs.txt | 25 +++++ baseline/pages/pdfforms-us-opm.txt | 25 +++++ baseline/pages/pdfforms-us-ssa.txt | 25 +++++ baseline/pages/pdfforms-us-uscis.txt | 25 +++++ baseline/pages/pdfforms-us-uscourts.txt | 25 +++++ baseline/pages/pdfscans-ia-americana.txt | 26 +++++ baseline/pages/pdfscans-ia-biodiversity.txt | 25 +++++ baseline/pages/pdfscans-ia-medical.txt | 25 +++++ baseline/pages/pdfscans-ia-texts.txt | 19 ++++ baseline/pages/pdfscans-ia-uscourts.txt | 25 +++++ 25 files changed, 694 insertions(+), 1 deletion(-) create mode 100644 baseline/pages/pdfforms-ca-cra.txt create mode 100644 baseline/pages/pdfforms-fr-cerfa.txt create mode 100644 baseline/pages/pdfforms-fr-impots.txt create mode 100644 baseline/pages/pdfforms-gh-openpdf.txt create mode 100644 baseline/pages/pdfforms-gh-pdfbox.txt create mode 100644 baseline/pages/pdfforms-gh-pdfcpu.txt create mode 100644 baseline/pages/pdfforms-gh-pypdf.txt create mode 100644 baseline/pages/pdfforms-gh-qpdf.txt create mode 100644 baseline/pages/pdfforms-gh-safedocs.txt create mode 100644 baseline/pages/pdfforms-gh-verapdf.txt create mode 100644 baseline/pages/pdfforms-int-wipo.txt create mode 100644 baseline/pages/pdfforms-uk-govuk.txt create mode 100644 baseline/pages/pdfforms-us-dol.txt create mode 100644 baseline/pages/pdfforms-us-irs.txt create mode 100644 baseline/pages/pdfforms-us-opm.txt create mode 100644 baseline/pages/pdfforms-us-ssa.txt create mode 100644 baseline/pages/pdfforms-us-uscis.txt create mode 100644 baseline/pages/pdfforms-us-uscourts.txt create mode 100644 baseline/pages/pdfscans-ia-americana.txt create mode 100644 baseline/pages/pdfscans-ia-biodiversity.txt create mode 100644 baseline/pages/pdfscans-ia-medical.txt create mode 100644 baseline/pages/pdfscans-ia-texts.txt create mode 100644 baseline/pages/pdfscans-ia-uscourts.txt diff --git a/baseline/README.md b/baseline/README.md index b652d74..5d30919 100644 --- a/baseline/README.md +++ b/baseline/README.md @@ -3226,3 +3226,108 @@ incomplete `/Resources` was hiding, all ten exact and byte-identical. These records are taken at the `go.mod` as it stands, so that row is **expected to move** the moment that lands, and this is the sentence that says so before it does. + +## §34 — The page compared against poppler, which this document had never recorded + +Everything above measures **extracted pictures**: `images` against `pdfimages`, +23 records, every codec drilled to one picture. §24 measures **time**. The one +thing missing was the page itself — and `baseline/pages/` held exactly one +population, as a control of one binary against itself (§21). + +So the page is recorded now. `compare` against `pdftoppm` at 72 dpi with +`-cropbox`, one population at a time, 2026-10-04, at the `go.mod` of the records +beside it (`render` v0.67.0, `go-images/jpeg2000` v0.13.2) and the same +`pdftoppm` 26.04.0. The raw output of all twenty-three runs is in +`baseline/pages/`, unedited. + +| population | compared | not | median | worst | byte-identical | mean \|diff\| | under 1% | worst page | +|---|---:|---:|---:|---:|---:|---:|---:|---| +| `ca-cra` | 84 | 0 | 0.200% | 4.00% | **84.51%** | 9.58 | 83 | `t2222-fill-21f.pdf` 4.00% | +| `fr-cerfa` | 450 | 0 | 0.110% | 6.70% | **85.07%** | 6.60 | 391 | `cerfa_12468.pdf` 6.70% | +| `fr-impots` | 50 | 0 | 0.000% | 1.09% | **65.22%** | 4.84 | 49 | `2735_2735_5088.pdf` 1.09% | +| `gh-openpdf` | 42 | 14 | 0.000% | 2.17% | **97.19%** | 1.36 | 41 | `openpdf-core_src_test_resources_pdf-2-0_PDF_` 2.17% | +| `gh-pdfbox` | 149 | 8 | 0.010% | 11.11% | **93.88%** | 2.56 | 144 | `PDFBOX-3127-RAU4G6QMOVRYBISJU7R6MOVZCRFUO7P4` 11.11% | +| `gh-pdfcpu` | 147 | 0 | 0.240% | 7.41% | **83.62%** | 2.40 | 141 | `form_primitives_comboboxGroup.pdf` 7.41% | +| `gh-pypdf` | 32 | 2 | 0.000% | 0.68% | **87.45%** | 3.57 | 32 | `019-grayscale-image_grayscale-image.pdf` 0.68% | +| `gh-qpdf` | 80 | 1 | 0.030% | 8.54% | **93.19%** | 1.61 | 76 | `qpdf_qtest_qpdf_form-xobjects-some-resources` 8.54% | +| `gh-safedocs` | 20 | 6 | 0.250% | 13.57% | **90.27%** | 4.32 | 14 | `Miscellaneous_Targeted_Test_PDFs_PatternText` 13.57% | +| `gh-verapdf` | 134 | 0 | 0.000% | 0.56% | **99.60%** | 0.12 | 134 | `PDF_A-1b_6.9_Interactive_Forms_veraPDF_test_` 0.56% | +| `int-wipo` | 116 | 0 | 0.010% | 0.34% | **89.83%** | 5.01 | 116 | `docs-fr-forms-ro-editable-ed-ro154.pdf` 0.34% | +| `uk-govuk` | 302 | 0 | 0.110% | 1.03% | **84.39%** | 5.97 | 301 | `lasting-power-of-attorney-forms__20250821-LP` 1.03% | +| `us-dol` | 140 | 0 | 0.350% | 1.95% | **84.03%** | 10.45 | 118 | `2025-form-5500-ez.pdf` 1.95% | +| `us-irs` | 69 | 0 | 0.080% | 0.43% | **80.63%** | 13.04 | 69 | `f720.pdf` 0.43% | +| `us-opm` | 66 | 0 | 0.120% | 5.64% | **83.97%** | 8.73 | 64 | `sf2820.pdf` 5.64% | +| `us-ssa` | 199 | 0 | 0.130% | 0.70% | **83.98%** | 9.67 | 199 | `ssa-454-bk.pdf` 0.70% | +| `us-uscis` | 88 | 0 | 0.150% | 0.44% | **85.29%** | 11.46 | 88 | `i-914.pdf` 0.44% | +| `us-uscourts` | 69 | 0 | 0.340% | 10.49% | **92.12%** | 4.69 | 55 | `ao038.pdf` 10.49% | +| `ia-americana` | 222 | 28 | 0.230% | 19.23% | **23.52%** | 11.74 | 179 | `sim_unitarian-register-and-the-universalist-` 19.23% | +| `ia-biodiversity` | 250 | 0 | 0.200% | 9.90% | **32.18%** | 11.23 | 220 | `bidragtillknne00falc.pdf` 9.90% | +| `ia-medical` | 250 | 0 | 0.000% | 3.30% | **15.69%** | 6.48 | 246 | `childrenwithorth00shar.pdf` 3.30% | +| `ia-texts` | 5 | 7 | 0.010% | 2.02% | **22.82%** | 10.40 | 4 | `bdrc-W1KG4230.pdf` 2.02% | +| `ia-uscourts` | 250 | 0 | 0.050% | 2.21% | **85.36%** | 5.21 | 237 | `gov.uscourts.casd.850253.pdf` 2.21% | +| **all 23** | **3214** | **66** | — | — | — | — | **3001** | — | + +**93.4% of the 3214 pages compared differ from poppler on under 1% of their +pixels**, and the MEDIAN is at or under 0.35% in every one of the twenty-three +populations — 0.000% in five of them. + +### What the sixty-six not compared are, and why that number keeps appearing + + 63 refused -- neither implementation opens the file + 2 different sizes -- the two renderers produced different pixel dimensions + 1 hung -- pdftoppm did not finish within the bound + +The 63 are the records' `unopenable` 63, the same set, by an instrument that +renders instead of extracting. §33 explains why §27 counts 66 of these and the +records count 63: §27 walks the directory and the records walk the MANIFEST, +which has four fewer files, three of them refusals. That this run's "not +compared" is also 66 is a coincidence of three different causes, not the same +66, and it is worth saying so plainly because the number invites the mistake. + +**The one `hung` is the paragraph the top-level README carries made concrete.** +`qpdf_qtest_qpdf_shared-unnamed-field.pdf`: `images` reports `hung: 0` in all 23 +records because that document draws no picture on its first page, so the +extraction harness never asks poppler about it at all. `compare` does meet it. +Two instruments, two populations, and neither count is wrong. + +### Do not read a low byte-identical share as a defect + +`ia-medical` is **15.69%** byte-identical and `gh-verapdf` is **99.60%**, and +the first is not twenty times worse: its median difference is **0.000%** of +pixels against veraPDF's 0.000% too. The scans are 9 000-pixel-wide pages +reduced to about 600 for the comparison — a 15× to 23× downscale, where any +difference in resampling puts a level or two into most pixels and byte-identity +collapses while the picture does not. §1 says the same thing about why this +baseline measures codecs through extraction rather than through a rasteriser. +**Byte-identity is a property of the scale asked for.** The median and the +`under 1%` column are the figures to read. + +### What the worst pages are + +`ia-americana`'s 19.23% is the worst page in the corpus and is not diagnosed +here. Below it: `gh-safedocs` 13.57% (`PatternTextInsideText.pdf` -- a pattern +used as the fill of text inside text), `gh-pdfbox` 11.11%, `us-uscourts` +10.49%, `ia-biodiversity` 9.90%, and then **`gh-qpdf` 8.54%, which +[render#103](https://github.com/go-pdfkit/render/pull/103) takes to 0.03%** -- +it is in this table because these records are taken at the `go.mod` as it +stands, and §33 says the same of the row it moves. + +### Two things these files are not + +**The timing columns are not to be read.** `compare` prints them and they are +in the raw output, measured while the load average moved between 7 and 59. +§24 is where timings live, with its own stated conditions; §22 is why a bound +under load is a property of the machine. The fidelity columns are deterministic +-- both rasterisers are -- so the load does not touch them, which is the only +reason this run is usable at all. + +**And they are text, not a record.** `images` has `-json` and twenty-three +machine-readable baselines; `compare` has neither, so this is a tabulation of +prose, exactly as §21's control is. It should have a `-json` of its own -- *a +number that is not written down cannot be regressed against* is this +repository's own sentence, and these are not written down in a form a later run +can subtract from. It was not built here because `judgeVersion`, `modules` and +the `baseline` record type live in `cmd/images`, package `main`, so `compare` +cannot import them; moving them to an internal package touches the very files +an unmerged pull request is changing. Named, with the reason, rather than +started. diff --git a/baseline/pages/README.md b/baseline/pages/README.md index 74cddb3..36cf65b 100644 --- a/baseline/pages/README.md +++ b/baseline/pages/README.md @@ -1,4 +1,14 @@ -# The page records §21 is measured from +# The page records + +**§34's twenty-three `pdfforms-*.txt` and `pdfscans-*.txt` are the whole corpus +compared against poppler**, which nothing in this repository had recorded until +2026-10-04: `compare` against `pdftoppm` at 72 dpi with `-cropbox`, one +population at a time, at the `go.mod` of the `.json` records beside this +directory. Read §34 for what they say and for the two things they are not -- +their timing columns were measured under a load average between 7 and 59 and +are not to be read, and they are text rather than a machine-readable record. + +## The two files §21 is measured from These are `compare`'s output, unedited, for the control in §21: the same 250 documents drawn twice by **one binary built twice**, current in everything but diff --git a/baseline/pages/pdfforms-ca-cra.txt b/baseline/pages/pdfforms-ca-cra.txt new file mode 100644 index 0000000..d1661e2 --- /dev/null +++ b/baseline/pages/pdfforms-ca-cra.txt @@ -0,0 +1,25 @@ +ca-cra 84 compared 0 not + pixels differing: median 0.0020 p90 0.0038 p99 0.0055 worst 0.0400 + under 1% 83 under 2% 83 under 5% 84 under 10% 84 + identical byte for byte 84.51% mean |diff| 9.583 levels + colour, levels of 255: worst square mean 17.22 max 33.74 worst pixel 193 + worst 4.00% t2222-fill-21f.pdf page 1 + worst 0.55% t4a-fill-25e.pdf page 1 + worst 0.52% t183corp-fill-23e.pdf page 1 + worst 0.51% t183corp-fill-24e.pdf page 1 + worst 0.47% t929-fill-24e.pdf page 1 + worst 0.44% t185-fill-25e.pdf page 1 + worst 0.41% t183-fill-25e.pdf page 1 + worst 0.40% gst498-fill-25e.pdf page 1 + worst 0.39% rc193-fill-25e.pdf page 1 + worst 0.38% rc325-fill-26e.pdf page 1 + worst 0.38% rc199-fill-25e.pdf page 1 + worst 0.37% t777-fill-25e.pdf page 1 + time, 84 pages both were timed on: ours 3.521s theirs 19.231s (0.183x) + per page ours/theirs: median 0.210x p90 0.298x faster on 84 of 84 + worst 0.67x faster ours 153ms theirs 228ms gst498-fill-25e.pdf page 1 + worst 0.48x faster ours 87ms theirs 180ms rc66sch-fill-26e.pdf page 1 + worst 0.40x faster ours 59ms theirs 149ms t1158-fill-25e.pdf page 1 + worst 0.35x faster ours 70ms theirs 200ms t1213-fill-25e.pdf page 1 + worst 0.34x faster ours 55ms theirs 164ms rc193-fill-25e.pdf page 1 + slowest page 229ms, 0 over the threshold diff --git a/baseline/pages/pdfforms-fr-cerfa.txt b/baseline/pages/pdfforms-fr-cerfa.txt new file mode 100644 index 0000000..a1d6bf9 --- /dev/null +++ b/baseline/pages/pdfforms-fr-cerfa.txt @@ -0,0 +1,25 @@ +fr-cerfa 450 compared 0 not + pixels differing: median 0.0011 p90 0.0129 p99 0.0536 worst 0.0670 + under 1% 391 under 2% 426 under 5% 442 under 10% 450 + identical byte for byte 85.07% mean |diff| 6.604 levels + colour, levels of 255: worst square mean 19.62 max 96.09 worst pixel 255 + worst 6.70% cerfa_12468.pdf page 1 + worst 6.52% cerfa_12453.pdf page 1 + worst 6.09% cerfa_10074.pdf page 1 + worst 5.58% cerfa_12147.pdf page 1 + worst 5.49% cerfa_11764.pdf page 1 + worst 5.36% cerfa_12622.pdf page 1 + worst 5.34% cerfa_11359.pdf page 1 + worst 5.07% cerfa_11723.pdf page 1 + worst 4.90% cerfa_11525.pdf page 1 + worst 4.90% cerfa_12146.pdf page 1 + worst 4.89% cerfa_12621.pdf page 1 + worst 4.63% cerfa_11527.pdf page 1 + time, 450 pages both were timed on: ours 18.477s theirs 1m20.522s (0.229x) + per page ours/theirs: median 0.180x p90 0.387x faster on 444 of 450 + worst 2.06x SLOWER ours 434ms theirs 210ms cerfa_10702.pdf page 1 + worst 1.14x SLOWER ours 329ms theirs 289ms cerfa_10020.pdf page 1 + worst 1.06x SLOWER ours 163ms theirs 153ms cerfa_10114.pdf page 1 + worst 1.05x SLOWER ours 158ms theirs 150ms cerfa_10115.pdf page 1 + worst 1.03x SLOWER ours 338ms theirs 328ms cerfa_12235.pdf page 1 + slowest page 434ms, 0 over the threshold diff --git a/baseline/pages/pdfforms-fr-impots.txt b/baseline/pages/pdfforms-fr-impots.txt new file mode 100644 index 0000000..2a18cf0 --- /dev/null +++ b/baseline/pages/pdfforms-fr-impots.txt @@ -0,0 +1,25 @@ +fr-impots 50 compared 0 not + pixels differing: median 0.0000 p90 0.0010 p99 0.0068 worst 0.0109 + under 1% 49 under 2% 50 under 5% 50 under 10% 50 + identical byte for byte 65.22% mean |diff| 4.839 levels + colour, levels of 255: worst square mean 15.64 max 28.35 worst pixel 182 + worst 1.09% 2735_2735_5088.pdf page 1 + worst 0.68% 2044_2044_5133.pdf page 1 + worst 0.46% 2044_2044_5484.pdf page 1 + worst 0.45% 2044_2044_4764.pdf page 1 + worst 0.14% 2042_2042_4752.pdf page 1 + worst 0.10% 2047_2047_4769.pdf page 1 + worst 0.03% 2042_2042_5126.pdf page 1 + worst 0.03% 2042_2042_5476.pdf page 1 + worst 0.03% 2042_2042_4756.pdf page 1 + worst 0.02% 2042_2042_5534.pdf page 1 + worst 0.02% 2735_2735_5089.pdf page 1 + worst 0.02% 2042_2042_5532.pdf page 1 + time, 50 pages both were timed on: ours 2.375s theirs 8.926s (0.266x) + per page ours/theirs: median 0.242x p90 0.363x faster on 50 of 50 + worst 0.59x faster ours 113ms theirs 192ms 2042_2042_5472.pdf page 1 + worst 0.43x faster ours 94ms theirs 220ms 2042_2042_5122.pdf page 1 + worst 0.40x faster ours 82ms theirs 205ms 2047_2047_4769.pdf page 1 + worst 0.39x faster ours 68ms theirs 176ms 2042_2042_4752.pdf page 1 + worst 0.36x faster ours 73ms theirs 201ms 2047_2047_5149.pdf page 1 + slowest page 113ms, 0 over the threshold diff --git a/baseline/pages/pdfforms-gh-openpdf.txt b/baseline/pages/pdfforms-gh-openpdf.txt new file mode 100644 index 0000000..e06782a --- /dev/null +++ b/baseline/pages/pdfforms-gh-openpdf.txt @@ -0,0 +1,26 @@ +gh-openpdf 42 compared 14 not + 14 refused + pixels differing: median 0.0000 p90 0.0026 p99 0.0059 worst 0.0217 + under 1% 41 under 2% 41 under 5% 42 under 10% 42 + identical byte for byte 97.19% mean |diff| 1.356 levels + colour, levels of 255: worst square mean 15.35 max 159.64 worst pixel 255 + worst 2.17% openpdf-core_src_test_resources_pdf-2-0_PDF_2.0_UTF-8_string_and_annotation.pdf page 1 + worst 0.59% openpdf-core_src_test_resources_SimulatedBoldAndStrokeWidth.pdf page 1 + worst 0.28% openpdf-core_src_test_resources_pdf-2-0_Simple_PDF_2.0_file.pdf page 1 + worst 0.27% openpdf-core_src_test_resources_pades_infinite_loop.pdf page 1 + worst 0.27% openpdf-core_src_test_resources_pades_opposite_infinite_loop.pdf page 1 + worst 0.26% pdf-toolbox_src_test_resources_org_openpdf_examples_fonts_form_PdfFormGlyphLayoutManager.pdf page 1 + worst 0.26% pdf-toolbox_src_test_resources_org_openpdf_examples_fonts_form_PdfFormLayoutProcessor.pdf page 1 + worst 0.26% pdf-toolbox_src_test_resources_pattern.pdf page 1 + worst 0.23% openpdf-core_src_test_resources_barcode_macro_pdf_417.pdf page 1 + worst 0.13% openpdf-core_src_test_resources_issue375_nontrivial-crypt-filter.pdf page 1 + worst 0.05% openpdf-core_src_test_resources_identity-h.pdf page 1 + worst 0.04% openpdf-core_src_test_resources_pdf-2-0_pdf20-utf8-test.pdf page 1 + time, 42 pages both were timed on: ours 241ms theirs 2.644s (0.091x) + per page ours/theirs: median 0.041x p90 0.149x faster on 42 of 42 + worst 0.53x faster ours 31ms theirs 58ms openpdf-core_src_test_resources_openpdf_bug_test.pdf page 1 + worst 0.33x faster ours 24ms theirs 72ms openpdf-core_src_test_resources_objectXref.pdf page 1 + worst 0.28x faster ours 27ms theirs 99ms openpdf-core_src_test_resources_pdfsmartcopy_bec.pdf page 1 + worst 0.17x faster ours 10ms theirs 59ms pdf-toolbox_src_test_resources_pattern.pdf page 1 + worst 0.15x faster ours 9ms theirs 60ms openpdf-core_src_test_resources_merge-acroforms.pdf page 1 + slowest page 31ms, 0 over the threshold diff --git a/baseline/pages/pdfforms-gh-pdfbox.txt b/baseline/pages/pdfforms-gh-pdfbox.txt new file mode 100644 index 0000000..ad4308e --- /dev/null +++ b/baseline/pages/pdfforms-gh-pdfbox.txt @@ -0,0 +1,26 @@ +gh-pdfbox 149 compared 8 not + 8 refused + pixels differing: median 0.0001 p90 0.0040 p99 0.0247 worst 0.1111 + under 1% 144 under 2% 146 under 5% 147 under 10% 148 + identical byte for byte 93.88% mean |diff| 2.565 levels + colour, levels of 255: worst square mean 12.43 max 89.26 worst pixel 255 + worst 11.11% PDFBOX-3127-RAU4G6QMOVRYBISJU7R6MOVZCRFUO7P4-VFont.pdf page 1 + worst 6.80% JBIG2Image.pdf page 1 + worst 2.47% raw_image_demo.pdf page 1 + worst 1.29% tiger-as-form-xobject.pdf page 1 + worst 1.11% jpegrgb.pdf page 1 + worst 0.99% PDFBOX-5840-410609.pdf page 1 + worst 0.71% PDFBOX-3656-SF1199AEG__Complete_.pdf page 1 + worst 0.69% PDFBOX-3123-ADSFWTRB3HBZBZKEVESVTBRZC2MNKZF5_reduced.pdf page 1 + worst 0.53% data-000001.pdf page 1 + worst 0.51% jpeg_demo.pdf page 1 + worst 0.50% PDFBOX-2984-rotations.pdf page 1 + worst 0.49% PDFBOX-6049-Source.pdf page 1 + time, 149 pages both were timed on: ours 2.195s theirs 14.219s (0.154x) + per page ours/theirs: median 0.061x p90 0.264x faster on 147 of 149 + worst 1.80x SLOWER ours 275ms theirs 153ms tiger-as-form-xobject.pdf page 1 + worst 1.54x SLOWER ours 392ms theirs 254ms AcroFormsBasicFields.pdf page 1 + worst 0.42x faster ours 55ms theirs 131ms JPXTestCMYK.pdf page 1 + worst 0.41x faster ours 40ms theirs 98ms PDFBOX-5838-0024320-reduced.pdf page 1 + worst 0.40x faster ours 38ms theirs 93ms ccitt4-cib-test.pdf page 1 + slowest page 392ms, 0 over the threshold diff --git a/baseline/pages/pdfforms-gh-pdfcpu.txt b/baseline/pages/pdfforms-gh-pdfcpu.txt new file mode 100644 index 0000000..9ddd723 --- /dev/null +++ b/baseline/pages/pdfforms-gh-pdfcpu.txt @@ -0,0 +1,25 @@ +gh-pdfcpu 147 compared 0 not + pixels differing: median 0.0024 p90 0.0044 p99 0.0183 worst 0.0741 + under 1% 141 under 2% 145 under 5% 145 under 10% 147 + identical byte for byte 83.62% mean |diff| 2.398 levels + colour, levels of 255: worst square mean 23.12 max 111.25 worst pixel 255 + worst 7.41% form_primitives_comboboxGroup.pdf page 1 + worst 5.24% form_primitives_combobox.pdf page 1 + worst 1.83% form_fill_person.pdf page 1 + worst 1.72% form_multifill_json_Jane.pdf page 1 + worst 1.72% form_multifill_json_merge_english.pdf page 1 + worst 1.03% form_multifill_json_Joe.pdf page 1 + worst 0.63% form_multifill_csv_Joe.pdf page 1 + worst 0.56% form_multifill_csv_person_25.pdf page 1 + worst 0.55% form_multifill_csv_person_10.pdf page 1 + worst 0.49% form_multifill_csv_Jane.pdf page 1 + worst 0.49% form_multifill_csv_merge_english.pdf page 1 + worst 0.47% form_multifill_csv_person_30.pdf page 1 + time, 147 pages both were timed on: ours 1.788s theirs 17.775s (0.101x) + per page ours/theirs: median 0.090x p90 0.160x faster on 147 of 147 + worst 0.24x faster ours 44ms theirs 186ms pkg_testdata_Acroforms2.pdf page 1 + worst 0.20x faster ours 25ms theirs 123ms form_primitives_listbox.pdf page 1 + worst 0.19x faster ours 19ms theirs 100ms form_demo_persian.pdf page 1 + worst 0.19x faster ours 16ms theirs 85ms form_lock_arabic-locked.pdf page 1 + worst 0.19x faster ours 25ms theirs 131ms form_primitives_textfieldGroup.pdf page 1 + slowest page 44ms, 0 over the threshold diff --git a/baseline/pages/pdfforms-gh-pypdf.txt b/baseline/pages/pdfforms-gh-pypdf.txt new file mode 100644 index 0000000..1be26ec --- /dev/null +++ b/baseline/pages/pdfforms-gh-pypdf.txt @@ -0,0 +1,27 @@ +gh-pypdf 32 compared 2 not + 1 refused + 1 different sizes + pixels differing: median 0.0000 p90 0.0012 p99 0.0065 worst 0.0068 + under 1% 32 under 2% 32 under 5% 32 under 10% 32 + identical byte for byte 87.45% mean |diff| 3.574 levels + colour, levels of 255: worst square mean 13.99 max 48.00 worst pixel 255 + worst 0.68% 019-grayscale-image_grayscale-image.pdf page 1 + worst 0.65% 027-cropped-rotated-scaled_cropped-rotated-scaled.pdf page 1 + worst 0.57% 028-image-references-deduplication_wrong-references.pdf page 1 + worst 0.18% 008-reportlab-inline-image_inline-image.pdf page 1 + worst 0.12% 018-base64-image_base64image.pdf page 1 + worst 0.10% 024-annotations_annotated_pdf.pdf page 1 + worst 0.08% 003-pdflatex-image_pdflatex-image.pdf page 1 + worst 0.06% 006-pdflatex-outline_pdflatex-outline.pdf page 1 + worst 0.06% 014-outlines_mistitled_outlines_example.pdf page 1 + worst 0.04% 022-pdfkit_pdfkit.pdf page 1 + worst 0.04% 012-libreoffice-form_libreoffice-form.pdf page 1 + worst 0.03% 011-google-doc-document_google-doc-document.pdf page 1 + time, 32 pages both were timed on: ours 435ms theirs 2.548s (0.171x) + per page ours/theirs: median 0.076x p90 0.296x faster on 32 of 32 + worst 0.52x faster ours 36ms theirs 69ms 027-cropped-rotated-scaled_cropped-rotated-scaled.pdf page 1 + worst 0.49x faster ours 61ms theirs 125ms 012-libreoffice-form_libreoffice-form.pdf page 1 + worst 0.38x faster ours 58ms theirs 152ms 004-pdflatex-4-pages_pdflatex-4-pages.pdf page 1 + worst 0.35x faster ours 49ms theirs 141ms 026-latex-multicolumn_multicolumn.pdf page 1 + worst 0.30x faster ours 15ms theirs 51ms 022-pdfkit_pdfkit.pdf page 1 + slowest page 61ms, 0 over the threshold diff --git a/baseline/pages/pdfforms-gh-qpdf.txt b/baseline/pages/pdfforms-gh-qpdf.txt new file mode 100644 index 0000000..2c8b1e0 --- /dev/null +++ b/baseline/pages/pdfforms-gh-qpdf.txt @@ -0,0 +1,27 @@ +gh-qpdf 80 compared 1 not + 1 hung + pixels differing: median 0.0003 p90 0.0039 p99 0.0854 worst 0.0854 + under 1% 76 under 2% 76 under 5% 78 under 10% 80 + identical byte for byte 93.19% mean |diff| 1.612 levels + colour, levels of 255: worst square mean 19.99 max 191.00 worst pixel 254 + worst 8.54% qpdf_qtest_qpdf_form-xobjects-some-resources2-out.pdf page 1 + worst 8.54% qpdf_qtest_qpdf_form-xobjects-some-resources2.pdf page 1 + worst 2.19% qpdf_qtest_qpdf_form-xobjects-some-resources1-out.pdf page 1 + worst 2.19% qpdf_qtest_qpdf_form-xobjects-some-resources1.pdf page 1 + worst 0.43% qpdf_qtest_qpdf_annotations-rotated-90.pdf page 1 + worst 0.43% qpdf_qtest_qpdf_rotated-shared-annotations-1.pdf page 1 + worst 0.43% qpdf_qtest_qpdf_rotated-shared-annotations-2.pdf page 1 + worst 0.39% qpdf_qtest_qpdf_annotations-rotated-270.pdf page 1 + worst 0.39% qpdf_qtest_qpdf_annotations-rotated-180.pdf page 1 + worst 0.29% qpdf_qtest_qpdf_overlay-copy-annotations-p1.pdf page 1 + worst 0.29% qpdf_qtest_qpdf_overlay-copy-annotations.pdf page 1 + worst 0.29% examples_qtest_overlay-page_annotations-out.pdf page 1 + time, 80 pages both were timed on: ours 298ms theirs 4.996s (0.060x) + per page ours/theirs: median 0.049x p90 0.093x faster on 80 of 80 + worst 0.15x faster ours 12ms theirs 82ms qpdf_qtest_qpdf_overlay-copy-annotations-p1.pdf page 1 + worst 0.14x faster ours 10ms theirs 72ms qpdf_qtest_qpdf_overlay-copy-annotations-p5.pdf page 1 + worst 0.13x faster ours 11ms theirs 81ms qpdf_qtest_qpdf_overlay-copy-annotations.pdf page 1 + worst 0.13x faster ours 8ms theirs 62ms qpdf_qtest_qpdf_overlay-copy-annotations-p2.pdf page 1 + worst 0.12x faster ours 10ms theirs 89ms examples_qtest_overlay-page_annotations-out.pdf page 1 + slowest page 12ms, 0 over the threshold + hung pdftoppm /Users/Shared/pdfforms/gh-qpdf/qpdf_qtest_qpdf_shared-unnamed-field.pdf page 1 diff --git a/baseline/pages/pdfforms-gh-safedocs.txt b/baseline/pages/pdfforms-gh-safedocs.txt new file mode 100644 index 0000000..1a8e74f --- /dev/null +++ b/baseline/pages/pdfforms-gh-safedocs.txt @@ -0,0 +1,27 @@ +gh-safedocs 20 compared 6 not + 1 different sizes + 5 refused + pixels differing: median 0.0025 p90 0.0154 p99 0.0856 worst 0.1357 + under 1% 14 under 2% 18 under 5% 18 under 10% 19 + identical byte for byte 90.27% mean |diff| 4.317 levels + colour, levels of 255: worst square mean 36.88 max 245.48 worst pixel 255 + worst 13.57% Miscellaneous_Targeted_Test_PDFs_PatternTextInsideText.pdf page 1 + worst 8.56% Inline_Image_Abbreviations_InlineAbbreviations.pdf page 1 + worst 1.54% Dialects_Dialect-ContentStreams.pdf page 1 + worst 1.54% Dialects_Dialect-ContentStreamsViaResourceNames.pdf page 1 + worst 1.54% Dialects_Dialect-ContentStreamsWithIndirectRefs.pdf page 1 + worst 1.09% CompactedSyntax_CompactedPDFSyntaxMatrix.pdf page 1 + worst 0.84% CompactedSyntax_CompactedPDFSyntaxTest.pdf page 1 + worst 0.82% Miscellaneous_Targeted_Test_PDFs_uri-ismap-test.pdf page 1 + worst 0.38% Miscellaneous_Targeted_Test_PDFs_list-of-uri-actions.pdf page 1 + worst 0.38% Miscellaneous_Targeted_Test_PDFs_list-of-uri-actions-with-base.pdf page 1 + worst 0.25% Miscellaneous_Targeted_Test_PDFs_LinkAnnot-appearances.pdf page 1 + worst 0.20% Unicode_passwords_baseline_unicode-test-actualtext.pdf page 1 + time, 20 pages both were timed on: ours 123ms theirs 1.438s (0.086x) + per page ours/theirs: median 0.041x p90 0.140x faster on 20 of 20 + worst 0.53x faster ours 38ms theirs 71ms Miscellaneous_Targeted_Test_PDFs_ContentStreamCycleType3insideType3.pdf page 1 + worst 0.17x faster ours 8ms theirs 48ms Miscellaneous_Targeted_Test_PDFs_PatternTextInsideText.pdf page 1 + worst 0.14x faster ours 25ms theirs 177ms CompactedSyntax_CompactedPDFSyntaxMatrix.pdf page 1 + worst 0.08x faster ours 6ms theirs 73ms CompactedSyntax_CompactedPDFSyntaxTest.pdf page 1 + worst 0.08x faster ours 5ms theirs 61ms Miscellaneous_Targeted_Test_PDFs_ContentStreamNoCycleType3insideType3.pdf page 1 + slowest page 38ms, 0 over the threshold diff --git a/baseline/pages/pdfforms-gh-verapdf.txt b/baseline/pages/pdfforms-gh-verapdf.txt new file mode 100644 index 0000000..1187491 --- /dev/null +++ b/baseline/pages/pdfforms-gh-verapdf.txt @@ -0,0 +1,25 @@ +gh-verapdf 134 compared 0 not + pixels differing: median 0.0000 p90 0.0000 p99 0.0033 worst 0.0056 + under 1% 134 under 2% 134 under 5% 134 under 10% 134 + identical byte for byte 99.60% mean |diff| 0.115 levels + colour, levels of 255: worst square mean 4.68 max 79.83 worst pixel 255 + worst 0.56% PDF_A-1b_6.9_Interactive_Forms_veraPDF_test_suite_6-9-t01-fail-b.pdf page 1 + worst 0.52% PDF_A-1b_6.9_Interactive_Forms_veraPDF_test_suite_6-9-t01-fail-a.pdf page 1 + worst 0.33% Isartor_test_files_PDFA-1b_6.9_Interactive_Forms_isartor-6-9-t01-fail-a.pdf page 1 + worst 0.06% Isartor_test_files_PDFA-1b_6.9_Interactive_Forms_isartor-6-9-t02-fail-a.pdf page 1 + worst 0.06% Isartor_test_files_PDFA-1b_6.9_Interactive_Forms_isartor-6-9-t02-fail-b.pdf page 1 + worst 0.02% PDF_A-2b_6.4_Interactive_forms_6.4.2_XFA_forms_veraPDF_test_suite_6-4-2-t01-fail-b.pdf page 1 + worst 0.01% PDF_A-2u_6.2_Graphics_6.2.11_Fonts_6.2.11.7_Unicode_character_maps_6.2.11.7.2_Level_A_and_Level_U_conformance_veraPDF_test_suite_6-2-11-7-2-t01-fail-d.pdf page 1 + worst 0.00% PDF_A-2u_6.2_Graphics_6.2.11_Fonts_6.2.11.7_Unicode_character_maps_6.2.11.7.2_Level_A_and_Level_U_conformance_veraPDF_test_suite_6-2-11-7-2-t01-pass-e.pdf page 1 + worst 0.00% PDF_A-2u_6.2_Graphics_6.2.11_Fonts_6.2.11.7_Unicode_character_maps_6.2.11.7.2_Level_A_and_Level_U_conformance_veraPDF_test_suite_6-2-11-7-2-t01-pass-a.pdf page 1 + worst 0.00% PDF_A-2b_6.4_Interactive_forms_6.4.2_XFA_forms_veraPDF_test_suite_6-4-2-t01-fail-a.pdf page 1 + worst 0.00% PDF_A-4_6.4_Interactive_forms_6.4.2_XFA_forms_veraPDF_test_suite_6-4-2-t01-fail-a.pdf page 1 + worst 0.00% PDF_A-4_6.4_Interactive_forms_6.4.2_XFA_forms_veraPDF_test_suite_6-4-2-t01-fail-b.pdf page 1 + time, 134 pages both were timed on: ours 138ms theirs 4.884s (0.028x) + per page ours/theirs: median 0.022x p90 0.043x faster on 134 of 134 + worst 0.15x faster ours 7ms theirs 48ms PDF_A-4_6.4_Interactive_forms_6.4.2_XFA_forms_veraPDF_test_suite_6-4-2-t01-fail-b.pdf page 1 + worst 0.15x faster ours 8ms theirs 53ms PDF_A-4_6.4_Interactive_forms_6.4.2_XFA_forms_veraPDF_test_suite_6-4-2-t01-fail-a.pdf page 1 + worst 0.14x faster ours 7ms theirs 54ms PDF_A-2b_6.4_Interactive_forms_6.4.2_XFA_forms_veraPDF_test_suite_6-4-2-t01-fail-a.pdf page 1 + worst 0.12x faster ours 3ms theirs 30ms PDF_A-2u_6.2_Graphics_6.2.11_Fonts_6.2.11.7_Unicode_character_maps_6.2.11.7.2_Level_A_and_Level_U_conformance_veraPDF_test_suite_6-2-11-7-2-t01-fail-b.pdf page 1 + worst 0.11x faster ours 3ms theirs 27ms PDF_A-2u_6.2_Graphics_6.2.11_Fonts_6.2.11.7_Unicode_character_maps_6.2.11.7.2_Level_A_and_Level_U_conformance_veraPDF_test_suite_6-2-11-7-2-t01-pass-e.pdf page 1 + slowest page 8ms, 0 over the threshold diff --git a/baseline/pages/pdfforms-int-wipo.txt b/baseline/pages/pdfforms-int-wipo.txt new file mode 100644 index 0000000..7eec5a9 --- /dev/null +++ b/baseline/pages/pdfforms-int-wipo.txt @@ -0,0 +1,25 @@ +int-wipo 116 compared 0 not + pixels differing: median 0.0001 p90 0.0011 p99 0.0022 worst 0.0034 + under 1% 116 under 2% 116 under 5% 116 under 10% 116 + identical byte for byte 89.83% mean |diff| 5.014 levels + colour, levels of 255: worst square mean 11.73 max 21.26 worst pixel 177 + worst 0.34% docs-fr-forms-ro-editable-ed-ro154.pdf page 1 + worst 0.27% docs-fr-forms-ro-editable-ed-ro153.pdf page 1 + worst 0.22% docs-en-forms-ro-editable-ed-ro153.pdf page 1 + worst 0.19% docs-en-forms-ro-editable-ed-ro154.pdf page 1 + worst 0.16% docs-en-forms-ro-editable-ed-ro152.pdf page 1 + worst 0.16% docs-fr-forms-ro-editable-ed-ro140.pdf page 1 + worst 0.15% docs-fr-forms-ro-editable-ed-ro152.pdf page 1 + worst 0.15% docs-fr-forms-ro-editable-ed-ro131.pdf page 1 + worst 0.14% docs-fr-forms-ro-editable-ed-ro124.pdf page 1 + worst 0.14% docs-fr-forms-ro-editable-ed-ro155.pdf page 1 + worst 0.13% docs-fr-forms-ro-editable-ed-ro127.pdf page 1 + worst 0.12% docs-fr-forms-ro-editable-ed-ro148.pdf page 1 + time, 116 pages both were timed on: ours 1.589s theirs 13.394s (0.119x) + per page ours/theirs: median 0.104x p90 0.169x faster on 116 of 116 + worst 0.62x faster ours 64ms theirs 103ms docs-en-forms-ro-editable-ed-ro142.pdf page 1 + worst 0.30x faster ours 55ms theirs 184ms docs-fr-forms-ro-editable-ed-ro136.pdf page 1 + worst 0.26x faster ours 24ms theirs 92ms docs-en-forms-ro-editable-ed-ro133.pdf page 1 + worst 0.23x faster ours 27ms theirs 116ms docs-en-forms-ro-editable-ed-ro148.pdf page 1 + worst 0.21x faster ours 30ms theirs 141ms docs-fr-forms-ro-editable-ed-ro126.pdf page 1 + slowest page 64ms, 0 over the threshold diff --git a/baseline/pages/pdfforms-uk-govuk.txt b/baseline/pages/pdfforms-uk-govuk.txt new file mode 100644 index 0000000..4cc4000 --- /dev/null +++ b/baseline/pages/pdfforms-uk-govuk.txt @@ -0,0 +1,25 @@ +uk-govuk 302 compared 0 not + pixels differing: median 0.0011 p90 0.0049 p99 0.0089 worst 0.0103 + under 1% 301 under 2% 302 under 5% 302 under 10% 302 + identical byte for byte 84.39% mean |diff| 5.969 levels + colour, levels of 255: worst square mean 17.33 max 40.89 worst pixel 254 + worst 1.03% lasting-power-of-attorney-forms__20250821-LP12-Make-and-register-your-lasting-power-of-attorney-a-guide.pdf page 1 + worst 0.95% challenge-a-decision-made-by-the-department-for-__crmr1a.pdf page 1 + worst 0.92% lasting-power-of-attorney-forms__20250821-LP12-Gwneud-a-chofrestru-eich-atwrneiaeth-arhosol-canllawiau.pdf page 1 + worst 0.90% form-n245-application-for-suspension-of-a-warran__n245-largeprint-eng.pdf page 1 + worst 0.89% challenge-a-decision-made-by-the-department-for-__crmr1a_w.pdf page 1 + worst 0.86% general-register-office-gro-certificate-applicat__5__-_Overseas_Civil_Partnership_1_25.pdf page 1 + worst 0.77% health-adjustment-passport__health-adjustment-passport-easy-read-form.pdf page 1 + worst 0.75% applying-for-a-passport-from-outside-the-uk-appl__CCS0226120416-001_PN10604791_Online_Overseas_Passport_Ap.pdf page 1 + worst 0.73% search-public-guardian-registers__20250901-print-bras-OPG100-canfod-a-oes-gan-rywun.pdf page 1 + worst 0.72% keep-a-record-of-your-pupils-driving-lessons__learning-to-drive-record-overall-progress.pdf page 1 + worst 0.72% apply-for-a-non-molestation-or-occupation-order-__FL401_template_statement_-_22pt.pdf page 1 + worst 0.70% apply-for-a-non-molestation-or-occupation-order-__FL401_ER.pdf page 1 + time, 302 pages both were timed on: ours 11.917s theirs 44.143s (0.270x) + per page ours/theirs: median 0.163x p90 0.520x faster on 302 of 302 + worst 0.94x faster ours 127ms theirs 136ms complete-your-self-assessment-tax-return-for-5-a__SA100_2024_Cymraeg.pdf page 1 + worst 0.94x faster ours 128ms theirs 137ms complete-your-self-assessment-tax-return-for-5-a__SA100-Cymraeg-2025.pdf page 1 + worst 0.92x faster ours 126ms theirs 136ms complete-your-self-assessment-tax-return-for-5-a__SA100_2023_Cymraeg.pdf page 1 + worst 0.92x faster ours 100ms theirs 110ms search-public-guardian-registers__20250901-OPG100-canfod-a-oes-gan-rywun.pdf page 1 + worst 0.88x faster ours 103ms theirs 117ms lasting-power-of-attorney-forms__20251117_lpc-dalenni-parhad.pdf page 1 + slowest page 448ms, 0 over the threshold diff --git a/baseline/pages/pdfforms-us-dol.txt b/baseline/pages/pdfforms-us-dol.txt new file mode 100644 index 0000000..f1b8699 --- /dev/null +++ b/baseline/pages/pdfforms-us-dol.txt @@ -0,0 +1,25 @@ +us-dol 140 compared 0 not + pixels differing: median 0.0035 p90 0.0122 p99 0.0194 worst 0.0195 + under 1% 118 under 2% 140 under 5% 140 under 10% 140 + identical byte for byte 84.03% mean |diff| 10.447 levels + colour, levels of 255: worst square mean 21.33 max 40.03 worst pixel 255 + worst 1.95% 2025-form-5500-ez.pdf page 1 + worst 1.95% 2024-form-5500-ez.pdf page 1 + worst 1.94% 2023-form-5500-ez.pdf page 1 + worst 1.54% 1996-schedule-e.pdf page 1 + worst 1.53% 1998-schedule-e.pdf page 1 + worst 1.53% 1997-schedule-e.pdf page 1 + worst 1.37% 1999-schedule-d.pdf page 1 + worst 1.32% 1997-schedule-f.pdf page 1 + worst 1.31% 2023-schedule-a.pdf page 1 + worst 1.30% 1996-schedule-f.pdf page 1 + worst 1.29% 2009-schedule-g.pdf page 1 + worst 1.27% 2011-schedule-d.pdf page 1 + time, 140 pages both were timed on: ours 4.33s theirs 19.072s (0.227x) + per page ours/theirs: median 0.214x p90 0.291x faster on 139 of 140 + worst 1.62x SLOWER ours 351ms theirs 217ms 2024-schedule-dcg.pdf page 1 + worst 0.52x faster ours 66ms theirs 127ms 2024-schedule-d.pdf page 1 + worst 0.44x faster ours 53ms theirs 121ms 2024-schedule-sb.pdf page 1 + worst 0.43x faster ours 53ms theirs 124ms 2024-schedule-mb.pdf page 1 + worst 0.41x faster ours 64ms theirs 155ms 2024-schedule-h.pdf page 1 + slowest page 351ms, 0 over the threshold diff --git a/baseline/pages/pdfforms-us-irs.txt b/baseline/pages/pdfforms-us-irs.txt new file mode 100644 index 0000000..161d9ce --- /dev/null +++ b/baseline/pages/pdfforms-us-irs.txt @@ -0,0 +1,25 @@ +us-irs 69 compared 0 not + pixels differing: median 0.0008 p90 0.0024 p99 0.0034 worst 0.0043 + under 1% 69 under 2% 69 under 5% 69 under 10% 69 + identical byte for byte 80.63% mean |diff| 13.039 levels + colour, levels of 255: worst square mean 17.76 max 21.98 worst pixel 128 + worst 0.43% f720.pdf page 1 + worst 0.34% f8962.pdf page 1 + worst 0.31% f1040.pdf page 1 + worst 0.28% f1040x.pdf page 1 + worst 0.28% f1065.pdf page 1 + worst 0.26% f990pf.pdf page 1 + worst 0.25% f940.pdf page 1 + worst 0.24% f4868.pdf page 1 + worst 0.23% f990ez.pdf page 1 + worst 0.21% f7004.pdf page 1 + worst 0.21% fw2.pdf page 1 + worst 0.19% f1120.pdf page 1 + time, 69 pages both were timed on: ours 2.227s theirs 10.736s (0.207x) + per page ours/theirs: median 0.197x p90 0.232x faster on 69 of 69 + worst 0.58x faster ours 200ms theirs 346ms f4868.pdf page 1 + worst 0.29x faster ours 59ms theirs 205ms f1040.pdf page 1 + worst 0.26x faster ours 33ms theirs 124ms f2848.pdf page 1 + worst 0.25x faster ours 30ms theirs 121ms fw8ben.pdf page 1 + worst 0.23x faster ours 31ms theirs 134ms fw9.pdf page 1 + slowest page 200ms, 0 over the threshold diff --git a/baseline/pages/pdfforms-us-opm.txt b/baseline/pages/pdfforms-us-opm.txt new file mode 100644 index 0000000..6c618ac --- /dev/null +++ b/baseline/pages/pdfforms-us-opm.txt @@ -0,0 +1,25 @@ +us-opm 66 compared 0 not + pixels differing: median 0.0012 p90 0.0068 p99 0.0261 worst 0.0564 + under 1% 64 under 2% 64 under 5% 65 under 10% 66 + identical byte for byte 83.97% mean |diff| 8.729 levels + colour, levels of 255: worst square mean 18.53 max 41.24 worst pixel 221 + worst 5.64% sf2820.pdf page 1 + worst 2.61% sf3106.pdf page 1 + worst 0.85% sf3100.pdf page 1 + worst 0.79% sf3108.pdf page 1 + worst 0.78% sf2810.pdf page 1 + worst 0.70% sf2812.pdf page 1 + worst 0.70% sf2817.pdf page 1 + worst 0.68% sf144.pdf page 1 + worst 0.67% sf256.pdf page 1 + worst 0.55% sf61.pdf page 1 + worst 0.53% sf3110.pdf page 1 + worst 0.45% sf62.pdf page 1 + time, 66 pages both were timed on: ours 2.038s theirs 12.986s (0.157x) + per page ours/theirs: median 0.141x p90 0.342x faster on 66 of 66 + worst 0.67x faster ours 159ms theirs 237ms sf144.pdf page 1 + worst 0.43x faster ours 36ms theirs 85ms of5057.pdf page 1 + worst 0.41x faster ours 114ms theirs 280ms sf1187.pdf page 1 + worst 0.40x faster ours 51ms theirs 126ms sf1152.pdf page 1 + worst 0.39x faster ours 89ms theirs 230ms sf113a.pdf page 1 + slowest page 159ms, 0 over the threshold diff --git a/baseline/pages/pdfforms-us-ssa.txt b/baseline/pages/pdfforms-us-ssa.txt new file mode 100644 index 0000000..f78e5bd --- /dev/null +++ b/baseline/pages/pdfforms-us-ssa.txt @@ -0,0 +1,25 @@ +us-ssa 199 compared 0 not + pixels differing: median 0.0013 p90 0.0034 p99 0.0057 worst 0.0070 + under 1% 199 under 2% 199 under 5% 199 under 10% 199 + identical byte for byte 83.98% mean |diff| 9.666 levels + colour, levels of 255: worst square mean 17.08 max 25.83 worst pixel 234 + worst 0.70% ssa-454-bk.pdf page 1 + worst 0.66% ssa-1372.pdf page 1 + worst 0.57% ssa-3820.pdf page 1 + worst 0.51% ssa-4111.pdf page 1 + worst 0.46% ssa-1383-fc.pdf page 1 + worst 0.45% ssa-1372-bk-fc-sp.pdf page 1 + worst 0.45% ssa-21.pdf page 1 + worst 0.44% ssa-4814.pdf page 1 + worst 0.44% ssa-792.pdf page 1 + worst 0.43% ssa-8011-f3.pdf page 1 + worst 0.42% ssa-7008.pdf page 1 + worst 0.41% ssa-1724.pdf page 1 + time, 199 pages both were timed on: ours 2.665s theirs 24.997s (0.107x) + per page ours/theirs: median 0.093x p90 0.139x faster on 199 of 199 + worst 0.35x faster ours 71ms theirs 204ms ss-5-ch-inst.pdf page 1 + worst 0.33x faster ours 51ms theirs 154ms ss-5-kor-inst.pdf page 1 + worst 0.27x faster ours 44ms theirs 162ms fss4sp.pdf page 1 + worst 0.23x faster ours 35ms theirs 151ms ss-5-ru-inst.pdf page 1 + worst 0.23x faster ours 34ms theirs 149ms fss4.pdf page 1 + slowest page 71ms, 0 over the threshold diff --git a/baseline/pages/pdfforms-us-uscis.txt b/baseline/pages/pdfforms-us-uscis.txt new file mode 100644 index 0000000..148b3c4 --- /dev/null +++ b/baseline/pages/pdfforms-us-uscis.txt @@ -0,0 +1,25 @@ +us-uscis 88 compared 0 not + pixels differing: median 0.0015 p90 0.0022 p99 0.0034 worst 0.0044 + under 1% 88 under 2% 88 under 5% 88 under 10% 88 + identical byte for byte 85.29% mean |diff| 11.456 levels + colour, levels of 255: worst square mean 19.73 max 24.72 worst pixel 147 + worst 0.44% i-914.pdf page 1 + worst 0.34% i-129f.pdf page 1 + worst 0.32% i-687.pdf page 1 + worst 0.27% i-590.pdf page 1 + worst 0.25% g-1145.pdf page 1 + worst 0.24% i-589.pdf page 1 + worst 0.23% i-918.pdf page 1 + worst 0.23% i-821d.pdf page 1 + worst 0.22% i-800a.pdf page 1 + worst 0.22% i-130.pdf page 1 + worst 0.21% i-526.pdf page 1 + worst 0.20% i-956h.pdf page 1 + time, 88 pages both were timed on: ours 2.13s theirs 15.249s (0.140x) + per page ours/theirs: median 0.141x p90 0.175x faster on 88 of 88 + worst 0.28x faster ours 49ms theirs 171ms i-956.pdf page 1 + worst 0.22x faster ours 34ms theirs 154ms i-9.pdf page 1 + worst 0.20x faster ours 36ms theirs 175ms g-1450.pdf page 1 + worst 0.19x faster ours 31ms theirs 159ms i-929.pdf page 1 + worst 0.18x faster ours 21ms theirs 113ms i-694.pdf page 1 + slowest page 49ms, 0 over the threshold diff --git a/baseline/pages/pdfforms-us-uscourts.txt b/baseline/pages/pdfforms-us-uscourts.txt new file mode 100644 index 0000000..13a9d66 --- /dev/null +++ b/baseline/pages/pdfforms-us-uscourts.txt @@ -0,0 +1,25 @@ +us-uscourts 69 compared 0 not + pixels differing: median 0.0034 p90 0.0117 p99 0.0517 worst 0.1049 + under 1% 55 under 2% 67 under 5% 67 under 10% 68 + identical byte for byte 92.12% mean |diff| 4.686 levels + colour, levels of 255: worst square mean 13.24 max 31.79 worst pixel 151 + worst 10.49% ao038.pdf page 1 + worst 5.17% ao436.pdf page 1 + worst 1.58% ao133.pdf page 1 + worst 1.34% ao088.pdf page 1 + worst 1.34% ao083.pdf page 1 + worst 1.32% ao458.pdf page 1 + worst 1.23% ao450.pdf page 1 + worst 1.17% ao442.pdf page 1 + worst 1.13% ao078.pdf page 1 + worst 1.12% ao085.pdf page 1 + worst 1.10% ao089.pdf page 1 + worst 1.09% ao246.pdf page 1 + time, 69 pages both were timed on: ours 1.344s theirs 10.874s (0.124x) + per page ours/theirs: median 0.106x p90 0.164x faster on 69 of 69 + worst 0.94x faster ours 124ms theirs 131ms ao390.pdf page 1 + worst 0.48x faster ours 170ms theirs 356ms ao213.pdf page 1 + worst 0.21x faster ours 20ms theirs 99ms ao078.pdf page 1 + worst 0.20x faster ours 19ms theirs 96ms ao030.pdf page 1 + worst 0.20x faster ours 33ms theirs 169ms ao240.pdf page 1 + slowest page 170ms, 0 over the threshold diff --git a/baseline/pages/pdfscans-ia-americana.txt b/baseline/pages/pdfscans-ia-americana.txt new file mode 100644 index 0000000..81e86b5 --- /dev/null +++ b/baseline/pages/pdfscans-ia-americana.txt @@ -0,0 +1,26 @@ +ia-americana 222 compared 28 not + 28 refused + pixels differing: median 0.0023 p90 0.0165 p99 0.0394 worst 0.1923 + under 1% 179 under 2% 206 under 5% 219 under 10% 220 + identical byte for byte 23.52% mean |diff| 11.743 levels + colour, levels of 255: worst square mean 26.99 max 149.65 worst pixel 212 + worst 19.23% sim_unitarian-register-and-the-universalist-leader_christian-register_1825-06-25_4_25.pdf page 1 + worst 14.70% sim_unitarian-register-and-the-universalist-leader_christian-register_1847-04-17_26_16.pdf page 1 + worst 6.00% 19511011-Hancock_County_Eagle.pdf page 1 + worst 3.94% 19530326-Hancock_County_Eagle.pdf page 1 + worst 3.83% 19530213-Sea_Coast_Echo.pdf page 1 + worst 3.71% 19951019-Sea_Coast_Echo.pdf page 1 + worst 3.45% 19610928-Sea_Coast_Echo.pdf page 1 + worst 3.28% 19900719-Sea_Coast_Echo.pdf page 1 + worst 3.05% 19511012-Sea_Coast_Echo.pdf page 1 + worst 2.69% 19890223-Sea_Coast_Echo.pdf page 1 + worst 2.66% 19981105-Sea_Coast_Echo.pdf page 1 + worst 2.49% 19920112-Sea_Coast_Echo.pdf page 1 + time, 222 pages both were timed on: ours 1m43.11s theirs 5m8.89s (0.334x) + per page ours/theirs: median 0.322x p90 0.718x faster on 214 of 222 + worst 1.50x SLOWER ours 307ms theirs 204ms 20000809onehundredandfivedesignssubmittedforuscapitolvisitorcentercommemorativecoins.pdf page 1 + worst 1.38x SLOWER ours 370ms theirs 268ms 20070425bronzemedalshonoringgolferbyronnelsonavailableapril26fromtheunitedstatesmint.pdf page 1 + worst 1.33x SLOWER ours 297ms theirs 222ms 20030814unitedstatesminttakesfastactiontocorrectmisprintedcertificatesofauthenticity.pdf page 1 + worst 1.30x SLOWER ours 303ms theirs 232ms 200406082004unitedstatesmintuncirculatedcoinsetavailablejune22.pdf page 1 + worst 1.09x SLOWER ours 257ms theirs 237ms calgrhs_000907.pdf page 1 + slowest page 3.869s, 0 over the threshold diff --git a/baseline/pages/pdfscans-ia-biodiversity.txt b/baseline/pages/pdfscans-ia-biodiversity.txt new file mode 100644 index 0000000..18e26bf --- /dev/null +++ b/baseline/pages/pdfscans-ia-biodiversity.txt @@ -0,0 +1,25 @@ +ia-biodiversity 250 compared 0 not + pixels differing: median 0.0020 p90 0.0110 p99 0.0586 worst 0.0990 + under 1% 220 under 2% 237 under 5% 245 under 10% 250 + identical byte for byte 32.18% mean |diff| 11.227 levels + colour, levels of 255: worst square mean 21.09 max 95.69 worst pixel 254 + worst 9.90% bidragtillknne00falc.pdf page 1 + worst 7.30% bulletinno38tasm.pdf page 1 + worst 5.89% checklistoffishi88alma.pdf page 1 + worst 5.86% bulletinsocit08pyre.pdf page 1 + worst 5.19% charlespickerin00pick.pdf page 1 + worst 3.36% injuryrecoveryde00oste.pdf page 1 + worst 3.31% bulletinofgeolo30geol.pdf page 1 + worst 3.19% bulletinofdepart03jama.pdf page 1 + worst 2.88% invertebrata00queeo.pdf page 1 + worst 2.76% infield883chic.pdf page 1 + worst 2.70% iterhelveticuman1740hall.pdf page 1 + worst 2.18% bidragtillkn35suom.pdf page 1 + time, 250 pages both were timed on: ours 1m20.014s theirs 2m43.041s (0.491x) + per page ours/theirs: median 0.528x p90 0.764x faster on 248 of 250 + worst 1.17x SLOWER ours 853ms theirs 730ms bulletin1901964smit.pdf page 1 + worst 1.10x SLOWER ours 714ms theirs 646ms interactiontrac510hilk.pdf page 1 + worst 0.92x faster ours 577ms theirs 626ms indigoferavalli199liuj.pdf page 1 + worst 0.92x faster ours 638ms theirs 693ms bulletin1021932smit.pdf page 1 + worst 0.91x faster ours 255ms theirs 281ms bulletindelasocif06soci.pdf page 1 + slowest page 1.07s, 0 over the threshold diff --git a/baseline/pages/pdfscans-ia-medical.txt b/baseline/pages/pdfscans-ia-medical.txt new file mode 100644 index 0000000..addcc33 --- /dev/null +++ b/baseline/pages/pdfscans-ia-medical.txt @@ -0,0 +1,25 @@ +ia-medical 250 compared 0 not + pixels differing: median 0.0000 p90 0.0031 p99 0.0108 worst 0.0330 + under 1% 246 under 2% 248 under 5% 250 under 10% 250 + identical byte for byte 15.69% mean |diff| 6.480 levels + colour, levels of 255: worst square mean 14.82 max 97.67 worst pixel 186 + worst 3.30% childrenwithorth00shar.pdf page 1 + worst 3.03% childrenwithhear00rita.pdf page 1 + worst 1.75% insaneinforeignc00letcuoft.pdf page 1 + worst 1.08% insanityitsdepen00gray.pdf page 1 + worst 0.95% buildinglibraryc00nati.pdf page 1 + worst 0.93% inflammationintr00adamuoft.pdf page 1 + worst 0.87% b22324392.pdf page 1 + worst 0.73% b2232916x.pdf page 1 + worst 0.71% b22349431.pdf page 1 + worst 0.66% b21963927.pdf page 1 + worst 0.64% introductiontoco1816lawr.pdf page 1 + worst 0.58% b22006370_0001.pdf page 1 + time, 250 pages both were timed on: ours 1m2.145s theirs 3m22.035s (0.308x) + per page ours/theirs: median 0.291x p90 0.671x faster on 249 of 250 + worst 1.47x SLOWER ours 970ms theirs 661ms b2232916x.pdf page 1 + worst 0.86x faster ours 296ms theirs 345ms inquiryintoprinc00fluh.pdf page 1 + worst 0.81x faster ours 236ms theirs 291ms b28132233.pdf page 1 + worst 0.79x faster ours 438ms theirs 557ms industrialcoloni00wooduoft.pdf page 1 + worst 0.78x faster ours 206ms theirs 266ms insanityitsdepen00gray.pdf page 1 + slowest page 970ms, 0 over the threshold diff --git a/baseline/pages/pdfscans-ia-texts.txt b/baseline/pages/pdfscans-ia-texts.txt new file mode 100644 index 0000000..b4ab768 --- /dev/null +++ b/baseline/pages/pdfscans-ia-texts.txt @@ -0,0 +1,19 @@ +ia-texts 5 compared 7 not + 7 refused + pixels differing: median 0.0001 p90 0.0026 p99 0.0026 worst 0.0202 + under 1% 4 under 2% 4 under 5% 5 under 10% 5 + identical byte for byte 22.82% mean |diff| 10.402 levels + colour, levels of 255: worst square mean 16.10 max 36.67 worst pixel 159 + worst 2.02% bdrc-W1KG4230.pdf page 1 + worst 0.26% kpbc.umk.pl.KM_01052_1924_12_266929.pdf page 1 + worst 0.01% cricmt_001073.pdf page 1 + worst 0.01% jbc.bj.uj.edu.pl.NDIGCZAS049827_1938_026.pdf page 1 + worst 0.00% jbc.bj.uj.edu.pl.NDIGCZAS066260_1884_021.pdf page 1 + time, 5 pages both were timed on: ours 1.122s theirs 3.795s (0.296x) + per page ours/theirs: median 0.424x p90 0.491x faster on 5 of 5 + worst 0.53x faster ours 500ms theirs 945ms cricmt_001073.pdf page 1 + worst 0.49x faster ours 36ms theirs 74ms bdrc-W1KG4230.pdf page 1 + worst 0.42x faster ours 200ms theirs 472ms jbc.bj.uj.edu.pl.NDIGCZAS049827_1938_026.pdf page 1 + worst 0.18x faster ours 234ms theirs 1.304s jbc.bj.uj.edu.pl.NDIGCZAS066260_1884_021.pdf page 1 + worst 0.15x faster ours 151ms theirs 1.001s kpbc.umk.pl.KM_01052_1924_12_266929.pdf page 1 + slowest page 500ms, 0 over the threshold diff --git a/baseline/pages/pdfscans-ia-uscourts.txt b/baseline/pages/pdfscans-ia-uscourts.txt new file mode 100644 index 0000000..a01d378 --- /dev/null +++ b/baseline/pages/pdfscans-ia-uscourts.txt @@ -0,0 +1,25 @@ +ia-uscourts 250 compared 0 not + pixels differing: median 0.0005 p90 0.0060 p99 0.0179 worst 0.0221 + under 1% 237 under 2% 249 under 5% 250 under 10% 250 + identical byte for byte 85.36% mean |diff| 5.211 levels + colour, levels of 255: worst square mean 15.91 max 96.19 worst pixel 255 + worst 2.21% gov.uscourts.casd.850253.pdf page 1 + worst 1.86% gov.uscourts.cacd.1009689.pdf page 1 + worst 1.85% gov.uscourts.mnd.138153.pdf page 1 + worst 1.79% gov.uscourts.txnd.218305.pdf page 1 + worst 1.78% gov.uscourts.wvsd.62483.pdf page 1 + worst 1.68% gov.uscourts.nynd.21695.pdf page 1 + worst 1.62% gov.uscourts.txsd.1677780.pdf page 1 + worst 1.54% gov.uscourts.almd.64443.pdf page 1 + worst 1.28% gov.uscourts.vaeb.554034.pdf page 1 + worst 1.25% gov.uscourts.pamd.98522.pdf page 1 + worst 1.19% gov.uscourts.almd.58141.pdf page 1 + worst 1.02% gov.uscourts.dcd.262861.pdf page 1 + time, 250 pages both were timed on: ours 6.555s theirs 28.743s (0.228x) + per page ours/theirs: median 0.132x p90 0.487x faster on 249 of 250 + worst 1.02x SLOWER ours 229ms theirs 225ms gov.uscourts.msnb.180080.pdf page 1 + worst 0.79x faster ours 100ms theirs 126ms gov.uscourts.txsd.2092352.pdf page 1 + worst 0.78x faster ours 85ms theirs 109ms gov.uscourts.nced.112866.pdf page 1 + worst 0.69x faster ours 57ms theirs 82ms gov.uscourts.txwd.1172889819.pdf page 1 + worst 0.69x faster ours 72ms theirs 105ms gov.uscourts.paed.653779.pdf page 1 + slowest page 229ms, 0 over the threshold From 7900049db837ff812e2d690ae6257da59ffc644d Mon Sep 17 00:00:00 2001 From: tannevaled Date: Sun, 4 Oct 2026 16:47:17 +0200 Subject: [PATCH 23/28] corpus: check that a corpus is what its manifest says, and say so MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The package comment of corpus has said since it was written that the manifest "is what makes the corpus a measurement rather than a pile: it says where each file came from, when, how big it is and what its bytes hash to, so a figure quoted from it can be reproduced and a file that changed underneath can be noticed". Nothing noticed. The hash was written and never read back, and a file in no row was invisible to every tool here. A documented capability with no implementation. corpus.Check reports a row with no file, a file that will not open, a size or digest that moved, and a file on disk in no row. harvest -check prints them and exits NON-ZERO, so it can be the first line of a measuring script rather than something to remember to run; harvest because harvest is what writes the manifest. Run over the two corpora: pdfforms 2268 rows, clean pdfscans 1012 rows, clean, and FOUR files on disk in no row ia-americana/calcflh_000254.pdf ia-americana/epn11-1968countclip2restricted.pdf ia-biodiversity/bidragtillknne38suom.pdf ia-medical/b22346703.pdf Three of those four are documents our reader refuses, which is the whole of why images and compare count 63 refusals and a directory-walking sweep counts 66 (§33). And every digest that IS recorded verifies, so no document has changed underneath -- 1 012 full hashes and 2 268 prefixes. The digest is compared over AS MANY CHARACTERS AS THE MANIFEST RECORDED. The forms corpus keeps sixteen under the header "sha256-8"; the scans corpus keeps sixty-four. Sixteen hex characters is sixty-four bits, which answers "did this file change" perfectly well, and a check demanding all sixty-four would call every one of 2 268 rows changed and be deleted by the first person to run it. That is the mutation this is proved against. An unreadable file is reported as unreadable and NOT as missing: missing would send a reader to re-fetch a document that is already on the disk. One branch was removed rather than covered: filepath.Rel cannot fail on a path WalkDir has just produced under its own root, and an error branch nothing can reach is a branch nobody can test. The relative path is trimmed instead. Three mutations, each caught: comparing the whole digest instead of the recorded prefix, not looking for files in no row, and calling an unreadable file missing. 100.0% of statements. --- README.md | 34 ++++++ cmd/harvest/run.go | 38 +++++++ cmd/harvest/run_test.go | 58 ++++++++++ corpus/check.go | 135 ++++++++++++++++++++++ corpus/check_test.go | 245 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 510 insertions(+) create mode 100644 corpus/check.go create mode 100644 corpus/check_test.go diff --git a/README.md b/README.md index 68a2453..f81738c 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,40 @@ harvest -dir /Users/Shared/pdfscans -origin ia-americana \ skipped, so an interrupted run is continued by running it again and a corpus is extended by asking for a larger `-want`. +**And it checks, because the sentence above promised something nothing read +back.** "What makes a number reproducible and lets a document that changed +underneath be noticed" was true of what the manifest RECORDS and false of what +anything here did with it: the hash was written and never compared, and a file +in no row was invisible to every tool in this repository. + +``` +$ harvest -check -dir /Users/Shared/pdfscans +ia-americana/calcflh_000254.pdf: on disk and in no manifest row +ia-americana/epn11-1968countclip2restricted.pdf: on disk and in no manifest row +ia-biodiversity/bidragtillknne38suom.pdf: on disk and in no manifest row +ia-medical/b22346703.pdf: on disk and in no manifest row +4 disagreement(s) +``` + +Four files, and **three of them are documents our reader refuses** — which is +the whole of why `images` and `compare`, which walk the manifest, count 63 +refusals where a sweep that walks the directory counts 66. Two true numbers for +one corpus and no way to tell which a figure came from. See +[`baseline/README.md`](baseline/README.md) §33. + +It reports a row with no file, a file that will not open, a size or a digest +that moved, and a file in no row; it exits non-zero when a corpus disagrees, so +it can be the first line of a measuring script rather than something to +remember. `/Users/Shared/pdfforms` comes back clean — all 2 268 rows — and so +does every one of `pdfscans`'s 1 012, **so no document has changed underneath**. + +The digest is compared over **as many characters as the manifest recorded**. The +forms corpus, gathered before this repository existed, keeps sixteen of them +under the header `sha256-8`; the scans corpus keeps all sixty-four. Sixteen hex +characters is sixty-four bits, which answers *did this file change* perfectly +well — and a check that demanded all sixty-four would report every one of those +2 268 rows as changed and be deleted by the first person who ran it. + It records the population per document because **a prevalence is per population or it is not a prevalence**. diff --git a/cmd/harvest/run.go b/cmd/harvest/run.go index 42cfe9e..dbce89f 100644 --- a/cmd/harvest/run.go +++ b/cmd/harvest/run.go @@ -27,9 +27,14 @@ func run(args []string, out, errOut io.Writer) int { want := fs.Int("want", 100, "how many documents this origin should end up with") maxBytes := fs.Int64("max-bytes", 40<<20, "refuse a document larger than this") workers := fs.Int("workers", 4, "how many to fetch at once") + check := fs.Bool("check", false, + "fetch nothing: report every way the corpus and its manifest disagree") if err := fs.Parse(args); err != nil { return 2 } + if *check { + return checkCorpus(*dir, out, errOut) + } if *dir == "" || *origin == "" || *query == "" { fmt.Fprintln(errOut, "harvest: -dir, -origin and -query are all needed") return 2 @@ -58,3 +63,36 @@ func run(args []string, out, errOut io.Writer) int { } return 0 } + +// checkCorpus reports where the corpus and the manifest disagree, and is in +// harvest because harvest is what writes the manifest. +// +// The package comment of corpus says the manifest is what makes a corpus a +// measurement rather than a pile, "so a figure quoted from it can be reproduced +// and a file that changed underneath can be noticed". Nothing noticed: the hash +// was written and never read back. Four files of the scans corpus are on disk +// and in no row, three of them documents our reader refuses, which is why one +// instrument counts 63 refusals and another 66. +// +// It exits NON-ZERO when a corpus disagrees with its manifest, so it can be the +// first line of a measuring script rather than something to remember to run. +func checkCorpus(dir string, out, errOut io.Writer) int { + if dir == "" { + fmt.Fprintln(errOut, "harvest: -check needs -dir") + return 2 + } + problems, err := corpus.Check(dir) + if err != nil { + fmt.Fprintf(errOut, "harvest: %v\n", err) + return 1 + } + for _, p := range problems { + fmt.Fprintf(out, "%s\n", p) + } + if len(problems) == 0 { + fmt.Fprintf(out, "the corpus and its manifest agree\n") + return 0 + } + fmt.Fprintf(out, "%d disagreement(s)\n", len(problems)) + return 1 +} diff --git a/cmd/harvest/run_test.go b/cmd/harvest/run_test.go index c7a93e4..1a6bb04 100644 --- a/cmd/harvest/run_test.go +++ b/cmd/harvest/run_test.go @@ -5,6 +5,7 @@ import ( "context" "errors" "os" + "path/filepath" "strings" "testing" @@ -91,3 +92,60 @@ func TestMainCallsRun(t *testing.T) { t.Errorf("main exited %d, want 2", got) } } + +func TestCheckReportsACorpusThatDisagreesAndExitsNonZero(t *testing.T) { + // -check is meant to be the first line of a measuring script, so the exit + // status carries the answer and not only the text. + dir := t.TempDir() + if err := os.MkdirAll(filepath.Join(dir, "a"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "a/stranger.pdf"), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + var out, errOut bytes.Buffer + if code := run([]string{"-check", "-dir", dir}, &out, &errOut); code != 1 { + t.Fatalf("exit %d: %s%s", code, out.String(), errOut.String()) + } + if !strings.Contains(out.String(), "in no manifest row") || + !strings.Contains(out.String(), "1 disagreement") { + t.Errorf("it said %q", out.String()) + } +} + +func TestCheckSaysSoWhenTheCorpusAgrees(t *testing.T) { + // A silent success and a success that says what it checked are not the + // same thing to anyone reading a log. + dir := t.TempDir() + var out, errOut bytes.Buffer + if code := run([]string{"-check", "-dir", dir}, &out, &errOut); code != 0 { + t.Fatalf("exit %d: %s%s", code, out.String(), errOut.String()) + } + if !strings.Contains(out.String(), "agree") { + t.Errorf("it said %q", out.String()) + } +} + +func TestCheckWithoutADirectory(t *testing.T) { + var out, errOut bytes.Buffer + if code := run([]string{"-check"}, &out, &errOut); code != 2 { + t.Errorf("exit %d", code) + } + if !strings.Contains(errOut.String(), "needs -dir") { + t.Errorf("it said %q", errOut.String()) + } +} + +func TestCheckOfAManifestThatCannotBeRead(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, corpus.ManifestName), []byte("nonsense\n"), 0o644); err != nil { + t.Fatal(err) + } + var out, errOut bytes.Buffer + if code := run([]string{"-check", "-dir", dir}, &out, &errOut); code != 1 { + t.Errorf("exit %d: %s", code, errOut.String()) + } + if !strings.Contains(errOut.String(), "harvest:") { + t.Errorf("it said %q", errOut.String()) + } +} diff --git a/corpus/check.go b/corpus/check.go new file mode 100644 index 0000000..d6993c0 --- /dev/null +++ b/corpus/check.go @@ -0,0 +1,135 @@ +package corpus + +import ( + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "sort" + "strings" +) + +// A Problem is one way a corpus and the manifest beside it disagree. +type Problem struct { + // Path is relative to the corpus directory, as the manifest writes it. + Path string + // Kind is why: "missing", "unreadable", "size", "digest" or + // "unrecorded". + Kind string + // Want and Got say what the manifest records and what is on disk. Both + // are empty for a kind where there is nothing to compare. + Want, Got string +} + +func (p Problem) String() string { + switch p.Kind { + case "missing": + return fmt.Sprintf("%s: in the manifest and not on disk", p.Path) + case "unrecorded": + return fmt.Sprintf("%s: on disk and in no manifest row", p.Path) + case "unreadable": + return fmt.Sprintf("%s: on disk and will not open: %s", p.Path, p.Got) + default: + return fmt.Sprintf("%s: the manifest says %s %s, the file is %s", + p.Path, p.Kind, p.Want, p.Got) + } +} + +// Check reports every way a corpus directory and its manifest disagree. +// +// The package comment says the manifest is what makes a corpus a measurement +// rather than a pile, "so a figure quoted from it can be reproduced and a file +// that changed underneath can be noticed". Nothing noticed: the hash was +// recorded and never read back, and a file in no row was invisible to every +// tool here. +// +// It is not a hypothetical. Measured 2026-10-04 over the two corpora this +// repository is run against, FOUR files of `pdfscans` were on disk and in no +// manifest row -- and three of those four are documents our reader refuses, so +// `images` and `compare`, which walk the manifest, counted 63 refusals where a +// sweep that walked the directory counted 66. Two true numbers for one corpus, +// and no way to tell which a figure came from. See baseline/README.md §33. +// +// The digest is compared over AS MANY CHARACTERS AS THE MANIFEST RECORDS. The +// forms corpus, gathered before this repository existed, keeps sixteen of them +// under the header "sha256-8"; the scans corpus keeps all sixty-four. Sixteen +// hex characters is sixty-four bits, which answers "did this file change" +// perfectly well, and a check that demanded all sixty-four would call every one +// of 2 268 rows changed and be deleted by the first person to run it. +func Check(dir string) ([]Problem, error) { + entries, err := Read(dir) + if err != nil { + return nil, err + } + var out []Problem + recorded := make(map[string]bool, len(entries)) + for _, e := range entries { + recorded[e.Path] = true + full := filepath.Join(dir, e.Path) + info, err := os.Stat(full) + if err != nil { + out = append(out, Problem{Path: e.Path, Kind: "missing"}) + continue + } + if e.Bytes > 0 && info.Size() != e.Bytes { + out = append(out, Problem{Path: e.Path, Kind: "size", + Want: fmt.Sprint(e.Bytes), Got: fmt.Sprint(info.Size())}) + // The digest is not read as well: a file of another size has + // another digest by construction, and saying so twice about one + // file makes a count of problems a count of nothing. + continue + } + if e.SHA256 == "" { + // Some documents predate the record that describes them, which + // Entry.SHA256 already says. There is nothing to check. + continue + } + f, err := os.Open(full) + if err != nil { + // NOT "missing": the file is there and its bytes cannot be read, + // which is a different thing to go and look at. Reporting it as + // missing would send a reader to re-fetch a document that is + // already on the disk. + out = append(out, Problem{Path: e.Path, Kind: "unreadable", Got: err.Error()}) + continue + } + sum, _, err := Digest(f, io.Discard) + f.Close() + if err != nil { + return nil, fmt.Errorf("%s: %w", e.Path, err) + } + if n := len(e.SHA256); !strings.EqualFold(sum[:min(n, len(sum))], e.SHA256) { + out = append(out, Problem{Path: e.Path, Kind: "digest", + Want: e.SHA256, Got: sum[:min(n, len(sum))]}) + } + } + // And the other direction, which is the one nothing here could see. + err = filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || !strings.EqualFold(filepath.Ext(p), ".pdf") { + return nil + } + // Trimmed rather than filepath.Rel'd: WalkDir hands back paths that + // begin with the root it was given, so there is no case where Rel + // fails and an error branch nothing can reach is a branch nobody can + // test. + rel := strings.TrimPrefix(p, dir+string(filepath.Separator)) + if !recorded[rel] { + out = append(out, Problem{Path: rel, Kind: "unrecorded"}) + } + return nil + }) + if err != nil { + return nil, err + } + sort.Slice(out, func(i, j int) bool { + if out[i].Kind != out[j].Kind { + return out[i].Kind < out[j].Kind + } + return out[i].Path < out[j].Path + }) + return out, nil +} diff --git a/corpus/check_test.go b/corpus/check_test.go new file mode 100644 index 0000000..c1e8d92 --- /dev/null +++ b/corpus/check_test.go @@ -0,0 +1,245 @@ +package corpus + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +// put writes a document into a corpus and returns the entry that describes it, +// with the digest cut to n characters so a manifest that records a PREFIX can +// be tested as such. The forms corpus keeps sixteen. +func put(t *testing.T, dir, path, body string, n int) Entry { + t.Helper() + full := filepath.Join(dir, path) + if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(full, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + f, err := os.Open(full) + if err != nil { + t.Fatal(err) + } + defer f.Close() + sum, size, err := Digest(f, nilWriter{}) + if err != nil { + t.Fatal(err) + } + if n > 0 && n < len(sum) { + sum = sum[:n] + } + return Entry{Path: path, Origin: strings.SplitN(path, "/", 2)[0], + Source: "https://x/" + path, Bytes: size, SHA256: sum, + Fetched: time.Date(2026, 10, 4, 0, 0, 0, 0, time.UTC)} +} + +type nilWriter struct{} + +func (nilWriter) Write(p []byte) (int, error) { return len(p), nil } + +func TestACorpusThatAgreesWithItsManifest(t *testing.T) { + dir := t.TempDir() + // One row with the full digest and one with sixteen characters of it, + // because the two corpora this repository is run against do exactly that + // and a check that demanded all sixty-four would call 2 268 rows changed. + es := []Entry{ + put(t, dir, "a/one.pdf", "first", 0), + put(t, dir, "b/two.pdf", "second", 16), + } + if err := Write(dir, es); err != nil { + t.Fatal(err) + } + got, err := Check(dir) + if err != nil { + t.Fatal(err) + } + if len(got) != 0 { + t.Errorf("a corpus that agrees reported %v", got) + } +} + +func TestEveryWayACorpusCanDisagree(t *testing.T) { + dir := t.TempDir() + es := []Entry{ + put(t, dir, "a/gone.pdf", "will be removed", 0), + put(t, dir, "a/resized.pdf", "same name, other length", 0), + put(t, dir, "a/rewritten.pdf", "same length, other bytes", 0), + put(t, dir, "a/nodigest.pdf", "the manifest did not say", 0), + put(t, dir, "a/kept.pdf", "untouched", 16), + } + es[3].SHA256 = "" // some documents predate the record that describes them + if err := Write(dir, es); err != nil { + t.Fatal(err) + } + if err := os.Remove(filepath.Join(dir, "a/gone.pdf")); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "a/resized.pdf"), []byte("shorter"), 0o644); err != nil { + t.Fatal(err) + } + // SAME LENGTH, other bytes: the size check cannot see this one and the + // digest is the only thing that can. + if err := os.WriteFile(filepath.Join(dir, "a/rewritten.pdf"), + []byte("same length, OTHER bytes"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "a/stranger.pdf"), []byte("nobody asked"), 0o644); err != nil { + t.Fatal(err) + } + // A file that is not a PDF is not a document of the corpus and must not + // be reported: a corpus directory holds a manifest, and may hold notes. + if err := os.WriteFile(filepath.Join(dir, "a/README.md"), []byte("notes"), 0o644); err != nil { + t.Fatal(err) + } + + got, err := Check(dir) + if err != nil { + t.Fatal(err) + } + want := map[string]string{ + "a/gone.pdf": "missing", + "a/resized.pdf": "size", + "a/rewritten.pdf": "digest", + "a/stranger.pdf": "unrecorded", + } + if len(got) != len(want) { + t.Fatalf("got %v", got) + } + for _, p := range got { + if want[p.Path] != p.Kind { + t.Errorf("%s came back as %q, want %q", p.Path, p.Kind, want[p.Path]) + } + if p.String() == "" { + t.Errorf("%+v has no message", p) + } + } + // Sorted by kind then path, so two runs over the same corpus print the + // same thing and a diff of two reports is readable. + for i := 1; i < len(got); i++ { + if got[i-1].Kind > got[i].Kind { + t.Errorf("not sorted by kind: %v", got) + } + } +} + +func TestAFileThatWillNotOpenIsNotReportedAsMissing(t *testing.T) { + // Reporting it as missing would send a reader to re-fetch a document that + // is already on the disk. + dir := t.TempDir() + e := put(t, dir, "a/locked.pdf", "unreadable", 0) + if err := Write(dir, []Entry{e}); err != nil { + t.Fatal(err) + } + full := filepath.Join(dir, "a/locked.pdf") + if err := os.Chmod(full, 0); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { os.Chmod(full, 0o644) }) + if os.Geteuid() == 0 { + t.Skip("root reads a mode-0 file, so there is nothing to see") + } + got, err := Check(dir) + if err != nil { + t.Fatal(err) + } + if len(got) != 1 || got[0].Kind != "unreadable" { + t.Fatalf("got %v", got) + } + if !strings.Contains(got[0].String(), "will not open") { + t.Errorf("the message reads %q", got[0].String()) + } +} + +func TestACheckOfSomethingThatIsNotACorpus(t *testing.T) { + // A directory with no manifest is an empty corpus rather than an error, + // which Read already decides -- so a check of one reports only what is on + // disk and in no row. + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "lone.pdf"), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + got, err := Check(dir) + if err != nil { + t.Fatal(err) + } + if len(got) != 1 || got[0].Kind != "unrecorded" { + t.Fatalf("got %v", got) + } + // And a manifest that cannot be parsed is an error rather than an empty + // corpus, or a corrupt record would read as a corpus that agrees. + if err := os.WriteFile(filepath.Join(dir, ManifestName), []byte("nonsense\n"), 0o644); err != nil { + t.Fatal(err) + } + if _, err := Check(dir); err == nil { + t.Error("a manifest with no header came back as a corpus") + } +} + +func TestARowThatNamesADirectory(t *testing.T) { + // The bytes of a directory cannot be read, and the walk never offers one, + // so only a manifest row can bring it here. It must come back as an error + // rather than as agreement. + dir := t.TempDir() + if err := os.MkdirAll(filepath.Join(dir, "a/folder.pdf"), 0o755); err != nil { + t.Fatal(err) + } + e := Entry{Path: "a/folder.pdf", Origin: "a", Source: "https://x/1", + Bytes: 0, SHA256: "deadbeef", Fetched: time.Now().UTC()} + if err := Write(dir, []Entry{e}); err != nil { + t.Fatal(err) + } + if _, err := Check(dir); err == nil { + t.Error("a row naming a directory came back without an error") + } +} + +func TestTwoProblemsOfOneKindReadInOrder(t *testing.T) { + // Sorted by kind AND then by path, so two runs over the same corpus print + // the same thing and a diff of two reports is about the corpus. + dir := t.TempDir() + e := put(t, dir, "a/kept.pdf", "untouched", 0) + if err := Write(dir, []Entry{e}); err != nil { + t.Fatal(err) + } + for _, name := range []string{"a/zeta.pdf", "a/alpha.pdf"} { + if err := os.WriteFile(filepath.Join(dir, name), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + } + got, err := Check(dir) + if err != nil { + t.Fatal(err) + } + if len(got) != 2 || got[0].Path != "a/alpha.pdf" || got[1].Path != "a/zeta.pdf" { + t.Fatalf("got %v", got) + } +} + +func TestADirectoryTheWalkCannotEnter(t *testing.T) { + // A corpus half of which cannot be listed must be an error and not a + // report of the half that could: "no problems" over an unreadable + // directory is the worst answer available. + if os.Geteuid() == 0 { + t.Skip("root enters a mode-0 directory, so there is nothing to see") + } + dir := t.TempDir() + e := put(t, dir, "a/kept.pdf", "untouched", 0) + if err := Write(dir, []Entry{e}); err != nil { + t.Fatal(err) + } + shut := filepath.Join(dir, "b") + if err := os.MkdirAll(shut, 0o755); err != nil { + t.Fatal(err) + } + if err := os.Chmod(shut, 0); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { os.Chmod(shut, 0o755) }) + if _, err := Check(dir); err == nil { + t.Error("a corpus with an unreadable directory came back without an error") + } +} From 6328264adc00be8d1a10d152882222c3d1cb5241 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Sun, 4 Oct 2026 16:49:58 +0200 Subject: [PATCH 24/28] corpus: compare the manifest's slashes against the manifest's slashes The windows-latest lane caught a defect in the check itself, not in its tests: Entry.Path is slash-separated -- deliberately, so a corpus can be moved -- and WalkDir hands back paths separated by the host's separator. On Windows the comparison was "a\one.pdf" against the manifest's "a/one.pdf", so EVERY file of a corpus came back as being in no manifest row. A check that reports 3 280 disagreements on a clean corpus is worse than no check. filepath.ToSlash on the walked path, which is a no-op on the two platforms I can run and the whole of the fix on the one I cannot. And two tests are skipped there rather than failing: Chmod(0) denies neither a file read nor a directory listing on Windows, so the unreadable cases cannot be built at all. Skipped with the reason, beside the existing skip for root, which cannot build them either. This fix is verified by CI and not locally -- on macOS ToSlash is the identity and the local run cannot tell the two versions apart. Said here because it is the kind of claim worth marking. --- corpus/check.go | 9 ++++++++- corpus/check_test.go | 8 ++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/corpus/check.go b/corpus/check.go index d6993c0..a070d6c 100644 --- a/corpus/check.go +++ b/corpus/check.go @@ -116,7 +116,14 @@ func Check(dir string) ([]Problem, error) { // begin with the root it was given, so there is no case where Rel // fails and an error branch nothing can reach is a branch nobody can // test. - rel := strings.TrimPrefix(p, dir+string(filepath.Separator)) + // + // ToSlash because Entry.Path is slash-separated -- deliberately, so a + // corpus can be moved -- and WalkDir is separated by the host's. On + // Windows the first version of this compared "a\one.pdf" against the + // manifest's "a/one.pdf" and reported EVERY file of the corpus as + // being in no row. Caught by the windows-latest lane, which is what + // that lane is for. + rel := filepath.ToSlash(strings.TrimPrefix(p, dir+string(filepath.Separator))) if !recorded[rel] { out = append(out, Problem{Path: rel, Kind: "unrecorded"}) } diff --git a/corpus/check_test.go b/corpus/check_test.go index c1e8d92..a70147a 100644 --- a/corpus/check_test.go +++ b/corpus/check_test.go @@ -3,6 +3,7 @@ package corpus import ( "os" "path/filepath" + "runtime" "strings" "testing" "time" @@ -142,6 +143,10 @@ func TestAFileThatWillNotOpenIsNotReportedAsMissing(t *testing.T) { if os.Geteuid() == 0 { t.Skip("root reads a mode-0 file, so there is nothing to see") } + if runtime.GOOS == "windows" { + // Chmod(0) does not deny a read there, so the case cannot be built. + t.Skip("a mode-0 file is still readable on Windows") + } got, err := Check(dir) if err != nil { t.Fatal(err) @@ -226,6 +231,9 @@ func TestADirectoryTheWalkCannotEnter(t *testing.T) { if os.Geteuid() == 0 { t.Skip("root enters a mode-0 directory, so there is nothing to see") } + if runtime.GOOS == "windows" { + t.Skip("a mode-0 directory is still listable on Windows") + } dir := t.TempDir() e := put(t, dir, "a/kept.pdf", "untouched", 0) if err := Write(dir, []Entry{e}); err != nil { From 0d7d21ad137f263f24fd1be0c4fa7b20fa4f3b8a Mon Sep 17 00:00:00 2001 From: tannevaled Date: Sun, 4 Oct 2026 17:51:56 +0200 Subject: [PATCH 25/28] images: a page we drew no picture for is still judged MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit judgePage returned nil as soon as render.Images handed back an empty slice, so a page we produce nothing for produced no row -- not a refusal, not an unseen picture, nothing. It has done that since it was written. The corpus's WORST page is one of those. Page 1 of sim_unitarian-...-1825-06-25_4_25.pdf: render.Images -> 0 images, err == nil render.Page -> 1269x1775, 154 055 dark pixels pdfimages -> 3 pictures of 7048x9856 Each picture is 69 465 088 pixels, 3.5% past the per-picture ceiling, and decodeBase drops it by returning nil -- which is not an error. compare measures that page at 19.23% of pixels differing from poppler, the worst of 3 214, and this instrument measured it not at all. Filed as go-pdfkit/render#108. Which also makes the count added in §32 an UNDERCOUNT, and systematically so for the heaviest pages -- the ones a reader would most want it for. So the judge is asked, and what it took out is counted as Unseen, which is exactly what that word means. A page neither side produced a picture for returns no row still: that is what most pages of most documents look like, and a row each would bury the ones above. A judge that will not finish is Hung, named. Two mutations, caught: returning nil again, and counting them as Neither. Both run after a go build, because a mutation that does not compile has not been tested -- which is how a mutation "survived" earlier in this branch. 100.0% of statements. --- images/images.go | 37 +++++++++++++++++++++++++++- images/images_test.go | 57 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 93 insertions(+), 1 deletion(-) diff --git a/images/images.go b/images/images.go index cdeb738..8f6b03d 100644 --- a/images/images.go +++ b/images/images.go @@ -506,7 +506,42 @@ func judgePage(d *reader.Document, path string, p int) []Result { return []Result{r} } if len(ours) == 0 { - return nil + // A page we produced NO picture for is not nothing to say, and this + // returned nothing to say for as long as it has existed. + // + // The corpus's worst page is one: page 1 of + // sim_unitarian-...-1825-06-25_4_25.pdf, where pdfimages takes out + // three pictures of 7 048 by 9 856, render.Page draws the page, and + // render.Images hands back an empty slice and NO ERROR -- each picture + // is 69 465 088 pixels, 3.5% past the per-picture ceiling, and is + // dropped in silence. compare measures that page at 19.23% of pixels + // differing, the worst of 3 214, and this instrument measured it not + // at all. See go-pdfkit/render#108. + // + // So the judge is asked, and what it took out is counted as Unseen, + // which is exactly what that word means. It also makes the count added + // in §32 honest: it was an UNDERCOUNT, and systematically so for the + // heaviest pages, which are the ones a reader would most want it for. + shots, tool, jerr := judgeShots(path, p) + switch { + case jerr != nil && tool != "": + return []Result{{Path: path, Page: p, Difference: unjudged(), + Missing: Hung, Tool: tool, + Note: tool + " hung on a page we drew no picture for"}} + case jerr != nil: + // Neither side produced a picture for this page, which is not a + // disagreement and is not worth a row: it is what most pages of + // most documents look like. + return nil + } + out := make([]Result, 0, len(shots)) + for _, t := range shots { + out = append(out, Result{Path: path, Page: p, Difference: unjudged(), + Missing: Unseen, + Note: fmt.Sprintf("we drew no picture for this page; the judge took out a %s of %dx%d (row %d, object %d)", + t.kind, t.pic.W, t.pic.H, t.num, t.object)}) + } + return out } theirs, tool, err := judgeShots(path, p) if err != nil { diff --git a/images/images_test.go b/images/images_test.go index 6ccc8d8..574f3a7 100644 --- a/images/images_test.go +++ b/images/images_test.go @@ -1953,3 +1953,60 @@ func TestARowTheJudgeListedTwiceIsNotAPictureWeMissed(t *testing.T) { t.Error("no result carried the repeat") } } + +// TestAPageWeDrewNoPictureForIsStillJudged is the hole the corpus's worst page +// fell through. +// +// page 1 of sim_unitarian-...-1825-06-25_4_25.pdf: pdfimages takes out three +// pictures of 7 048 by 9 856, render.Page draws the page, and render.Images +// hands back an empty slice and NO ERROR -- each picture is 3.5% past the +// per-picture ceiling and is dropped in silence. compare measures that page at +// 19.23% of pixels differing, the worst of 3 214 pages, and this instrument +// measured it not at all: judgePage returned nil before the judge was asked. +// +// Which also made §32's count an UNDERCOUNT, and systematically so for the +// heaviest pages -- the ones a reader would most want it for. +func TestAPageWeDrewNoPictureForIsStillJudged(t *testing.T) { + empty := func(*reader.Writer) reader.Dict { return reader.Dict{} } + + t.Run("the judge took pictures out", func(t *testing.T) { + standIn(t, image.NewRGBA(image.Rect(0, 0, 2, 1)), image.NewRGBA(image.Rect(0, 0, 2, 1))) + got := Judge(pageOfPictures(t, empty), Options{}) + if len(got) != 2 { + t.Fatalf("got %+v", got) + } + for _, r := range got { + if r.Missing != Unseen { + t.Errorf("came back as %q", r.Missing) + } + if !strings.Contains(r.Note, "we drew no picture for this page") { + t.Errorf("the note reads %q", r.Note) + } + } + if s := Summarize("p", 1, got); s.Unseen != 2 { + t.Errorf("the record says %d unseen, want 2", s.Unseen) + } + }) + + t.Run("neither side did", func(t *testing.T) { + // Not a disagreement and not worth a row: it is what most pages of + // most documents look like, and a row each would bury the ones above. + standIn(t) + if got := Judge(pageOfPictures(t, empty), Options{}); len(got) != 0 { + t.Errorf("got %+v", got) + } + }) + + t.Run("the judge would not finish", func(t *testing.T) { + was := popplerCommand + t.Cleanup(func() { popplerCommand = was }) + popplerCommand = func(...string) (bool, error) { return true, os.ErrDeadlineExceeded } + got := Judge(pageOfPictures(t, empty), Options{}) + if len(got) != 1 || got[0].Missing != Hung || got[0].Tool != "pdfimages" { + t.Fatalf("got %+v", got) + } + if !strings.Contains(got[0].Note, "a page we drew no picture for") { + t.Errorf("the note reads %q", got[0].Note) + } + }) +} From d21b1c6f29bd54cc7c088eeaa3d76dc91b7716ff Mon Sep 17 00:00:00 2001 From: tannevaled Date: Sun, 4 Oct 2026 19:52:00 +0200 Subject: [PATCH 26/28] Treat the manifest and a fetched name as data something else wrote MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A security audit of the harness, from one question: what in here is data that something else wrote? Three answers, and two had no guard. 1. A MANIFEST PATH WAS READ VERBATIM AND JOINED TO THE CORPUS. Entry.Path comes out of MANIFEST.tsv -- a data file beside the documents, in a shared directory, which nothing here necessarily wrote -- and was joined to the corpus directory by os.Stat, os.Open, a hash and the argv of a poppler invocation. filepath.Join neutralises an ABSOLUTE path by construction and does NOT neutralise "..": Join("/corpus", "../../etc/passwd") == "/etc/passwd" corpus.Read now refuses such a row. Refuses rather than cleans: a row trying to leave the corpus is not a row with a typo, and rewriting it would put a file in a measurement under a name the manifest did not give. 2. A REMOTE SERVER'S IDENTIFIER BECAME A LOCAL FILE NAME. harvest names what it fetches ReplaceAll(id, "/", "_") + ".pdf", where id comes from the archive's search results. The slash was replaced, which blocks a directory escape; a leading dash was not. Every poppler tool parses its arguments with getopt and the document path is positional, so a corpus file called "-v.pdf" makes pdfimages print its version and take no picture -- and the harness records the page as one the judge drew nothing for. A wrong measurement, from a file name. Guarded twice on purpose: internal/poppler.Document prefixes "./" at every call site (pdfinfo, pdfimages, pdfimages -list, pdftoppm), and corpus.safeName strips a leading "-" or "." when the file is created so the awkward name never enters the corpus. The "." goes for §33's reason: a file nothing lists is measured by every tool that walks the manifest and by none that walks the directory. 3. What was already right is stated in §36 rather than assumed: no shell anywhere (exec takes an argv, so command injection was never available -- argument injection was), every poppler call bounded and a bound that fires named, a decode charged what it will cost before a byte is spent, and temporary files under os.MkdirTemp. Both fixes mutation-checked, each after a go build: accepting a ".." row fails two cases, leaving a leading dash alone fails the poppler test. 100.0% of statements. --- baseline/README.md | 70 ++++++++++++++++++++++++++++++++ compare/compare.go | 3 +- corpus/harvest.go | 30 +++++++++++++- corpus/harvest_test.go | 27 ++++++++++++ corpus/manifest.go | 34 ++++++++++++++++ corpus/manifest_test.go | 53 ++++++++++++++++++++++++ images/images.go | 7 ++-- internal/poppler/poppler.go | 28 +++++++++++++ internal/poppler/poppler_test.go | 24 +++++++++++ 9 files changed, 271 insertions(+), 5 deletions(-) create mode 100644 corpus/harvest_test.go diff --git a/baseline/README.md b/baseline/README.md index 5d30919..9103d50 100644 --- a/baseline/README.md +++ b/baseline/README.md @@ -3331,3 +3331,73 @@ the `baseline` record type live in `cmd/images`, package `main`, so `compare` cannot import them; moving them to an internal package touches the very files an unmerged pull request is changing. Named, with the reason, rather than started. + +## §36 — A security audit of the harness, and the two trust boundaries it was not treating as such + +Everything else in this file measures fidelity. This section is about the +tools, and it comes from asking one question of them: **what in here is data +that something else wrote?** + +Three answers, and two of them had no guard. + +### 1. A manifest path was read verbatim and joined to the corpus + +`Entry.Path` comes out of `MANIFEST.tsv`, a data file that travels beside the +documents in a shared directory, and which nothing in this repository +necessarily wrote — the forms corpus was gathered before this repository +existed, which the header-reading code exists for. That path was then joined to +the corpus directory by `os.Stat`, `os.Open`, a hash, and the argument list of a +poppler invocation. + +`filepath.Join` neutralises an **absolute** path by construction: +`Join("/corpus", "/etc/passwd")` is `/corpus/etc/passwd`. It does **not** +neutralise `..`: + + Join("/corpus", "../../etc/passwd") == "/etc/passwd" + +So a manifest row could name a file outside the corpus and every tool here would +read it. `corpus.Read` now **refuses** such a row — refuses rather than cleans, +because a row trying to leave the corpus is not a row with a typo, and +rewriting it would put a file in a measurement under a name the manifest did not +give. + +### 2. A remote server's identifier became a local file name, and a file name beginning with `-` is a flag + +`harvest` names what it fetches `strings.ReplaceAll(id, "/", "_") + ".pdf"`, +where `id` comes from the archive's **search results**. The `/` was replaced, +which blocks a directory escape. A leading `-` was not. + +Every poppler tool parses its arguments with getopt, and the document path is +positional. A corpus file called `-v.pdf` makes `pdfimages` print its version +and take no picture — and the harness then records the page as one **the judge +drew nothing for**. A wrong measurement, from a file name. + +Guarded twice, deliberately: + +| | | +|---|---| +| `internal/poppler.Document` | prefixes `./` when a path would be read as a flag, at every call — `pdfinfo`, `pdfimages`, `pdfimages -list`, `pdftoppm` | +| `corpus.safeName` | strips a leading `-` or `.` when the file is created, so the awkward name never enters the corpus | + +A leading `.` goes too, for a reason this document already has a section about: +a corpus file nothing lists is measured by every tool that walks the manifest +and by no tool that walks the directory, which is §33's disagreement. + +### 3. What was already right, and is worth saying + +- **No shell, anywhere.** `exec.CommandContext(ctx, name, args...)` takes an + argv, so no amount of quoting in a file name reaches a shell. Command + injection was never available; argument injection was. +- **Every poppler call is bounded** (§21, §22) — a hostile document cannot hold + a run open, and a bound that fires is *named* rather than retried. +- **A decode is bounded by what it will cost**, not by what the file claims: + §25 and §26 are that work, and `afford` charges a page before a byte is + spent, so *"a 208 KB file names 87 GB"* is refused rather than attempted. +- Temporary files go through `os.MkdirTemp` and are removed with the directory, + so nothing is written to a predictable path. + +Both fixes are mutation-checked: accepting a `..` row fails two cases of +`TestAManifestRowThatLeavesTheCorpus`, and leaving a leading dash alone fails +`TestAPathThatWouldBeReadAsAFlag`. **Neither is a finding about a document in +the corpus** — they are findings about the instrument, which is what an audit of +an instrument should produce. diff --git a/compare/compare.go b/compare/compare.go index 37f36fd..8307927 100644 --- a/compare/compare.go +++ b/compare/compare.go @@ -224,7 +224,8 @@ func draw(path string, page int, dpi float64) (*raster.Image, time.Duration, boo stem := filepath.Join(dir, "p") start := time.Now() hung, err := popplerCommand("-cropbox", "-r", fmt.Sprint(int(dpi)), - "-f", fmt.Sprint(page), "-l", fmt.Sprint(page), "-png", path, stem) + "-f", fmt.Sprint(page), "-l", fmt.Sprint(page), "-png", + poppler.Document(path), stem) took := time.Since(start) if hung { return nil, took, true, err diff --git a/corpus/harvest.go b/corpus/harvest.go index a78a5c0..2e82d03 100644 --- a/corpus/harvest.go +++ b/corpus/harvest.go @@ -169,7 +169,7 @@ func fetchOne(ctx context.Context, a *Archive, p Plan, into, id string) (Entry, if p.MaxBytes > 0 && f.Bytes > p.MaxBytes { return Entry{}, fmt.Errorf("%d bytes, over the %d limit", f.Bytes, p.MaxBytes) } - name := strings.ReplaceAll(id, "/", "_") + ".pdf" + name := safeName(id) path := filepath.Join(into, name) w, err := os.Create(path) if err != nil { @@ -215,3 +215,31 @@ func looksLikePDF(path string) bool { } return string(head[:]) == "%PDF-" } + +// safeName turns an archive identifier into a local file name. +// +// The identifier comes out of a REMOTE SERVER's search results, and this is +// where it becomes a path on disk and a row in a manifest. Two characters have +// to go: +// +// "/" because it would make a directory, which is why it was already +// replaced, and because a manifest path is one segment under an origin. +// "-" at the START, because every poppler tool parses its arguments with +// getopt and a positional path beginning with a dash is read as a FLAG. +// internal/poppler.Document also guards that at the call, which is the +// belt; this is the braces, and it keeps the awkward name out of the +// corpus rather than working around it at one consumer. +// +// A leading dot goes too: a corpus file nothing lists is a document that is +// measured by no tool that walks the directory and by every tool that walks +// the manifest, which is exactly the disagreement §33 of the baseline is about. +func safeName(id string) string { + name := strings.ReplaceAll(id, "/", "_") + name = strings.TrimLeft(name, "-.") + if name == "" { + // An identifier of nothing but dashes and dots. Keep something: a + // refusal here would drop a document for its name. + name = "document" + } + return name + ".pdf" +} diff --git a/corpus/harvest_test.go b/corpus/harvest_test.go new file mode 100644 index 0000000..8a8d0ed --- /dev/null +++ b/corpus/harvest_test.go @@ -0,0 +1,27 @@ +package corpus + +import "testing" + +func TestAnIdentifierThatWouldMakeAnAwkwardFile(t *testing.T) { + // The identifier comes out of a remote server's search results, and this + // is where it becomes a path on disk and a row in a manifest. + for _, c := range []struct{ id, want string }{ + // A dash at the start is read as a FLAG by every poppler tool. + {"-v", "v.pdf"}, + {"--help", "help.pdf"}, + // A slash would make a directory; already handled, kept tested. + {"a/b", "a_b.pdf"}, + // A dot at the start hides the file from a directory walk while the + // manifest still lists it -- the disagreement §33 is about. + {".hidden", "hidden.pdf"}, + // Nothing but dashes and dots: keep something rather than drop a + // document for its name. + {"-.-", "document.pdf"}, + // Everything else is left alone. + {"sim_unitarian-register", "sim_unitarian-register.pdf"}, + } { + if got := safeName(c.id); got != c.want { + t.Errorf("safeName(%q) = %q, want %q", c.id, got, c.want) + } + } +} diff --git a/corpus/manifest.go b/corpus/manifest.go index cbb67e0..a8a1e08 100644 --- a/corpus/manifest.go +++ b/corpus/manifest.go @@ -15,6 +15,7 @@ import ( "fmt" "io" "os" + slashpath "path" "path/filepath" "sort" "strconv" @@ -192,9 +193,42 @@ func (l layout) parse(line string) (Entry, error) { if e.Origin != "" && !strings.ContainsRune(e.Path, '/') { e.Path = e.Origin + "/" + e.Path } + if err := inside(e.Path); err != nil { + return e, err + } return e, nil } +// inside refuses a row whose path is not under the corpus directory. +// +// The path is read verbatim out of a DATA FILE and then joined to the corpus +// directory by every tool here -- os.Stat, os.Open, a hash, and the argument +// list of a poppler invocation. filepath.Join neutralises an ABSOLUTE path by +// construction ("/corpus" + "/etc/passwd" is "/corpus/etc/passwd") and does +// NOT neutralise "..": Join("/corpus", "../../etc/passwd") is "/etc/passwd". +// +// So a manifest is a trust boundary and was not treated as one. It travels +// beside the documents, in a shared directory, and nothing wrote it that this +// repository controls -- the forms corpus was gathered before this repository +// existed, which the header-reading code above exists for. +// +// Refused rather than cleaned. A row that tries to leave the corpus is not a +// row with a typo, and silently rewriting it would put a file in a measurement +// under a name that is not the one the manifest gave. +func inside(p string) error { + if p == "" { + return fmt.Errorf("the row names no path") + } + if slashpath.IsAbs(p) || filepath.IsAbs(p) { + return fmt.Errorf("%q is absolute, and a manifest path is relative to the corpus", p) + } + clean := slashpath.Clean(p) + if clean == ".." || strings.HasPrefix(clean, "../") { + return fmt.Errorf("%q leaves the corpus directory", p) + } + return nil +} + // parseTime accepts the shapes a manifest has been written in, and reports the // zero time for anything else — which Entry.Fetched documents as "the manifest // did not say". diff --git a/corpus/manifest_test.go b/corpus/manifest_test.go index e4a8678..fa8d765 100644 --- a/corpus/manifest_test.go +++ b/corpus/manifest_test.go @@ -219,3 +219,56 @@ func TestARowShortOfAnyColumnIsReported(t *testing.T) { } } } + +func TestAManifestRowThatLeavesTheCorpus(t *testing.T) { + // A manifest is a trust boundary and was not treated as one. Its paths are + // read verbatim out of a data file and then joined to the corpus directory + // by os.Stat, os.Open, a hash, and the argument list of a poppler + // invocation. + // + // filepath.Join neutralises an ABSOLUTE path by construction and does NOT + // neutralise "..": Join("/corpus", "../../etc/passwd") is "/etc/passwd". + for _, c := range []struct { + name, path string + ok bool + }{ + {"a plain row", "a/one.pdf", true}, + {"a deeper one", "a/b/one.pdf", true}, + {"a dot segment that goes nowhere", "a/./one.pdf", true}, + {"a dot-dot that stays inside", "a/b/../one.pdf", true}, + {"a dot-dot that leaves", "../one.pdf", false}, + {"a dot-dot that leaves the long way", "a/../../one.pdf", false}, + {"nothing but dot-dot", "..", false}, + {"an absolute slash path", "/etc/passwd", false}, + {"empty", "", false}, + } { + err := inside(c.path) + if (err == nil) != c.ok { + t.Errorf("%s: inside(%q) = %v, want ok=%v", c.name, c.path, err, c.ok) + } + } + + // And it has to refuse at READ, not only in a helper: a corrupt record + // that reads as a corpus is the failure this is for. + dir := t.TempDir() + body := "path\torigin\tsource\tbytes\tsha256\tfetched\n" + + "../../escape.pdf\ta\thttps://x/1\t1\taa\t2026-10-04T00:00:00Z\n" + if err := os.WriteFile(filepath.Join(dir, ManifestName), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + if _, err := Read(dir); err == nil { + t.Error("a row that leaves the corpus was read as a corpus") + } else if !strings.Contains(err.Error(), "leaves the corpus") { + t.Errorf("the refusal reads %q", err) + } +} + +func TestAWindowsAbsolutePathInAManifest(t *testing.T) { + // filepath.IsAbs is asked as well as the slash one, because "C:\x" is + // absolute on Windows and a bare relative name everywhere else -- and a + // manifest travels between machines, which is the whole reason its paths + // are slash-separated. + if err := inside(`C:\Windows\System32\drivers\etc\hosts`); err != nil { + t.Logf("refused on this platform: %v", err) + } +} diff --git a/images/images.go b/images/images.go index 8f6b03d..75a3329 100644 --- a/images/images.go +++ b/images/images.go @@ -1080,7 +1080,7 @@ func calibratedSpace(d *reader.Document, v reader.Object, res reader.Dict) bool // read perfectly well and one it could not read at all, and those are the two // things that must not be confused here. var infoCommand = func(path string) (bool, error) { - _, hung, err := poppler.Run("pdfinfo", path) + _, hung, err := poppler.Run("pdfinfo", poppler.Document(path)) return hung, err } @@ -1154,7 +1154,7 @@ func judgeShots(path string, page int) ([]shot, string, error) { defer os.RemoveAll(dir) stem := filepath.Join(dir, "i") hung, err := popplerCommand("-png", "-f", fmt.Sprint(page), "-l", fmt.Sprint(page), - path, stem) + poppler.Document(path), stem) if hung { return nil, "pdfimages", poppler.DidNotFinish("pdfimages") } @@ -1219,7 +1219,8 @@ func number(name string) int { // that could not be taken at all leaves every picture unclassified, which the // package comment explains is deliberately loud. func listing(path string, page int) (map[int]listRow, bool) { - out, hung, err := listCommand("-list", "-f", fmt.Sprint(page), "-l", fmt.Sprint(page), path) + out, hung, err := listCommand("-list", "-f", fmt.Sprint(page), "-l", fmt.Sprint(page), + poppler.Document(path)) if hung { return nil, true } diff --git a/internal/poppler/poppler.go b/internal/poppler/poppler.go index 3a57248..2d8ef08 100644 --- a/internal/poppler/poppler.go +++ b/internal/poppler/poppler.go @@ -22,6 +22,8 @@ package poppler import ( "context" "os/exec" + "path/filepath" + "strings" "time" ) @@ -88,3 +90,29 @@ type hang struct{ tool string } func (h *hang) Error() string { return h.tool + " did not finish within " + Timeout.String() } + +// Document makes a path safe to hand a poppler tool as a POSITIONAL argument. +// +// Every one of these tools parses its arguments with getopt, so a path that +// begins with "-" is read as a flag and not as a file. None of them is given a +// "--" terminator here because the flags and the file are interleaved +// differently by each one, and a terminator in the wrong place is worse than +// none. +// +// It is reachable, which is why it exists. harvest names a local file after an +// identifier a REMOTE SERVER chose -- strings.ReplaceAll(id, "/", "_") + ".pdf" +// -- so an identifier beginning with a dash becomes a corpus file beginning +// with a dash, and every later invocation hands poppler a flag. "-v.pdf" would +// make pdfimages print its version and take no picture; the harness would then +// record the page as one the judge drew nothing for. +// +// "./" is the fix getopt itself prescribes and it changes nothing else: the +// path still names the same file, and a path that is already absolute or +// already relative-explicit is left alone so that an error message says what a +// reader typed. +func Document(path string) string { + if strings.HasPrefix(path, "-") { + return "." + string(filepath.Separator) + path + } + return path +} diff --git a/internal/poppler/poppler_test.go b/internal/poppler/poppler_test.go index a9e47db..6c642b2 100644 --- a/internal/poppler/poppler_test.go +++ b/internal/poppler/poppler_test.go @@ -1,6 +1,7 @@ package poppler import ( + "path/filepath" "strings" "testing" "time" @@ -76,3 +77,26 @@ func TestCombinedIsBoundedToo(t *testing.T) { t.Fatalf("a tool that does not return was not called a hang: %v", err) } } + +func TestAPathThatWouldBeReadAsAFlag(t *testing.T) { + // harvest names a local file after an identifier a REMOTE SERVER chose, so + // a corpus file can begin with a dash and every poppler tool parses its + // arguments with getopt. "-v.pdf" would make pdfimages print its version + // and take no picture, and the harness would record the page as one the + // judge drew nothing for -- a wrong measurement, from a filename. + for _, c := range []struct{ in, want string }{ + {"-v.pdf", "." + string(filepath.Separator) + "-v.pdf"}, + {"--help.pdf", "." + string(filepath.Separator) + "--help.pdf"}, + // Everything else is left exactly as it was, so an error message says + // what a reader typed. + {"/Users/Shared/pdfscans/a.pdf", "/Users/Shared/pdfscans/a.pdf"}, + {"a.pdf", "a.pdf"}, + {"./a.pdf", "./a.pdf"}, + {"dir/-odd.pdf", "dir/-odd.pdf"}, + {"", ""}, + } { + if got := Document(c.in); got != c.want { + t.Errorf("Document(%q) = %q, want %q", c.in, got, c.want) + } + } +} From 066679b79266885c42f9361907b39e9d8ba432dd Mon Sep 17 00:00:00 2001 From: tannevaled Date: Sun, 4 Oct 2026 20:45:44 +0200 Subject: [PATCH 27/28] =?UTF-8?q?baseline:=20=C2=A735,=20the=20undercount?= =?UTF-8?q?=20measured,=20and=20a=20hung=20column=20that=20was=20empty=20b?= =?UTF-8?q?ecause=20nobody=20asked?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit judgePage returned nil the moment render.Images handed back an empty slice, so a page we produce no picture for produced no row at all. Re-taken over all 23 populations with the fix: fr-cerfa 4153 -> 5351 +1198 gh-safedocs 0 -> 11 +11 ia-americana 24 -> 32 +8 gh-pdfbox 4 -> 10 +6 gh-qpdf 13 -> 19 +6 four others +1 each ---------------------------------------- 23 populations 4277 -> 5510 +1233 = a 29% undercount Nine of twenty-three moved. fr-cerfa's 1 198 come from THIRTEEN documents whose page 1 draws only inline images -- 1 197 of the 1 198 rows carry object 0 -- so those pages did not lose a count, they vanished whole. And §34 had already found the consequence without looking for it: the page compare ranks worst of 3 214, at 19.23%, is one of these, and so is the second worst at 14.70%. ONE CELL NOBODY EXPECTED. Everything else in all 23 records is identical cell for cell, and the exception is not an unseen: gh-qpdf hung: qpdf_qtest_qpdf_shared-unnamed-field.pdf page 1, pdfimages That document draws no picture on page 1, so images NEVER ASKED poppler about it -- which is precisely what this file and the top-level README have both said for weeks to explain hung: 0. The sentence was right and it described a hole, not a corpus. Asking the judge on a page we drew nothing for asks about that document for the first time, and pdfimages does not come back. §34's independent page comparison reached the same file, page and tool. Two instruments, neither looking for it, agreeing on the one document in 3 280 that stops a poppler tool. The top-level README's hung paragraph and its unseen row are corrected to 1 and 5 510. --- README.md | 21 +++++--- baseline/README.md | 70 ++++++++++++++++++++++++++ baseline/pdfforms-ca-cra.json | 2 +- baseline/pdfforms-fr-cerfa.json | 4 +- baseline/pdfforms-fr-impots.json | 2 +- baseline/pdfforms-gh-openpdf.json | 3 +- baseline/pdfforms-gh-pdfbox.json | 4 +- baseline/pdfforms-gh-pdfcpu.json | 2 +- baseline/pdfforms-gh-pypdf.json | 3 +- baseline/pdfforms-gh-qpdf.json | 11 +++- baseline/pdfforms-gh-safedocs.json | 3 +- baseline/pdfforms-gh-verapdf.json | 2 +- baseline/pdfforms-int-wipo.json | 2 +- baseline/pdfforms-uk-govuk.json | 2 +- baseline/pdfforms-us-dol.json | 2 +- baseline/pdfforms-us-irs.json | 2 +- baseline/pdfforms-us-opm.json | 2 +- baseline/pdfforms-us-ssa.json | 2 +- baseline/pdfforms-us-uscis.json | 4 +- baseline/pdfforms-us-uscourts.json | 2 +- baseline/pdfscans-ia-americana.json | 4 +- baseline/pdfscans-ia-biodiversity.json | 2 +- baseline/pdfscans-ia-medical.json | 2 +- baseline/pdfscans-ia-texts.json | 2 +- baseline/pdfscans-ia-uscourts.json | 4 +- 25 files changed, 122 insertions(+), 37 deletions(-) diff --git a/README.md b/README.md index f81738c..bdb0eee 100644 --- a/README.md +++ b/README.md @@ -742,13 +742,18 @@ tool that will not finish is `Hung`, a refusal is `Neither` — and the first thing that changed was a test demanding `Ours` for page nine of a one-page document. See §31. -**`hung` is 0 in all 23 populations, and that is not "nothing hangs".** The -document that hangs draws no picture on its first page, so `images` never asks -poppler about it at all; `compare` does meet it and names it. ~~The bound is -unexercised by `images` on this corpus~~ — **§22 of the baseline strikes that: -the bound fired three times, and it is a property of the MACHINE and not of the -corpus.** What is true of the records beside this file is the narrower thing: -none of these 23 has a `hung` entry. +~~**`hung` is 0 in all 23 populations, and that is not "nothing hangs".**~~ +**`hung` is 1, and the reason it used to be 0 is the reason given here.** The +document that hangs draws no picture on its first page, so `images` never asked +poppler about it at all — and §35 made it ask, on exactly the pages we draw no +picture for. `pdfimages` does not come back from +`qpdf_qtest_qpdf_shared-unnamed-field.pdf`, and §34's page comparison reached +the same file, page and tool independently. The sentence above was right and it +described a hole rather than a corpus. + +~~The bound is unexercised by `images` on this corpus~~ — **§22 of the baseline +strikes that too: the bound fired three times, and it is a property of the +MACHINE and not of the corpus.** ## What it comes to today @@ -776,7 +781,7 @@ instrument needed to say about ITSELF. | | | |---|---| | `sizePaired` | how many pictures were matched to the judge's by SIZE rather than by object number. The weaker matcher, so a large share is a run whose other numbers are worth less. It is **0** here, and §32 says why that zero was not the good news it looked like. | -| `unseen` | how many pictures the judge took out that ours produced nothing for — **the one direction the pairing could not see at all**, because it walks our pictures and appends one result for each. **4 277** over 23 populations; §32 names the three causes and [render#101](https://github.com/go-pdfkit/render/issues/101) and [#104](https://github.com/go-pdfkit/render/issues/104) hold two of them. | +| `unseen` | how many pictures the judge took out that ours produced nothing for — **the one direction the pairing could not see at all**, because it walks our pictures and appends one result for each. **5 510** over 23 populations; §32 names the three causes, §35 is the 29% of them that a page we draw NOTHING for used to hide, and [render#101](https://github.com/go-pdfkit/render/issues/101), [#104](https://github.com/go-pdfkit/render/issues/104) and [#108](https://github.com/go-pdfkit/render/issues/108) hold them. | | `repeated` | how many of the judge's rows name an object we returned once. `pdfimages` lists one row per DRAW and `Images` returns one entry per object, so this is a difference of **unit** and not of fidelity. **3 371**, and counting it apart is what keeps `unseen` meaningful. | All three are `omitempty`, so a record written before them keeps its shape. diff --git a/baseline/README.md b/baseline/README.md index 9103d50..69b9214 100644 --- a/baseline/README.md +++ b/baseline/README.md @@ -3401,3 +3401,73 @@ Both fixes are mutation-checked: accepting a `..` row fails two cases of `TestAPathThatWouldBeReadAsAFlag`. **Neither is a finding about a document in the corpus** — they are findings about the instrument, which is what an audit of an instrument should produce. + +## §35 — The count of §32 was an undercount, and the `hung` column was empty because nobody asked + +`judgePage` returned `nil` the moment `render.Images` handed back an empty +slice. A page we produce no picture for produced **no row** — not a refusal, not +an unseen picture, nothing — and it had done that since it was written. + +§34 found the consequence without looking for it. The page `compare` ranks +**worst of 3 214**, at 19.23% of pixels differing, is one of those pages: + +``` +render.Images -> 0 images, err == nil +render.Page -> 1269x1775, 154 055 dark pixels (it draws) +pdfimages -> 3 pictures of 7048x9856 +``` + +Each of those pictures is 69 465 088 pixels, **3.5% past** the per-picture +ceiling, and `decodeBase` drops it by returning nil, which is not an error +([render#108](https://github.com/go-pdfkit/render/issues/108)). The +second-worst page, at 14.70%, is the same shape 5.5% past it. So **the two pages +at the top of this corpus's page-level disagreement were the two pages the +picture-level instrument could not see.** + +### What it cost, measured over all twenty-three populations + +The judge is now asked on such a page, and what it took out is counted as +`Unseen`: + +| population | before | after | | +|---|---:|---:|---:| +| `fr-cerfa` | 4 153 | **5 351** | +1 198 | +| `gh-safedocs` | 0 | **11** | +11 | +| `ia-americana` | 24 | **32** | +8 | +| `gh-pdfbox` | 4 | 10 | +6 | +| `gh-qpdf` | 13 | 19 | +6 | +| `gh-openpdf`, `gh-pypdf`, `us-uscis`, `ia-uscourts` | | | +1 each | +| **23 populations** | **4 277** | **5 510** | **+1 233 — a 29% undercount** | + +Nine of twenty-three moved. `fr-cerfa`'s 1 198 come from **13 documents whose +page 1 draws only inline images** — 1 197 of the 1 198 rows carry object 0, +which is [render#101](https://github.com/go-pdfkit/render/issues/101) — so those +pages did not merely lose a count, they vanished whole. `gh-safedocs` going from +nought to eleven is the same thing in a test suite built out of inline images. + +**It was an undercount systematically for the heaviest pages**, which are the +ones a reader would most want it for. + +### And one cell nobody expected: `hung` is no longer zero + +Everything else in all 23 records is identical, cell for cell. One thing is +not, and it is not an `unseen`: + + gh-qpdf hung: qpdf_qtest_qpdf_shared-unnamed-field.pdf page 1, pdfimages + +This document draws no picture on page 1, so `images` **never asked poppler +about it** — which is exactly what this file and the top-level README have both +said for weeks to explain `hung: 0`: + +> The document that hangs draws no picture on its first page, so `images` never +> asks poppler about it at all; `compare` does meet it and names it. + +The sentence was right and it was a description of the hole, not of the corpus. +Asking the judge on a page we drew nothing for asks about that document for the +first time, and `pdfimages` does not come back. §34's independent page +comparison hit the same file, the same page, the same tool — **two instruments, +neither looking for it, agreeing on the one document in 3 280 that stops a +poppler tool.** + +So `hung` is 1 and the column has a worked example at last, which is better +than a zero that meant "unasked". diff --git a/baseline/pdfforms-ca-cra.json b/baseline/pdfforms-ca-cra.json index 0a229f4..3d298eb 100644 --- a/baseline/pdfforms-ca-cra.json +++ b/baseline/pdfforms-ca-cra.json @@ -1,6 +1,6 @@ { "corpus": "/Users/Shared/pdfforms", - "taken": "2026-10-04T09:37:56Z", + "taken": "2026-10-04T15:50:55Z", "pages": 1, "gate": 2, "timeout": "2m0s", diff --git a/baseline/pdfforms-fr-cerfa.json b/baseline/pdfforms-fr-cerfa.json index fc43470..c884397 100644 --- a/baseline/pdfforms-fr-cerfa.json +++ b/baseline/pdfforms-fr-cerfa.json @@ -1,6 +1,6 @@ { "corpus": "/Users/Shared/pdfforms", - "taken": "2026-10-04T09:38:07Z", + "taken": "2026-10-04T15:51:09Z", "pages": 1, "gate": 2, "timeout": "2m0s", @@ -70,7 +70,7 @@ "refused": 0, "unopenable": 0, "declined": 0, - "unseen": 4153, + "unseen": 5351, "repeated": 3327, "filters": [ { diff --git a/baseline/pdfforms-fr-impots.json b/baseline/pdfforms-fr-impots.json index a32247b..0aa141a 100644 --- a/baseline/pdfforms-fr-impots.json +++ b/baseline/pdfforms-fr-impots.json @@ -1,6 +1,6 @@ { "corpus": "/Users/Shared/pdfforms", - "taken": "2026-10-04T09:38:44Z", + "taken": "2026-10-04T15:52:08Z", "pages": 1, "gate": 2, "timeout": "2m0s", diff --git a/baseline/pdfforms-gh-openpdf.json b/baseline/pdfforms-gh-openpdf.json index e7b1d4b..e332f1c 100644 --- a/baseline/pdfforms-gh-openpdf.json +++ b/baseline/pdfforms-gh-openpdf.json @@ -1,6 +1,6 @@ { "corpus": "/Users/Shared/pdfforms", - "taken": "2026-10-04T09:38:46Z", + "taken": "2026-10-04T15:52:11Z", "pages": 1, "gate": 2, "timeout": "2m0s", @@ -70,6 +70,7 @@ "refused": 0, "unopenable": 14, "declined": 0, + "unseen": 1, "repeated": 4, "filters": [ { diff --git a/baseline/pdfforms-gh-pdfbox.json b/baseline/pdfforms-gh-pdfbox.json index 818560b..071108a 100644 --- a/baseline/pdfforms-gh-pdfbox.json +++ b/baseline/pdfforms-gh-pdfbox.json @@ -1,6 +1,6 @@ { "corpus": "/Users/Shared/pdfforms", - "taken": "2026-10-04T09:38:47Z", + "taken": "2026-10-04T15:52:14Z", "pages": 1, "gate": 2, "timeout": "2m0s", @@ -70,7 +70,7 @@ "refused": 0, "unopenable": 8, "declined": 0, - "unseen": 4, + "unseen": 10, "repeated": 2, "filters": [ { diff --git a/baseline/pdfforms-gh-pdfcpu.json b/baseline/pdfforms-gh-pdfcpu.json index 54ae229..636e1a5 100644 --- a/baseline/pdfforms-gh-pdfcpu.json +++ b/baseline/pdfforms-gh-pdfcpu.json @@ -1,6 +1,6 @@ { "corpus": "/Users/Shared/pdfforms", - "taken": "2026-10-04T09:38:49Z", + "taken": "2026-10-04T15:52:21Z", "pages": 1, "gate": 2, "timeout": "2m0s", diff --git a/baseline/pdfforms-gh-pypdf.json b/baseline/pdfforms-gh-pypdf.json index 06e5665..21c113f 100644 --- a/baseline/pdfforms-gh-pypdf.json +++ b/baseline/pdfforms-gh-pypdf.json @@ -1,6 +1,6 @@ { "corpus": "/Users/Shared/pdfforms", - "taken": "2026-10-04T09:39:03Z", + "taken": "2026-10-04T15:52:39Z", "pages": 1, "gate": 2, "timeout": "2m0s", @@ -70,6 +70,7 @@ "refused": 0, "unopenable": 1, "declined": 0, + "unseen": 1, "repeated": 30, "filters": [ { diff --git a/baseline/pdfforms-gh-qpdf.json b/baseline/pdfforms-gh-qpdf.json index 51ec454..6f4f245 100644 --- a/baseline/pdfforms-gh-qpdf.json +++ b/baseline/pdfforms-gh-qpdf.json @@ -1,6 +1,6 @@ { "corpus": "/Users/Shared/pdfforms", - "taken": "2026-10-04T09:39:04Z", + "taken": "2026-10-04T15:52:41Z", "pages": 1, "gate": 2, "timeout": "2m0s", @@ -70,7 +70,14 @@ "refused": 0, "unopenable": 0, "declined": 0, - "unseen": 13, + "unseen": 19, + "hung": [ + { + "path": "/Users/Shared/pdfforms/gh-qpdf/qpdf_qtest_qpdf_shared-unnamed-field.pdf", + "page": 1, + "tool": "pdfimages" + } + ], "filters": [ { "filter": "DCTDecode", diff --git a/baseline/pdfforms-gh-safedocs.json b/baseline/pdfforms-gh-safedocs.json index 7c0fc98..28554c8 100644 --- a/baseline/pdfforms-gh-safedocs.json +++ b/baseline/pdfforms-gh-safedocs.json @@ -1,6 +1,6 @@ { "corpus": "/Users/Shared/pdfforms", - "taken": "2026-10-04T09:39:04Z", + "taken": "2026-10-04T15:54:44Z", "pages": 1, "gate": 2, "timeout": "2m0s", @@ -70,6 +70,7 @@ "refused": 0, "unopenable": 5, "declined": 0, + "unseen": 11, "filters": [ { "filter": "DCTDecode", diff --git a/baseline/pdfforms-gh-verapdf.json b/baseline/pdfforms-gh-verapdf.json index 003afae..eaa01c7 100644 --- a/baseline/pdfforms-gh-verapdf.json +++ b/baseline/pdfforms-gh-verapdf.json @@ -1,6 +1,6 @@ { "corpus": "/Users/Shared/pdfforms", - "taken": "2026-10-04T09:39:05Z", + "taken": "2026-10-04T15:54:45Z", "pages": 1, "gate": 2, "timeout": "2m0s", diff --git a/baseline/pdfforms-int-wipo.json b/baseline/pdfforms-int-wipo.json index 331e8d3..6df42e4 100644 --- a/baseline/pdfforms-int-wipo.json +++ b/baseline/pdfforms-int-wipo.json @@ -1,6 +1,6 @@ { "corpus": "/Users/Shared/pdfforms", - "taken": "2026-10-04T09:39:05Z", + "taken": "2026-10-04T15:54:55Z", "pages": 1, "gate": 2, "timeout": "2m0s", diff --git a/baseline/pdfforms-uk-govuk.json b/baseline/pdfforms-uk-govuk.json index ccb0189..fb6d826 100644 --- a/baseline/pdfforms-uk-govuk.json +++ b/baseline/pdfforms-uk-govuk.json @@ -1,6 +1,6 @@ { "corpus": "/Users/Shared/pdfforms", - "taken": "2026-10-04T09:39:05Z", + "taken": "2026-10-04T15:55:00Z", "pages": 1, "gate": 2, "timeout": "2m0s", diff --git a/baseline/pdfforms-us-dol.json b/baseline/pdfforms-us-dol.json index c64f445..0b4699b 100644 --- a/baseline/pdfforms-us-dol.json +++ b/baseline/pdfforms-us-dol.json @@ -1,6 +1,6 @@ { "corpus": "/Users/Shared/pdfforms", - "taken": "2026-10-04T09:39:46Z", + "taken": "2026-10-04T15:56:05Z", "pages": 1, "gate": 2, "timeout": "2m0s", diff --git a/baseline/pdfforms-us-irs.json b/baseline/pdfforms-us-irs.json index e57ec09..467e270 100644 --- a/baseline/pdfforms-us-irs.json +++ b/baseline/pdfforms-us-irs.json @@ -1,6 +1,6 @@ { "corpus": "/Users/Shared/pdfforms", - "taken": "2026-10-04T09:39:48Z", + "taken": "2026-10-04T15:56:13Z", "pages": 1, "gate": 2, "timeout": "2m0s", diff --git a/baseline/pdfforms-us-opm.json b/baseline/pdfforms-us-opm.json index fa01bfd..f4a6e95 100644 --- a/baseline/pdfforms-us-opm.json +++ b/baseline/pdfforms-us-opm.json @@ -1,6 +1,6 @@ { "corpus": "/Users/Shared/pdfforms", - "taken": "2026-10-04T09:39:50Z", + "taken": "2026-10-04T15:56:17Z", "pages": 1, "gate": 2, "timeout": "2m0s", diff --git a/baseline/pdfforms-us-ssa.json b/baseline/pdfforms-us-ssa.json index bf7c397..9269e07 100644 --- a/baseline/pdfforms-us-ssa.json +++ b/baseline/pdfforms-us-ssa.json @@ -1,6 +1,6 @@ { "corpus": "/Users/Shared/pdfforms", - "taken": "2026-10-04T09:39:51Z", + "taken": "2026-10-04T15:56:21Z", "pages": 1, "gate": 2, "timeout": "2m0s", diff --git a/baseline/pdfforms-us-uscis.json b/baseline/pdfforms-us-uscis.json index ddde97e..a5faccd 100644 --- a/baseline/pdfforms-us-uscis.json +++ b/baseline/pdfforms-us-uscis.json @@ -1,6 +1,6 @@ { "corpus": "/Users/Shared/pdfforms", - "taken": "2026-10-04T09:39:52Z", + "taken": "2026-10-04T15:56:28Z", "pages": 1, "gate": 2, "timeout": "2m0s", @@ -70,7 +70,7 @@ "refused": 0, "unopenable": 0, "declined": 0, - "unseen": 73, + "unseen": 74, "filters": [ { "filter": "(samples)", diff --git a/baseline/pdfforms-us-uscourts.json b/baseline/pdfforms-us-uscourts.json index f766490..7d31050 100644 --- a/baseline/pdfforms-us-uscourts.json +++ b/baseline/pdfforms-us-uscourts.json @@ -1,6 +1,6 @@ { "corpus": "/Users/Shared/pdfforms", - "taken": "2026-10-04T09:40:07Z", + "taken": "2026-10-04T15:56:47Z", "pages": 1, "gate": 2, "timeout": "2m0s", diff --git a/baseline/pdfscans-ia-americana.json b/baseline/pdfscans-ia-americana.json index d5aff6d..c9308fd 100644 --- a/baseline/pdfscans-ia-americana.json +++ b/baseline/pdfscans-ia-americana.json @@ -1,6 +1,6 @@ { "corpus": "/Users/Shared/pdfscans", - "taken": "2026-10-04T10:08:26Z", + "taken": "2026-10-04T15:57:48Z", "pages": 1, "gate": 2, "timeout": "2m0s", @@ -70,7 +70,7 @@ "refused": 0, "unopenable": 28, "declined": 0, - "unseen": 24, + "unseen": 32, "filters": [ { "filter": "DCTDecode", diff --git a/baseline/pdfscans-ia-biodiversity.json b/baseline/pdfscans-ia-biodiversity.json index 1ffd499..26c1f32 100644 --- a/baseline/pdfscans-ia-biodiversity.json +++ b/baseline/pdfscans-ia-biodiversity.json @@ -1,6 +1,6 @@ { "corpus": "/Users/Shared/pdfscans", - "taken": "2026-10-04T10:52:41Z", + "taken": "2026-10-04T16:50:53Z", "pages": 1, "gate": 2, "timeout": "2m0s", diff --git a/baseline/pdfscans-ia-medical.json b/baseline/pdfscans-ia-medical.json index e1a7183..e9b51d4 100644 --- a/baseline/pdfscans-ia-medical.json +++ b/baseline/pdfscans-ia-medical.json @@ -1,6 +1,6 @@ { "corpus": "/Users/Shared/pdfscans", - "taken": "2026-10-04T11:19:51Z", + "taken": "2026-10-04T17:24:32Z", "pages": 1, "gate": 2, "timeout": "2m0s", diff --git a/baseline/pdfscans-ia-texts.json b/baseline/pdfscans-ia-texts.json index af1f5c9..ef33957 100644 --- a/baseline/pdfscans-ia-texts.json +++ b/baseline/pdfscans-ia-texts.json @@ -1,6 +1,6 @@ { "corpus": "/Users/Shared/pdfscans", - "taken": "2026-10-04T11:55:43Z", + "taken": "2026-10-04T15:56:50Z", "pages": 1, "gate": 2, "timeout": "2m0s", diff --git a/baseline/pdfscans-ia-uscourts.json b/baseline/pdfscans-ia-uscourts.json index 6d54c9d..eba6f93 100644 --- a/baseline/pdfscans-ia-uscourts.json +++ b/baseline/pdfscans-ia-uscourts.json @@ -1,6 +1,6 @@ { "corpus": "/Users/Shared/pdfscans", - "taken": "2026-10-04T11:56:03Z", + "taken": "2026-10-04T15:57:23Z", "pages": 1, "gate": 2, "timeout": "2m0s", @@ -70,7 +70,7 @@ "refused": 0, "unopenable": 0, "declined": 0, - "unseen": 3, + "unseen": 4, "filters": [ { "filter": "DCTDecode", From f1f8af962a7222a157795c2b4fa17b28d00ad79e Mon Sep 17 00:00:00 2001 From: tannevaled Date: Sun, 4 Oct 2026 21:01:39 +0200 Subject: [PATCH 28/28] images: bound what the judge's answer costs to read back MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The audit's fourth finding, from carrying the same question past the inputs a FILE provides to the inputs a TOOL provides. render refuses to decode a page past a ceiling; three png.Decode calls read poppler's answer about the same page with no bound at all. Measured on the page of §35, where our own decoder refuses to spend 277 MB: PNGs pdfimages writes for page 1 3 x 7048 x 9856 on disk 44 MB png.Decode -> three *image.Gray 69.5 MB each raster.FromImage -> RGBA 278 MB each retained 834 MB An instrument that refuses to spend what it asks the other side to spend is measuring two different things, and nothing here limits what poppler is asked to write. maxJudgePixels is the same number render bounds a page by, in the same unit, read from png.DecodeConfig BEFORE a byte is decoded -- for the reason afford() gives: a limit noticed after the allocation has not helped. Today's corpus is inside it (that page is 3 x 69 465 088 = 208 395 264 against 268 435 456), so no figure in baseline/ moves; ia-texts re-taken to confirm, cell for cell. And the silence beside it is gone. A PNG that could not be read was SKIPPED with continue, so the pairing saw fewer of the judge's pictures and said nothing -- the same mistake as §35, three functions away. It is reported now, the way the hung listing beside it already was. readPNG reads the file once and decodes from memory twice rather than seeking one open file back to the start: the Seek error is a branch nothing can reach on a regular file, and a branch nothing can reach is a branch nobody can test. Mutation-checked, each after a go build: dropping the bound lets a picture past the budget be read, restoring the continue turns a page that should be reported into a page measured on nothing. 100.0% of statements. --- baseline/README.md | 39 +++++++++++++++++++-- images/images.go | 68 ++++++++++++++++++++++++++++++++----- images/images_test.go | 79 ++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 173 insertions(+), 13 deletions(-) diff --git a/baseline/README.md b/baseline/README.md index 69b9214..1d5e0c3 100644 --- a/baseline/README.md +++ b/baseline/README.md @@ -3396,9 +3396,42 @@ and by no tool that walks the directory, which is §33's disagreement. - Temporary files go through `os.MkdirTemp` and are removed with the directory, so nothing is written to a predictable path. -Both fixes are mutation-checked: accepting a `..` row fails two cases of -`TestAManifestRowThatLeavesTheCorpus`, and leaving a leading dash alone fails -`TestAPathThatWouldBeReadAsAFlag`. **Neither is a finding about a document in +### 4. The ceiling was one-sided: we bounded our own decode and not the judge's answer + +Found by continuing the same question past the inputs a file provides to the +inputs a TOOL provides. `render` refuses to decode a page past a ceiling; three +`png.Decode` calls read poppler's answer about the same page with no bound at +all. + +Measured on the page of §35, where our own decoder refuses to spend 277 MB: + +| | | +|---|---:| +| PNGs `pdfimages` writes for page 1 | 3 × 7 048 × 9 856 | +| on disk | 44 MB | +| `png.Decode` → three `*image.Gray` | 69.5 MB each | +| `raster.FromImage` → RGBA, which `raster.Image` always is | **278 MB each** | +| **retained** | **834 MB** | + +**An instrument that refuses to spend what it asks the other side to spend is +measuring two different things**, and nothing here limits what poppler is asked +to write. `maxJudgePixels` is now the same number `render` bounds a page by, in +the same unit, checked from `png.DecodeConfig` **before** a byte is decoded — +for the reason `afford` gives: a limit noticed after the allocation has not +helped. Today's corpus is inside it (that page is 3 × 69 465 088 = 208 395 264 +against 268 435 456), so **no figure in `baseline/` moves**; what changes is +that an unbounded read is bounded. + +And the silence beside it is gone. A PNG that could not be read was **skipped** +with `continue`, so the pairing saw fewer of the judge's pictures and said +nothing — the same mistake as §35, three functions away. It is reported now, +the way the hung listing beside it already was. + +Both of the first two fixes are mutation-checked: accepting a `..` row fails two +cases of `TestAManifestRowThatLeavesTheCorpus`, and leaving a leading dash alone +fails `TestAPathThatWouldBeReadAsAFlag`. So are these: dropping the bound lets a +picture past the budget be read, and restoring the `continue` turns a page that +should be reported into a page measured on nothing. **Neither is a finding about a document in the corpus** — they are findings about the instrument, which is what an audit of an instrument should produce. diff --git a/images/images.go b/images/images.go index 75a3329..c07afe2 100644 --- a/images/images.go +++ b/images/images.go @@ -178,6 +178,7 @@ package images import ( + "bytes" "fmt" "image/png" "os" @@ -1175,11 +1176,17 @@ func judgeShots(path string, page int) ([]shot, string, error) { return nil, "pdfimages -list", poppler.DidNotFinish("pdfimages -list") } out := make([]shot, 0, len(names)) + left := maxJudge for _, name := range names { - im, err := readPNG(name) + im, rest, err := readPNG(name, left) if err != nil { - continue + // NOT skipped. A picture of the judge's that silently does not + // arrive makes the pairing see fewer of them, which is a real + // number in a real column -- the same argument the hung listing + // above is reported by, and the same mistake §35 is about. + return nil, "", fmt.Errorf("the judge's own picture could not be read: %w", err) } + left = rest n := number(name) row := spaces[n] out = append(out, shot{pic: im, num: n, space: row.space, @@ -1259,17 +1266,60 @@ type listRow struct { } // readPNG reads one of the files pdfimages wrote. -func readPNG(name string) (*raster.Image, error) { - f, err := os.Open(name) +// maxJudgePixels bounds what the JUDGE's answer about one page may cost to +// read back, and it is deliberately the same number render bounds its own +// decode of a page by. +// +// The two were not symmetric and the asymmetry was large. Measured on page 1 of +// sim_unitarian-...-1825-06-25_4_25.pdf: pdfimages writes three PNGs of +// 7 048 by 9 856, 44 MB on disk; png.Decode makes three *image.Gray of 69.5 MB +// and raster.FromImage turns each into RGBA, which raster.Image always is -- +// FOUR bytes a pixel. 834 MB RETAINED, on the one page where our own decoder +// refuses to spend 277 MB (go-pdfkit/render#108). +// +// An instrument that refuses to spend what it asks the other side to spend is +// measuring two different things, and the bound it does not have is the one a +// hostile document reaches: nothing here limits what poppler is asked to write. +// +// Counted in PIXELS over the whole page, as render counts it, so the two sides +// read the same arithmetic. Today's corpus is inside it -- that page is +// 3 x 69 465 088 = 208 395 264 against 268 435 456 -- so no figure in +// baseline/ moves. What changes is that an unbounded read is bounded. +const maxJudgePixels = 256 << 20 + +// maxJudge is a variable so a test can reach the refusal without writing a +// 256-megapixel PNG, exactly as popplerCommand is one. +var maxJudge = maxJudgePixels + +// readPNG reads one picture the judge wrote. +// +// The configuration is read FIRST and the bound applied before a byte is +// decoded, for the reason render's own afford() gives: a limit noticed after +// the allocation has not helped. +func readPNG(name string, left int) (*raster.Image, int, error) { + // Read in full and decode from memory, twice over the same bytes. Seeking + // one open file back to the start would do it in one read and adds a + // branch nothing can reach on a regular file -- and a branch nothing can + // reach is a branch nobody can test. The file is on disk and bounded by + // what poppler wrote. + b, err := os.ReadFile(name) if err != nil { - return nil, err + return nil, left, err + } + cfg, err := png.DecodeConfig(bytes.NewReader(b)) + if err != nil { + return nil, left, err + } + px := cfg.Width * cfg.Height + if cfg.Width <= 0 || cfg.Height <= 0 || px > left { + return nil, left, fmt.Errorf("%s is %dx%d and %d of %d pixels are left", + filepath.Base(name), cfg.Width, cfg.Height, left, maxJudge) } - defer f.Close() - img, err := png.Decode(f) + img, err := png.Decode(bytes.NewReader(b)) if err != nil { - return nil, err + return nil, left, err } - return raster.FromImage(img), nil + return raster.FromImage(img), left - px, nil } // A Bucket is what one group of a filter's comparable pictures came to. diff --git a/images/images_test.go b/images/images_test.go index 574f3a7..21d3746 100644 --- a/images/images_test.go +++ b/images/images_test.go @@ -657,11 +657,69 @@ func TestAFileTheyWroteThatIsNotAPicture(t *testing.T) { } func TestAFileTheyWroteThatCannotBeOpened(t *testing.T) { - if _, err := readPNG(filepath.Join(t.TempDir(), "gone.png")); err == nil { + if _, _, err := readPNG(filepath.Join(t.TempDir(), "gone.png"), maxJudgePixels); err == nil { t.Error("a file that is not there was read") } } +// TestTheJudgesOwnAnswerIsBoundedToo closes an asymmetry: render refuses to +// decode a page past a ceiling, and nothing bounded what we read BACK from +// poppler about the same page. +// +// Measured on page 1 of sim_unitarian-...-1825-06-25_4_25.pdf, where our own +// decoder refuses to spend 277 MB: pdfimages writes three PNGs of 7 048 by +// 9 856, 44 MB on disk, which png.Decode makes into three *image.Gray of +// 69.5 MB and raster.FromImage turns into RGBA -- raster.Image is always four +// bytes a pixel. 834 MB retained, for a page we declined to read ourselves. +func TestTheJudgesOwnAnswerIsBoundedToo(t *testing.T) { + dir := t.TempDir() + name := filepath.Join(dir, "p-000.png") + f, err := os.Create(name) + if err != nil { + t.Fatal(err) + } + if err := png.Encode(f, image.NewGray(image.Rect(0, 0, 40, 30))); err != nil { + f.Close() + t.Fatal(err) + } + f.Close() + + // Inside the budget: read, and the budget comes back smaller by exactly + // what the picture holds. + im, left, err := readPNG(name, 10_000) + if err != nil { + t.Fatalf("a 40x30 picture was refused with 10 000 pixels left: %v", err) + } + if im.W != 40 || im.H != 30 { + t.Errorf("read it as %dx%d", im.W, im.H) + } + if left != 10_000-40*30 { + t.Errorf("%d pixels left, want %d", left, 10_000-40*30) + } + + // Past it: refused, and refused BEFORE the decode -- the message is about + // the configuration, which is all that was read. + if _, _, err := readPNG(name, 40*30-1); err == nil { + t.Error("a picture past the budget was read") + } else if !strings.Contains(err.Error(), "40x30") { + t.Errorf("the refusal does not say what it refused: %v", err) + } + + // And a page whose pictures do not fit is REPORTED, not quietly measured + // on the ones that did -- the mistake §35 is about. + was := maxJudge + t.Cleanup(func() { maxJudge = was }) + maxJudge = 1 + standIn(t, image.NewRGBA(image.Rect(0, 0, 2, 1))) + got := Judge(pageOfPictures(t, func(w *reader.Writer) reader.Dict { + return reader.Dict{"I": grey(w)} + }), Options{}) + if len(got) != 1 || got[0].Missing != Theirs || + !strings.Contains(got[0].Note, "could not be read") { + t.Fatalf("got %+v", got) + } +} + func TestMorePagesThanTheDocumentHas(t *testing.T) { standIn(t, twoPixels(true)) got := Judge(pageOfPictures(t, func(w *reader.Writer) reader.Dict { @@ -2010,3 +2068,22 @@ func TestAPageWeDrewNoPictureForIsStillJudged(t *testing.T) { } }) } + +func TestAPictureOfTheirsThatStopsHalfWay(t *testing.T) { + // The configuration is read before the bound is applied, so a file whose + // header parses and whose pixels do not is a distinct case from a file + // that is not a PNG at all. + dir := t.TempDir() + var buf bytes.Buffer + if err := png.Encode(&buf, image.NewGray(image.Rect(0, 0, 40, 30))); err != nil { + t.Fatal(err) + } + b := buf.Bytes() + name := filepath.Join(dir, "half.png") + if err := os.WriteFile(name, b[:len(b)/2], 0o644); err != nil { + t.Fatal(err) + } + if _, _, err := readPNG(name, maxJudgePixels); err == nil { + t.Error("half a PNG was read as a picture") + } +}