corpus: track glued.go, which a bare ignore rule kept out of #24

corpus/.gitignore

@@ -1,7 +1,9 @@
 # Rendered PDFs are large and fully regenerable from urls.txt — keep the
 # page-1 PNG previews (small, useful as a visual record) but not these.
 out/*.pdf
-corpus
+# The binary `go build ./cmd/corpus` leaves here. Anchored: a bare `corpus`
+# also matched cmd/corpus/ and silently kept glued.go out of #24.
+/corpus
 out/**/*.pdf
 out/judges/
 .judges-node/

corpus/cmd/corpus/glued.go

@@ -0,0 +1,120 @@
+// Copyright (c) the go-pdfkit/html2pdf authors. All rights reserved.
+// Use of this source code is governed by a BSD-3-Clause license that can be
+// found in the LICENSE file at the root of this repository.
+
+package main
+
+import (
+	"context"
+	"os/exec"
+	"sort"
+	"strings"
+	"unicode"
+
+	"github.com/go-webengine/engine"
+	"github.com/go-webengine/engine/css"
+	"github.com/go-webengine/engine/dom"
+)
+
+// gluedWords counts the words of the PDF's extracted text that are two of
+// the page's words run together — "PuissanceIT" where the page says
+// "Puissance IT" — the signature of a space lost in layout: a table column
+// narrower than its longest word, a measured advance shorter than the drawn
+// one. Both defects were found on a real report before this census existed,
+// each by hand; it names them in one run, and a raster comparison never
+// would, since the raster draws the same overlap.
+//
+// Two of the page's words form a candidate pair when whitespace separates
+// them in the source, or when they belong to different block-level boxes (a
+// table cell's last word and the next cell's first). An inline boundary
+// with no whitespace — `<b>a</b>b`, `kW<sup>‡</sup>` — is not a pair: the
+// PDF is right to join those. Returns the count and the glued words.
+func gluedWords(e *engine.Engine, doc *engine.Document, pdf string) (int, string) {
+	media := css.Media{Type: css.Print, Width: 1024}
+	sheets := e.LoadStylesheets(context.Background(), doc, media)
+	sm := css.CascadeMedia(doc.Root, media, sheets)
+
+	words := map[string]bool{}
+	pairs := map[string]bool{}
+	// joins are the pairs the page itself runs together somewhere — an
+	// inline boundary with no whitespace. A word that is such a join is
+	// never counted, even if the same two words also stand apart elsewhere
+	// (RFC 9110 has "obs-text )" in its ABNF and "obs-text</a>)" in prose:
+	// the PDF's "obs-text)" is the prose, correctly joined).
+	joins := map[string]bool{}
+	var prevWord string
+	var prevBlock *dom.Node
+	prevEndsSpace := false
+
+	var walk func(n *dom.Node, block *dom.Node, hidden bool)
+	walk = func(n *dom.Node, block *dom.Node, hidden bool) {
+		if n.Type == dom.Element {
+			st := sm[n]
+			if st != nil && st.Display == css.DisplayNone {
+				hidden = true
+			}
+			if n.Tag == "script" || n.Tag == "style" || n.Tag == "noscript" || n.Tag == "template" {
+				return
+			}
+			if st == nil || st.Display != css.DisplayInline {
+				block = n
+			}
+		}
+		if n.Type == dom.Text && !hidden {
+			text := normalizeSpaces(n.Text)
+			toks := strings.Fields(text)
+			if len(toks) > 0 {
+				startsSpace := unicode.IsSpace([]rune(text)[0])
+				if prevWord != "" {
+					if prevEndsSpace || startsSpace || block != prevBlock {
+						pairs[prevWord+toks[0]] = true
+					} else {
+						joins[prevWord+toks[0]] = true
+					}
+				}
+				for i, t := range toks {
+					words[t] = true
+					if i > 0 {
+						pairs[toks[i-1]+t] = true
+					}
+				}
+				prevWord = toks[len(toks)-1]
+				prevBlock = block
+				prevEndsSpace = unicode.IsSpace([]rune(text)[len([]rune(text))-1])
+			}
+		}
+		for _, c := range n.Children {
+			walk(c, block, hidden)
+		}
+	}
+	walk(doc.Root, doc.Root, false)
+
+	out, err := exec.Command("pdftotext", "-enc", "UTF-8", pdf, "-").Output()
+	if err != nil {
+		return -1, ""
+	}
+	seen := map[string]bool{}
+	var glued []string
+	for _, t := range strings.Fields(normalizeSpaces(string(out))) {
+		if !words[t] && pairs[t] && !joins[t] && !seen[t] {
+			seen[t] = true
+			glued = append(glued, t)
+		}
+	}
+	sort.Strings(glued)
+	return len(glued), strings.Join(glued, " ")
+}
+
+// normalizeSpaces turns the no-break spaces (U+00A0, U+202F, U+2007) into
+// plain spaces so a word split on them the way a browser shows it is one
+// word here too, and the PDF's extracted text — which carries them as
+// written — tokenises the same way.
+func normalizeSpaces(s string) string {
+	return strings.Map(func(r rune) rune {
+		switch r {
+		case ' ', ' ', ' ':
+			return ' '
+		}
+		return r
+	}, s)
+}

corpus/cmd/corpus/glued_test.go

@@ -0,0 +1,80 @@
+// Copyright (c) the go-pdfkit/html2pdf authors. All rights reserved.
+// Use of this source code is governed by a BSD-3-Clause license that can be
+// found in the LICENSE file at the root of this repository.
+
+package main
+
+import (
+	"os"
+	"os/exec"
+	"path/filepath"
+	"testing"
+
+	"github.com/go-webengine/engine"
+	"github.com/go-webengine/engine/dom"
+)
+
+// gluedOf renders src with html2pdf and runs the census on the result.
+func gluedOf(t *testing.T, src string) (int, string) {
+	t.Helper()
+	if _, err := exec.LookPath("pdftotext"); err != nil {
+		t.Skip("pdftotext not installed")
+	}
+	pdf, err := renderWithRecover(src, "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	path := filepath.Join(t.TempDir(), "page.pdf")
+	if err := os.WriteFile(path, pdf, 0o644); err != nil {
+		t.Fatal(err)
+	}
+	root, err := dom.Parse(src)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return gluedWords(engine.New(), &engine.Document{Root: root, HTML: src}, path)
+}
+
+// TestGluedWordsNamesAWordDrawnIntoItsNeighbour: a negative margin pulls
+// "IT" back over the end of "Puissance" — the geometry of a table column
+// narrower than its word, reproduced without a table — and pdftotext reads
+// one word where the page has two.
+func TestGluedWordsNamesAWordDrawnIntoItsNeighbour(t *testing.T) {
+	n, words := gluedOf(t, `<html><body><p>Puissance <span style="margin-left:-8px">IT</span> seul</p></body></html>`)
+	if n != 1 || words != "PuissanceIT" {
+		t.Fatalf("glued = %d %q, want 1 \"PuissanceIT\"", n, words)
+	}
+}
+
+// TestGluedWordsIgnoresJoinsThePageAsks: an inline boundary with no
+// whitespace — a superscript dagger, a bold prefix — is one word on the
+// page too, so the PDF joining it is right; a no-break space is a space.
+func TestGluedWordsIgnoresJoinsThePageAsks(t *testing.T) {
+	n, words := gluedOf(t, `<html><body><p>6,0 kW<sup>‡</sup> et <b>ex</b>iste, <i>a</i> b ; 428&nbsp;630 € — Puissance&nbsp;IT.</p></body></html>`)
+	if n != 0 {
+		t.Fatalf("glued = %d %q, want 0", n, words)
+	}
+}
+
+// TestGluedWordsForgivesAJoinThePageMakesElsewhere: RFC 9110 writes
+// "obs-text )" in an ABNF block and "<a>obs-text</a>)" in prose; the PDF's
+// "obs-text)" is the prose, correctly joined, and is not counted even
+// though the same two words stand apart elsewhere.
+func TestGluedWordsForgivesAJoinThePageMakesElsewhere(t *testing.T) {
+	n, words := gluedOf(t, `<html><body><pre>x = ( VCHAR / obs-text )</pre><p>octets (i.e., <a href="#x">obs-text</a>) as</p></body></html>`)
+	if n != 0 {
+		t.Fatalf("glued = %d %q, want 0", n, words)
+	}
+}
+
+// TestGluedWordsCrossesBlockBoundaries: the last word of one cell and the
+// first of the next are a pair even with no whitespace between the cells in
+// the source — that is exactly where a narrow column glues them.
+func TestGluedWordsCrossesBlockBoundaries(t *testing.T) {
+	// Both cells are laid out correctly here, so nothing is glued; the pair
+	// exists all the same (a glued "MémoireCalcul" would be counted).
+	n, words := gluedOf(t, `<html><body><table><tr><td>Mémoire</td><td>Calcul</td></tr></table></body></html>`)
+	if n != 0 {
+		t.Fatalf("glued = %d %q, want 0", n, words)
+	}
+}