diff --git a/corpus/.gitignore b/corpus/.gitignore index 29e67f0..bad3a4f 100644 --- a/corpus/.gitignore +++ b/corpus/.gitignore @@ -1,7 +1,9 @@ # Rendered PDFs are large and fully regenerable from urls.txt — keep the # page-1 PNG previews (small, useful as a visual record) but not these. out/*.pdf -corpus +# The binary `go build ./cmd/corpus` leaves here. Anchored: a bare `corpus` +# also matched cmd/corpus/ and silently kept glued.go out of #24. +/corpus out/**/*.pdf out/judges/ .judges-node/ diff --git a/corpus/cmd/corpus/glued.go b/corpus/cmd/corpus/glued.go new file mode 100644 index 0000000..26484e9 --- /dev/null +++ b/corpus/cmd/corpus/glued.go @@ -0,0 +1,120 @@ +// Copyright (c) the go-pdfkit/html2pdf authors. All rights reserved. +// Use of this source code is governed by a BSD-3-Clause license that can be +// found in the LICENSE file at the root of this repository. + +package main + +import ( + "context" + "os/exec" + "sort" + "strings" + "unicode" + + "github.com/go-webengine/engine" + "github.com/go-webengine/engine/css" + "github.com/go-webengine/engine/dom" +) + +// gluedWords counts the words of the PDF's extracted text that are two of +// the page's words run together — "PuissanceIT" where the page says +// "Puissance IT" — the signature of a space lost in layout: a table column +// narrower than its longest word, a measured advance shorter than the drawn +// one. Both defects were found on a real report before this census existed, +// each by hand; it names them in one run, and a raster comparison never +// would, since the raster draws the same overlap. +// +// Two of the page's words form a candidate pair when whitespace separates +// them in the source, or when they belong to different block-level boxes (a +// table cell's last word and the next cell's first). An inline boundary +// with no whitespace — `ab`, `kW‡` — is not a pair: the +// PDF is right to join those. Returns the count and the glued words. +func gluedWords(e *engine.Engine, doc *engine.Document, pdf string) (int, string) { + media := css.Media{Type: css.Print, Width: 1024} + sheets := e.LoadStylesheets(context.Background(), doc, media) + sm := css.CascadeMedia(doc.Root, media, sheets) + + words := map[string]bool{} + pairs := map[string]bool{} + // joins are the pairs the page itself runs together somewhere — an + // inline boundary with no whitespace. A word that is such a join is + // never counted, even if the same two words also stand apart elsewhere + // (RFC 9110 has "obs-text )" in its ABNF and "obs-text)" in prose: + // the PDF's "obs-text)" is the prose, correctly joined). + joins := map[string]bool{} + var prevWord string + var prevBlock *dom.Node + prevEndsSpace := false + + var walk func(n *dom.Node, block *dom.Node, hidden bool) + walk = func(n *dom.Node, block *dom.Node, hidden bool) { + if n.Type == dom.Element { + st := sm[n] + if st != nil && st.Display == css.DisplayNone { + hidden = true + } + if n.Tag == "script" || n.Tag == "style" || n.Tag == "noscript" || n.Tag == "template" { + return + } + if st == nil || st.Display != css.DisplayInline { + block = n + } + } + if n.Type == dom.Text && !hidden { + text := normalizeSpaces(n.Text) + toks := strings.Fields(text) + if len(toks) > 0 { + startsSpace := unicode.IsSpace([]rune(text)[0]) + if prevWord != "" { + if prevEndsSpace || startsSpace || block != prevBlock { + pairs[prevWord+toks[0]] = true + } else { + joins[prevWord+toks[0]] = true + } + } + for i, t := range toks { + words[t] = true + if i > 0 { + pairs[toks[i-1]+t] = true + } + } + prevWord = toks[len(toks)-1] + prevBlock = block + prevEndsSpace = unicode.IsSpace([]rune(text)[len([]rune(text))-1]) + } + } + for _, c := range n.Children { + walk(c, block, hidden) + } + } + walk(doc.Root, doc.Root, false) + + out, err := exec.Command("pdftotext", "-enc", "UTF-8", pdf, "-").Output() + if err != nil { + return -1, "" + } + seen := map[string]bool{} + var glued []string + for _, t := range strings.Fields(normalizeSpaces(string(out))) { + if !words[t] && pairs[t] && !joins[t] && !seen[t] { + seen[t] = true + glued = append(glued, t) + } + } + sort.Strings(glued) + return len(glued), strings.Join(glued, " ") +} + +// normalizeSpaces turns the no-break spaces (U+00A0, U+202F, U+2007) into +// plain spaces so a word split on them the way a browser shows it is one +// word here too, and the PDF's extracted text — which carries them as +// written — tokenises the same way. +func normalizeSpaces(s string) string { + return strings.Map(func(r rune) rune { + switch r { + case ' ', ' ', ' ': + return ' ' + } + return r + }, s) +} diff --git a/corpus/cmd/corpus/glued_test.go b/corpus/cmd/corpus/glued_test.go new file mode 100644 index 0000000..36c8e22 --- /dev/null +++ b/corpus/cmd/corpus/glued_test.go @@ -0,0 +1,80 @@ +// Copyright (c) the go-pdfkit/html2pdf authors. All rights reserved. +// Use of this source code is governed by a BSD-3-Clause license that can be +// found in the LICENSE file at the root of this repository. + +package main + +import ( + "os" + "os/exec" + "path/filepath" + "testing" + + "github.com/go-webengine/engine" + "github.com/go-webengine/engine/dom" +) + +// gluedOf renders src with html2pdf and runs the census on the result. +func gluedOf(t *testing.T, src string) (int, string) { + t.Helper() + if _, err := exec.LookPath("pdftotext"); err != nil { + t.Skip("pdftotext not installed") + } + pdf, err := renderWithRecover(src, "") + if err != nil { + t.Fatal(err) + } + path := filepath.Join(t.TempDir(), "page.pdf") + if err := os.WriteFile(path, pdf, 0o644); err != nil { + t.Fatal(err) + } + root, err := dom.Parse(src) + if err != nil { + t.Fatal(err) + } + return gluedWords(engine.New(), &engine.Document{Root: root, HTML: src}, path) +} + +// TestGluedWordsNamesAWordDrawnIntoItsNeighbour: a negative margin pulls +// "IT" back over the end of "Puissance" — the geometry of a table column +// narrower than its word, reproduced without a table — and pdftotext reads +// one word where the page has two. +func TestGluedWordsNamesAWordDrawnIntoItsNeighbour(t *testing.T) { + n, words := gluedOf(t, `

Puissance IT seul

`) + if n != 1 || words != "PuissanceIT" { + t.Fatalf("glued = %d %q, want 1 \"PuissanceIT\"", n, words) + } +} + +// TestGluedWordsIgnoresJoinsThePageAsks: an inline boundary with no +// whitespace — a superscript dagger, a bold prefix — is one word on the +// page too, so the PDF joining it is right; a no-break space is a space. +func TestGluedWordsIgnoresJoinsThePageAsks(t *testing.T) { + n, words := gluedOf(t, `

6,0 kW‡ et existe, a b ; 428 630 € — Puissance IT.

`) + if n != 0 { + t.Fatalf("glued = %d %q, want 0", n, words) + } +} + +// TestGluedWordsForgivesAJoinThePageMakesElsewhere: RFC 9110 writes +// "obs-text )" in an ABNF block and "obs-text)" in prose; the PDF's +// "obs-text)" is the prose, correctly joined, and is not counted even +// though the same two words stand apart elsewhere. +func TestGluedWordsForgivesAJoinThePageMakesElsewhere(t *testing.T) { + n, words := gluedOf(t, `
x = ( VCHAR / obs-text )

octets (i.e., obs-text) as

`) + if n != 0 { + t.Fatalf("glued = %d %q, want 0", n, words) + } +} + +// TestGluedWordsCrossesBlockBoundaries: the last word of one cell and the +// first of the next are a pair even with no whitespace between the cells in +// the source — that is exactly where a narrow column glues them. +func TestGluedWordsCrossesBlockBoundaries(t *testing.T) { + // Both cells are laid out correctly here, so nothing is glued; the pair + // exists all the same (a glued "MémoireCalcul" would be counted). + n, words := gluedOf(t, `
MémoireCalcul
`) + if n != 0 { + t.Fatalf("glued = %d %q, want 0", n, words) + } +}