TextString: what the document says in a human language

textstring.go

@@ -0,0 +1,120 @@
+package reader
+
+import (
+	"strings"
+	"unicode/utf16"
+	"unicode/utf8"
+)
+
+// TextString decodes a PDF text string: what a document says in a human
+// language, as opposed to the bytes of a name, a date or a content stream. A
+// title, an author, a bookmark's label, a form field's value.
+//
+// ⛔ [ToString] is NOT this. It hands back the bytes, and the bytes of a text
+// string are not text: `pdfops info` printed a document's creator as
+// "\xfe\xffL\x00a\x00T\x00e\x00X" — a UTF-16 string read a byte at a time —
+// and that output had been shipping for as long as the verb had.
+//
+// A text string is one of three things (ISO 32000-1 §7.9.2.2 and §7.9.2.2.1 of
+// ISO 32000-2):
+//
+//	FE FF …         UTF-16BE, which is how a document says anything outside
+//	                Latin 1 and most of what any modern producer writes
+//	EF BB BF …      UTF-8, which PDF 2.0 added
+//	anything else   PDFDocEncoding
+//
+// ⛔ Only those two byte orders of mark, and no guessing beyond them. A string
+// that merely looks like UTF-16LE is PDFDocEncoding that happens to contain a
+// zero byte, and decoding it as UTF-16 turns a readable title into mojibake —
+// which is the opposite failure to the one above and just as silent.
+func TextString(o Object) (string, bool) {
+	b, ok := ToString(o)
+	if !ok {
+		return "", false
+	}
+	switch {
+	case len(b) >= 2 && b[0] == 0xFE && b[1] == 0xFF:
+		return fromUTF16BE(b[2:]), true
+	case len(b) >= 3 && b[0] == 0xEF && b[1] == 0xBB && b[2] == 0xBF:
+		return fromUTF8(b[3:]), true
+	}
+	return fromPDFDoc(b), true
+}
+
+// fromUTF16BE decodes big-endian UTF-16, surrogate pairs included.
+func fromUTF16BE(b []byte) string {
+	units := make([]uint16, 0, len(b)/2)
+	for i := 0; i+1 < len(b); i += 2 {
+		units = append(units, uint16(b[i])<<8|uint16(b[i+1]))
+	}
+	out := string(utf16.Decode(units))
+	if len(b)%2 == 1 {
+		// An odd number of bytes: the last one is half a code unit, and half a
+		// code unit is not a character. It is said rather than dropped, so
+		// that a truncated string looks truncated.
+		out += string(utf8.RuneError)
+	}
+	return out
+}
+
+// fromUTF8 keeps valid UTF-8 and replaces what is not, so that the result is
+// always a string Go can print, compare and put in JSON.
+func fromUTF8(b []byte) string {
+	if utf8.Valid(b) {
+		return string(b)
+	}
+	var out strings.Builder
+	for len(b) > 0 {
+		r, size := utf8.DecodeRune(b)
+		out.WriteRune(r)
+		b = b[size:]
+	}
+	return out.String()
+}
+
+// fromPDFDoc decodes PDFDocEncoding, which is Latin 1 with the holes in it
+// filled: the spacing accents at 0x18 to 0x1F where Latin 1 has control
+// characters, the typographic marks at 0x80 to 0x9E, and the Euro at 0xA0.
+func fromPDFDoc(b []byte) string {
+	var out strings.Builder
+	out.Grow(len(b))
+	for _, c := range b {
+		out.WriteRune(pdfDocRune(c))
+	}
+	return out.String()
+}
+
+// pdfDocRune is one byte of PDFDocEncoding.
+func pdfDocRune(c byte) rune {
+	switch {
+	case c >= 0x18 && c <= 0x1F:
+		return pdfDocAccents[c-0x18]
+	case c >= 0x80 && c <= 0x9E:
+		return pdfDocMarks[c-0x80]
+	case c == 0xA0:
+		return '€'
+	case c == 0x7F || c == 0x9F || c == 0xAD:
+		// Undefined by the standard. A byte that means nothing is said to
+		// mean nothing rather than quietly becoming a soft hyphen or a
+		// control character.
+		return utf8.RuneError
+	}
+	// The rest is Latin 1, where the byte IS the code point.
+	return rune(c)
+}
+
+// pdfDocAccents is 0x18 to 0x1F: breve, caron, circumflex, dot accent, double
+// acute, ogonek, ring, tilde. Latin 1 has control characters here, which is
+// the first place a Latin-1 decoder goes wrong on a PDF.
+var pdfDocAccents = [8]rune{
+	'˘', 'ˇ', 'ˆ', '˙', '˝', '˛', '˚', '˜',
+}
+
+// pdfDocMarks is 0x80 to 0x9E, the typographic marks: Annex D.2 of ISO
+// 32000-1, checked against it entry by entry rather than remembered.
+var pdfDocMarks = [31]rune{
+	'•', '†', '‡', '…', '—', '–', 'ƒ', '⁄',
+	'‹', '›', '−', '‰', '„', '“', '”', '‘',
+	'’', '‚', '™', 'fi', 'fl', 'Ł', 'Œ', 'Š',
+	'Ÿ', 'Ž', 'ı', 'ł', 'œ', 'š', 'ž',
+}

textstring_test.go

@@ -0,0 +1,148 @@
+package reader
+
+import (
+	"testing"
+	"unicode/utf8"
+)
+
+// ⛔ THE DEFECT. `pdfops info` printed a document's creator as
+// "\xfe\xffL\x00a\x00T\x00e\x00X" for as long as the verb had existed,
+// because ToString hands back BYTES and the bytes of a text string are not
+// text.
+func TestAUTF16StringIsNotReadAByteAtATime(t *testing.T) {
+	// How xdvipdfmx actually wrote it, BOM and all.
+	raw := "\xfe\xff\x00L\x00a\x00T\x00e\x00X\x00 \x00v\x00i\x00a\x00 \x00p\x00a\x00n\x00d\x00o\x00c"
+	got, ok := TextString(String(raw))
+	if !ok {
+		t.Fatal("a string object was refused")
+	}
+	if got != "LaTeX via pandoc" {
+		t.Errorf("read %q", got)
+	}
+	// And the bytes are still available for whoever wants them.
+	if b, _ := ToString(String(raw)); string(b) != raw {
+		t.Error("ToString no longer hands back the bytes")
+	}
+}
+
+func TestAUTF16StringOutsideLatin1(t *testing.T) {
+	// "Résumé" in UTF-16BE: the accented letters are what a producer reaches
+	// for UTF-16 to say in the first place.
+	raw := "\xfe\xff\x00R\x00\xe9\x00s\x00u\x00m\x00\xe9"
+	if got, _ := TextString(String(raw)); got != "Résumé" {
+		t.Errorf("read %q, want %q", got, "Résumé")
+	}
+}
+
+// A character outside the basic plane arrives as a surrogate pair, and a pair
+// decoded one unit at a time is two replacement characters.
+func TestASurrogatePair(t *testing.T) {
+	// U+1F4C4 PAGE FACING UP is D83D DCC4.
+	raw := "\xfe\xff\xd8\x3d\xdc\xc4"
+	if got, _ := TextString(String(raw)); got != "\U0001F4C4" {
+		t.Errorf("read %q (%d runes)", got, utf8.RuneCountInString(got))
+	}
+}
+
+// Half a code unit is not a character, and a truncated string should look
+// truncated rather than lose its last letter silently.
+func TestAUTF16StringWithAnOddNumberOfBytes(t *testing.T) {
+	raw := "\xfe\xff\x00a\x00"
+	got, _ := TextString(String(raw))
+	if want := "a" + string(utf8.RuneError); got != want {
+		t.Errorf("read %q, want %q", got, want)
+	}
+}
+
+// A BOM and nothing after it is an empty string, not a failure.
+func TestNothingButABOM(t *testing.T) {
+	if got, ok := TextString(String("\xfe\xff")); !ok || got != "" {
+		t.Errorf("read %q, %v", got, ok)
+	}
+}
+
+// PDF 2.0 added UTF-8 with its own mark.
+func TestAUTF8StringWithItsMark(t *testing.T) {
+	if got, _ := TextString(String("\xef\xbb\xbfRésumé")); got != "Résumé" {
+		t.Errorf("read %q", got)
+	}
+	// And bytes that are not UTF-8 after a mark that says they are: each one
+	// becomes a replacement character rather than making the whole string
+	// unprintable.
+	got, _ := TextString(String("\xef\xbb\xbfa\xffb"))
+	if want := "a" + string(utf8.RuneError) + "b"; got != want {
+		t.Errorf("read %q, want %q", got, want)
+	}
+}
+
+// ⛔ PDFDocEncoding is NOT Latin 1, and 0x18 to 0x1F is the first place a
+// Latin-1 decoder goes wrong: Latin 1 has control characters there and PDF
+// has the spacing accents.
+func TestTheAccentsWhereLatin1HasControlCharacters(t *testing.T) {
+	got, _ := TextString(String("\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"))
+	if want := "˘ˇˆ˙˝˛˚˜"; got != want {
+		t.Errorf("read %q, want %q", got, want)
+	}
+}
+
+// The typographic marks at 0x80 to 0x9E, where Latin 1 has nothing at all.
+func TestTheTypographicMarks(t *testing.T) {
+	cases := map[byte]rune{
+		0x80: '•', 0x83: '…', 0x84: '—', 0x85: '–', 0x8D: '“', 0x8E: '”',
+		0x90: '’', 0x92: '™', 0x93: 'fi', 0x96: 'Œ', 0x9E: 'ž',
+	}
+	for b, want := range cases {
+		got, _ := TextString(String([]byte{b}))
+		if got != string(want) {
+			t.Errorf("0x%02X read %q, want %q", b, got, want)
+		}
+	}
+	// 0xA0 is the Euro, not a non-breaking space.
+	if got, _ := TextString(String("\xa0")); got != "€" {
+		t.Errorf("0xA0 read %q, want €", got)
+	}
+}
+
+// Three bytes the standard leaves undefined. A byte that means nothing is said
+// to mean nothing, rather than quietly becoming a soft hyphen.
+func TestTheBytesTheStandardLeavesUndefined(t *testing.T) {
+	for _, b := range []byte{0x7F, 0x9F, 0xAD} {
+		got, _ := TextString(String([]byte{b}))
+		if got != string(utf8.RuneError) {
+			t.Errorf("0x%02X read %q, want the replacement character", b, got)
+		}
+	}
+}
+
+// The rest of PDFDocEncoding is Latin 1, where the byte is the code point.
+func TestTheRestIsLatin1(t *testing.T) {
+	if got, _ := TextString(String("R\xe9sum\xe9")); got != "Résumé" {
+		t.Errorf("read %q, want %q", got, "Résumé")
+	}
+	if got, _ := TextString(String("plain ASCII")); got != "plain ASCII" {
+		t.Errorf("read %q", got)
+	}
+}
+
+// ⛔ A string that merely LOOKS like UTF-16LE is PDFDocEncoding with a zero
+// byte in it. Guessing at it turns a readable title into mojibake, which is
+// the opposite failure to reading UTF-16BE a byte at a time and just as
+// silent.
+func TestAStringThatLooksLikeUTF16ButHasNoMark(t *testing.T) {
+	// The whole string, not "it does not say La": accepting FF FE as a mark
+	// and then decoding big-endian anyway gives mojibake too, so only the
+	// exact reading tells the two apart.
+	got, _ := TextString(String("\xff\xfeL\x00a\x00"))
+	if want := "ÿþL\x00a\x00"; got != want {
+		t.Errorf("read %q, want %q — it guessed at a little-endian mark", got, want)
+	}
+}
+
+// Something that is not a string at all is refused rather than rendered.
+func TestWhatIsNotAStringAtAll(t *testing.T) {
+	for _, o := range []Object{Name("Title"), Integer(4), nil, Dict{}} {
+		if got, ok := TextString(o); ok {
+			t.Errorf("%T was read as a text string: %q", o, got)
+		}
+	}
+}