diff --git a/textstring.go b/textstring.go new file mode 100644 index 0000000..7d0c8a1 --- /dev/null +++ b/textstring.go @@ -0,0 +1,120 @@ +package reader + +import ( + "strings" + "unicode/utf16" + "unicode/utf8" +) + +// TextString decodes a PDF text string: what a document says in a human +// language, as opposed to the bytes of a name, a date or a content stream. A +// title, an author, a bookmark's label, a form field's value. +// +// ⛔ [ToString] is NOT this. It hands back the bytes, and the bytes of a text +// string are not text: `pdfops info` printed a document's creator as +// "\xfe\xffL\x00a\x00T\x00e\x00X" — a UTF-16 string read a byte at a time — +// and that output had been shipping for as long as the verb had. +// +// A text string is one of three things (ISO 32000-1 §7.9.2.2 and §7.9.2.2.1 of +// ISO 32000-2): +// +// FE FF … UTF-16BE, which is how a document says anything outside +// Latin 1 and most of what any modern producer writes +// EF BB BF … UTF-8, which PDF 2.0 added +// anything else PDFDocEncoding +// +// ⛔ Only those two byte orders of mark, and no guessing beyond them. A string +// that merely looks like UTF-16LE is PDFDocEncoding that happens to contain a +// zero byte, and decoding it as UTF-16 turns a readable title into mojibake — +// which is the opposite failure to the one above and just as silent. +func TextString(o Object) (string, bool) { + b, ok := ToString(o) + if !ok { + return "", false + } + switch { + case len(b) >= 2 && b[0] == 0xFE && b[1] == 0xFF: + return fromUTF16BE(b[2:]), true + case len(b) >= 3 && b[0] == 0xEF && b[1] == 0xBB && b[2] == 0xBF: + return fromUTF8(b[3:]), true + } + return fromPDFDoc(b), true +} + +// fromUTF16BE decodes big-endian UTF-16, surrogate pairs included. +func fromUTF16BE(b []byte) string { + units := make([]uint16, 0, len(b)/2) + for i := 0; i+1 < len(b); i += 2 { + units = append(units, uint16(b[i])<<8|uint16(b[i+1])) + } + out := string(utf16.Decode(units)) + if len(b)%2 == 1 { + // An odd number of bytes: the last one is half a code unit, and half a + // code unit is not a character. It is said rather than dropped, so + // that a truncated string looks truncated. + out += string(utf8.RuneError) + } + return out +} + +// fromUTF8 keeps valid UTF-8 and replaces what is not, so that the result is +// always a string Go can print, compare and put in JSON. +func fromUTF8(b []byte) string { + if utf8.Valid(b) { + return string(b) + } + var out strings.Builder + for len(b) > 0 { + r, size := utf8.DecodeRune(b) + out.WriteRune(r) + b = b[size:] + } + return out.String() +} + +// fromPDFDoc decodes PDFDocEncoding, which is Latin 1 with the holes in it +// filled: the spacing accents at 0x18 to 0x1F where Latin 1 has control +// characters, the typographic marks at 0x80 to 0x9E, and the Euro at 0xA0. +func fromPDFDoc(b []byte) string { + var out strings.Builder + out.Grow(len(b)) + for _, c := range b { + out.WriteRune(pdfDocRune(c)) + } + return out.String() +} + +// pdfDocRune is one byte of PDFDocEncoding. +func pdfDocRune(c byte) rune { + switch { + case c >= 0x18 && c <= 0x1F: + return pdfDocAccents[c-0x18] + case c >= 0x80 && c <= 0x9E: + return pdfDocMarks[c-0x80] + case c == 0xA0: + return '€' + case c == 0x7F || c == 0x9F || c == 0xAD: + // Undefined by the standard. A byte that means nothing is said to + // mean nothing rather than quietly becoming a soft hyphen or a + // control character. + return utf8.RuneError + } + // The rest is Latin 1, where the byte IS the code point. + return rune(c) +} + +// pdfDocAccents is 0x18 to 0x1F: breve, caron, circumflex, dot accent, double +// acute, ogonek, ring, tilde. Latin 1 has control characters here, which is +// the first place a Latin-1 decoder goes wrong on a PDF. +var pdfDocAccents = [8]rune{ + '˘', 'ˇ', 'ˆ', '˙', '˝', '˛', '˚', '˜', +} + +// pdfDocMarks is 0x80 to 0x9E, the typographic marks: Annex D.2 of ISO +// 32000-1, checked against it entry by entry rather than remembered. +var pdfDocMarks = [31]rune{ + '•', '†', '‡', '…', '—', '–', 'ƒ', '⁄', + '‹', '›', '−', '‰', '„', '“', '”', '‘', + '’', '‚', '™', 'fi', 'fl', 'Ł', 'Œ', 'Š', + 'Ÿ', 'Ž', 'ı', 'ł', 'œ', 'š', 'ž', +} diff --git a/textstring_test.go b/textstring_test.go new file mode 100644 index 0000000..bde1587 --- /dev/null +++ b/textstring_test.go @@ -0,0 +1,148 @@ +package reader + +import ( + "testing" + "unicode/utf8" +) + +// ⛔ THE DEFECT. `pdfops info` printed a document's creator as +// "\xfe\xffL\x00a\x00T\x00e\x00X" for as long as the verb had existed, +// because ToString hands back BYTES and the bytes of a text string are not +// text. +func TestAUTF16StringIsNotReadAByteAtATime(t *testing.T) { + // How xdvipdfmx actually wrote it, BOM and all. + raw := "\xfe\xff\x00L\x00a\x00T\x00e\x00X\x00 \x00v\x00i\x00a\x00 \x00p\x00a\x00n\x00d\x00o\x00c" + got, ok := TextString(String(raw)) + if !ok { + t.Fatal("a string object was refused") + } + if got != "LaTeX via pandoc" { + t.Errorf("read %q", got) + } + // And the bytes are still available for whoever wants them. + if b, _ := ToString(String(raw)); string(b) != raw { + t.Error("ToString no longer hands back the bytes") + } +} + +func TestAUTF16StringOutsideLatin1(t *testing.T) { + // "Résumé" in UTF-16BE: the accented letters are what a producer reaches + // for UTF-16 to say in the first place. + raw := "\xfe\xff\x00R\x00\xe9\x00s\x00u\x00m\x00\xe9" + if got, _ := TextString(String(raw)); got != "Résumé" { + t.Errorf("read %q, want %q", got, "Résumé") + } +} + +// A character outside the basic plane arrives as a surrogate pair, and a pair +// decoded one unit at a time is two replacement characters. +func TestASurrogatePair(t *testing.T) { + // U+1F4C4 PAGE FACING UP is D83D DCC4. + raw := "\xfe\xff\xd8\x3d\xdc\xc4" + if got, _ := TextString(String(raw)); got != "\U0001F4C4" { + t.Errorf("read %q (%d runes)", got, utf8.RuneCountInString(got)) + } +} + +// Half a code unit is not a character, and a truncated string should look +// truncated rather than lose its last letter silently. +func TestAUTF16StringWithAnOddNumberOfBytes(t *testing.T) { + raw := "\xfe\xff\x00a\x00" + got, _ := TextString(String(raw)) + if want := "a" + string(utf8.RuneError); got != want { + t.Errorf("read %q, want %q", got, want) + } +} + +// A BOM and nothing after it is an empty string, not a failure. +func TestNothingButABOM(t *testing.T) { + if got, ok := TextString(String("\xfe\xff")); !ok || got != "" { + t.Errorf("read %q, %v", got, ok) + } +} + +// PDF 2.0 added UTF-8 with its own mark. +func TestAUTF8StringWithItsMark(t *testing.T) { + if got, _ := TextString(String("\xef\xbb\xbfRésumé")); got != "Résumé" { + t.Errorf("read %q", got) + } + // And bytes that are not UTF-8 after a mark that says they are: each one + // becomes a replacement character rather than making the whole string + // unprintable. + got, _ := TextString(String("\xef\xbb\xbfa\xffb")) + if want := "a" + string(utf8.RuneError) + "b"; got != want { + t.Errorf("read %q, want %q", got, want) + } +} + +// ⛔ PDFDocEncoding is NOT Latin 1, and 0x18 to 0x1F is the first place a +// Latin-1 decoder goes wrong: Latin 1 has control characters there and PDF +// has the spacing accents. +func TestTheAccentsWhereLatin1HasControlCharacters(t *testing.T) { + got, _ := TextString(String("\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f")) + if want := "˘ˇˆ˙˝˛˚˜"; got != want { + t.Errorf("read %q, want %q", got, want) + } +} + +// The typographic marks at 0x80 to 0x9E, where Latin 1 has nothing at all. +func TestTheTypographicMarks(t *testing.T) { + cases := map[byte]rune{ + 0x80: '•', 0x83: '…', 0x84: '—', 0x85: '–', 0x8D: '“', 0x8E: '”', + 0x90: '’', 0x92: '™', 0x93: 'fi', 0x96: 'Œ', 0x9E: 'ž', + } + for b, want := range cases { + got, _ := TextString(String([]byte{b})) + if got != string(want) { + t.Errorf("0x%02X read %q, want %q", b, got, want) + } + } + // 0xA0 is the Euro, not a non-breaking space. + if got, _ := TextString(String("\xa0")); got != "€" { + t.Errorf("0xA0 read %q, want €", got) + } +} + +// Three bytes the standard leaves undefined. A byte that means nothing is said +// to mean nothing, rather than quietly becoming a soft hyphen. +func TestTheBytesTheStandardLeavesUndefined(t *testing.T) { + for _, b := range []byte{0x7F, 0x9F, 0xAD} { + got, _ := TextString(String([]byte{b})) + if got != string(utf8.RuneError) { + t.Errorf("0x%02X read %q, want the replacement character", b, got) + } + } +} + +// The rest of PDFDocEncoding is Latin 1, where the byte is the code point. +func TestTheRestIsLatin1(t *testing.T) { + if got, _ := TextString(String("R\xe9sum\xe9")); got != "Résumé" { + t.Errorf("read %q, want %q", got, "Résumé") + } + if got, _ := TextString(String("plain ASCII")); got != "plain ASCII" { + t.Errorf("read %q", got) + } +} + +// ⛔ A string that merely LOOKS like UTF-16LE is PDFDocEncoding with a zero +// byte in it. Guessing at it turns a readable title into mojibake, which is +// the opposite failure to reading UTF-16BE a byte at a time and just as +// silent. +func TestAStringThatLooksLikeUTF16ButHasNoMark(t *testing.T) { + // The whole string, not "it does not say La": accepting FF FE as a mark + // and then decoding big-endian anyway gives mojibake too, so only the + // exact reading tells the two apart. + got, _ := TextString(String("\xff\xfeL\x00a\x00")) + if want := "ÿþL\x00a\x00"; got != want { + t.Errorf("read %q, want %q — it guessed at a little-endian mark", got, want) + } +} + +// Something that is not a string at all is refused rather than rendered. +func TestWhatIsNotAStringAtAll(t *testing.T) { + for _, o := range []Object{Name("Title"), Integer(4), nil, Dict{}} { + if got, ok := TextString(o); ok { + t.Errorf("%T was read as a text string: %q", o, got) + } + } +}