From 72cd01527eea090d881d254a5d579316657eedbf Mon Sep 17 00:00:00 2001 From: tannevaled Date: Sun, 11 Oct 2026 20:03:14 +0200 Subject: [PATCH] A table is the stretch of page that every row of it leaves empty Tables reports the grids a page's text lines up into: the capability behind BentoPDF's extract-tables and pdf-to-csv, which nothing in the fleet had. A column boundary is a stretch every row leaves empty. The first line of a block proposes its own gaps, each line after it subtracts whatever it draws in them, and a line that leaves nothing closes the block. Subtraction and not the intersection of the lines' gaps, which is where this started: a cell whose text wraps leaves a line with one piece of text on it and so with no gaps of its own, so intersecting ended the table there. A four-row table with one wrapped cell came back as two tables of two rows. Three things are not columns, each found by running this over real documents rather than over a fixture, and measured on one 213-page document as each went in: 588 tables, then 213, then 83, then 40. a bullet a bulleted list IS a two-column grid; what tells them apart is a column that says the same thing wherever it says anything a column nothing was that is a stretch of page nothing was drawn in, drawn in which is what a boundary already is a line far below a page number draws nothing in any boundary, which is exactly what a row of a table does The forty are that document's Code/Intitule pairs. A 3-page prose report in the same run reports none, which is the control. 100% of statements with no uncovered blocks, 31 mutations caught, and the new entry point added to the fuzz target: 7.2M executions, no crash. Said in the README: ruled lines are not read, so this is the text strategy only, and a page in two columns of prose is a grid too. Co-Authored-By: Claude Opus 5 --- README.md | 49 +++++ fuzz_test.go | 5 + tables.go | 425 +++++++++++++++++++++++++++++++++++++ tables_test.go | 565 +++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 1044 insertions(+) create mode 100644 tables.go create mode 100644 tables_test.go diff --git a/README.md b/README.md index ae653c3..21c14b1 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,7 @@ the images it places. text, err := extract.Text(doc, 1) // the page as words, lines put back together runs, err := extract.Runs(doc, 1) // every piece, with its place and its size imgs, err := extract.Images(doc, 1) // every picture, and where it lands +tabs, err := extract.Tables(doc, 1) // the grids its text lines up into ``` Pure Go, no C. It builds for `GOOS=js/wasm` and every architecture the fleet @@ -49,6 +50,54 @@ Turning the other kinds into pixels means reading their colour space, which is [`render`](https://github.com/go-pdfkit/render)'s work rather than this package's. +## Tables + +A page holds no table. It holds text put at places, and a table is what a +reader sees when those places line up — so `Tables` reports an **inference**, +and there is nothing in the file to check it against. + +A column boundary is a stretch of the page that **every row of the table +leaves empty**. The first line of a block proposes its own gaps, each line +after it takes away whatever it draws in them, and a line that leaves nothing +closes the block and begins the next. That is the one decision everything else +follows from, and it is **subtraction** rather than the intersection of the +lines' gaps, which is where this started: + +| | | +|---|---| +| a cell whose text **wraps** | leaves a line with one piece of text on it and so no gaps of its own. Intersecting with nothing ends the table, and a four-row table with one wrapped cell came back as **two tables of two rows**. A wrapped line draws nothing in the boundary beside it, so subtracting keeps it. | + +Three things are not columns, and each one was found by running this over real +documents rather than over a fixture: + +| | | +|---|---| +| a **bullet** | a bulleted list *is* a two-column grid. What tells them apart is that one column says the same thing wherever it says anything — and a column like that distinguishes nothing. A mark of one or two characters, repeated, is dropped; `CNRS` repeated down a column is data and is kept. | +| a column **nothing was drawn in** | that is a stretch of page nothing was drawn in, which is what a boundary already is. A run of nothing but spaces splits a boundary in two and leaves an empty column between the halves. | +| a line **far below** the row above | a page number sits in the left-hand column and draws nothing in any boundary, which is exactly what a row of a table does. What says it is not one is the distance: more than three times the row's own text size. | + +Measured on one 213-page document, as each of those went in: + +``` +588 tables, 1841 rows gaps the rows share +213 tables, 1631 rows + boundaries the rows leave empty (wrapped cells) + 83 tables, 268 rows + a repeated mark is a bullet + 40 tables, 120 rows + an empty column is not a column, and a page + number is not a row +``` + +The forty are the document's `Code` / `Intitulé` pairs, one per job +description, with their second cells wrapped. A 3-page prose report in the same +run reports **none**, which is the control: a detector that cannot say no to a +paragraph says yes to every page there is. + +⛔ **What it still gets wrong.** A page laid out in two columns of prose is a +grid too — the gutter is a stretch every line leaves empty. What saves the +ordinary case is that prose does not stay out of its gutter for long, since one +full-width line closes the block. And ruled lines are not read at all: this is +the *text* strategy, the one a bordered table does not need, and a table whose +columns are held apart by rules rather than by whitespace is not found by it. + ## How it is checked 100% of statements, including every branch that handles a page saying diff --git a/fuzz_test.go b/fuzz_test.go index 5f00bcc..c076bde 100644 --- a/fuzz_test.go +++ b/fuzz_test.go @@ -68,6 +68,11 @@ func FuzzExtract(f *testing.F) { } _, _ = extract.Text(d, i) _, _ = extract.Images(d, i) + // Tables works over the same runs, but what it does with them is + // compare coordinates the document chose: a page is free to say a + // word sits at minus infinity, and a comparison is the one + // operation that will not complain about it. + _, _ = extract.Tables(d, i) } if el := time.Since(start); el > 5*time.Second { t.Fatalf("%d bytes took %s over %d pages", len(b), el, n) diff --git a/tables.go b/tables.go new file mode 100644 index 0000000..b99c7c9 --- /dev/null +++ b/tables.go @@ -0,0 +1,425 @@ +package extract + +import ( + "sort" + "strings" + + "github.com/go-pdfkit/reader" +) + +// A Table is a grid of text read off a page. +// +// A PDF page does not contain a table. It contains text put at places, and a +// table is what a reader sees when those places line up. So a Table here is +// an inference, and [Table.Rows] is what the alignment said rather than what +// the author declared — there is nothing in the file to declare it with. +type Table struct { + // Page is the page the table was found on, counting from one. + Page int + + // Rows holds the cells, top row first and left to right within a row. + // Every row has the same number of cells, and a cell the page left empty + // is an empty string. + Rows [][]string + + // X and Y are the bottom left corner of the TEXT the table is made of, + // and Width and Height its extent, in the page's own coordinates. A table + // drawn inside a frame has a frame a little larger than this; a table + // drawn without one has no frame at all, which is the usual case and the + // reason the box is measured from the words. + // + // It is every word of the block, including a bullet left out of + // [Table.Rows]: the box says where to look on the page, and a reader + // looking there wants to see the bullets too. + X, Y, Width, Height float64 +} + +// TableOptions tunes what counts as a table. A zero field means the default, +// so TableOptions{} is the default everywhere and a caller changes one thing +// by naming one field. +type TableOptions struct { + // ColumnGapSpaces is how wide a gap must be, counted in spaces, before it + // separates two columns rather than two words. The default is 1.5: one + // space is a word break, and a column boundary is wider than a word break + // or there would be no way to tell a table from a sentence. + ColumnGapSpaces float64 + + // MinRows and MinColumns are the smallest grid worth reporting, both 2 by + // default. One row is a line of text and one column is a paragraph. + MinRows, MinColumns int +} + +// The defaults, named so that the test and the documentation cannot drift +// apart from the code. +const ( + defaultColumnGapSpaces = 1.5 + defaultMinRows = 2 + defaultMinColumns = 2 + + // markerRunes is how long a cell can be and still be a bullet rather than + // data. One rune covers the bullet characters, two covers "a." and "1)". + markerRunes = 2 + + // rowSpacingLimit is how far below the row above a line can sit and still + // be the next row, counted in the taller row's own text size. Ordinary + // leading is 1.2 of it and a generous table's is about 2, so three leaves + // room for a table set loosely while still putting the bottom of the page + // out of reach. + rowSpacingLimit = 3 +) + +// Tables is every table on the page, top to bottom, with the default options. +func Tables(d *reader.Document, page int) ([]Table, error) { + return TablesWithOptions(d, page, TableOptions{}) +} + +// TablesWithOptions is [Tables] with the thresholds said out loud. +// +// ⛔ A page laid out in two columns of prose is a grid too: the gutter is a +// stretch every line leaves empty, so a short enough stretch of such a page +// comes back as a two-column table whose cells are lines of text. There is +// nothing in the page that distinguishes the two. What saves the ordinary case +// is that prose does not stay out of the gutter for long — one full-width line +// closes the block — but a caller who knows the document is prose should not +// be asking. This reports alignment; it does not read minds. +func TablesWithOptions(d *reader.Document, page int, o TableOptions) ([]Table, error) { + runs, err := Runs(d, page) + if err != nil { + return nil, err + } + return tablesIn(runs, page, o.withDefaults()), nil +} + +// withDefaults fills in the fields left at zero. +func (o TableOptions) withDefaults() TableOptions { + if o.ColumnGapSpaces <= 0 { + o.ColumnGapSpaces = defaultColumnGapSpaces + } + if o.MinRows <= 0 { + o.MinRows = defaultMinRows + } + if o.MinColumns <= 0 { + o.MinColumns = defaultMinColumns + } + return o +} + +// an interval is a stretch of the x axis. +type interval struct{ start, end float64 } + +// tablesIn finds the tables among a page's runs. +// +// A column boundary is a stretch of the page that EVERY row of the table +// leaves empty. So the first line of a block proposes its own gaps as +// boundaries, and each line after that takes away whatever it draws in them. +// A line that leaves nothing of them closes the block and starts the next one. +// +// ⛔ It is subtraction and not the intersection of the lines' gaps, which is +// what this did first and what cost it the tables it was written for. A cell +// whose text wraps leaves a line with ONE piece of text on it and therefore no +// gaps of its own; intersecting with nothing ends the table, so a four-row +// table with one wrapped cell came back as two tables of two rows. Subtracting +// asks the question that was meant all along — is there anything DRAWN there — +// and a wrapped line sitting inside a column draws nothing in the boundary +// beside it. +func tablesIn(runs []Run, page int, o TableOptions) []Table { + lines := tableLines(runs) + if len(lines) == 0 { + return nil + } + gap := minGap(runs, o.ColumnGapSpaces) + var out []Table + for start := 0; start < len(lines); { + seps := gapsIn(lines[start], gap) + end := start + 1 + for ; end < len(lines); end++ { + if tooFarBelow(lines[end-1], lines[end]) { + break + } + left := clip(seps, lines[end], gap) + if len(left) == 0 { + break + } + seps = left + } + if t, ok := tableOf(lines[start:end], seps, page, o); ok { + out = append(out, t) + } + start = end + } + return out +} + +// tableLines is the page's lines with the blank ones left out. A line of +// nothing but spaces draws nothing anywhere, so keeping it would add a row of +// empty cells to every table that has whitespace drawn with text in it. +func tableLines(runs []Run) [][]Run { + var out [][]Run + for _, line := range groupIntoLines(runs) { + if strings.TrimSpace(joinLine(line)) == "" { + continue + } + out = append(out, line) + } + return out +} + +// minGap is how many points wide a gap has to be here. It is measured from +// the page's own text — the middle space width among its runs — because a gap +// of four points separates columns in a footnote and falls inside a word in a +// title. +// +// The middle one, and not the narrowest or the widest, because both of those +// let one line decide for the page and they fail in opposite directions: the +// narrowest turns every word gap in the body text into a column, the widest +// turns every column into a word gap. With an even number of runs it is the +// upper of the two middles, which is a tie-break and not a judgement. +func minGap(runs []Run, spaces float64) float64 { + widths := make([]float64, 0, len(runs)) + for _, r := range runs { + w := r.Space + if w <= 0 { + // A font that says nothing about its space: a quarter of an em is + // what [wantsSpace] assumes for the same reason. + w = r.Size * 0.25 + } + if w > 0 { + widths = append(widths, w) + } + } + if len(widths) == 0 { + // Text set at no size at all. Any positive number will do, since + // nothing on this page has a width to compare it with; what matters is + // that it is NOT zero, or every pair of touching runs would be two + // columns. + return spaces + } + sort.Float64s(widths) + return widths[len(widths)/2] * spaces +} + +// gapsIn is the stretches of a line wide enough to be column boundaries. +// +// The right edge is carried forward as a maximum rather than read off the +// previous run, because runs are ordered by where they START: a wide run +// followed by a narrow one inside it would otherwise look like a gap. +func gapsIn(line []Run, gap float64) []interval { + var out []interval + right := line[0].X + line[0].Width + for _, r := range line[1:] { + if r.X-right >= gap { + out = append(out, interval{right, r.X}) + } + if end := r.X + r.Width; end > right { + right = end + } + } + return out +} + +// tooFarBelow reports that a line is too far down the page to be the next row +// of the block above it. +// +// ⛔ Measured: without this, the page number at the foot of the page became +// the last row of a bullet list two thirds of the way up it. Nothing else +// stopped it — a page number sits in the left-hand column and draws nothing in +// any boundary, which is exactly what a row of a table does — and because its +// cell was neither a bullet nor empty it was also what kept that list from +// being recognised as one. +func tooFarBelow(prev, next []Run) bool { + size := prev[0].Size + for _, r := range prev { + if r.Size > size { + size = r.Size + } + } + if size <= 0 { + // Text set at no size says nothing about how far apart its lines + // should be, so this has no opinion either. + return false + } + return prev[0].Y-next[0].Y > rowSpacingLimit*size +} + +// clip takes out of each boundary whatever the line draws in it, and drops +// what is left over of a boundary too narrow to be one. A boundary a run +// crosses in the middle comes back as the two pieces either side of it, so a +// stretch of white can survive as a narrower boundary rather than only +// surviving whole. +func clip(seps []interval, line []Run, gap float64) []interval { + var out []interval + for _, s := range seps { + pieces := []interval{s} + for _, r := range line { + pieces = without(pieces, interval{r.X, r.X + r.Width}) + } + for _, p := range pieces { + if p.end-p.start >= gap { + out = append(out, p) + } + } + } + return out +} + +// without is the pieces with the stretch taken out of them. +// +// The two comparisons below are deliberately strict and no test can tell them +// from the loose ones: a loose one manufactures a piece of no width, and a +// piece of no width cannot get past the width check in [clip]. They are strict +// so that nothing downstream has to tolerate one, not because the behaviour +// differs — which is worth saying, because a mutation that swaps them looks +// like an uncovered branch and is not one. +func without(pieces []interval, drawn interval) []interval { + var out []interval + for _, p := range pieces { + if drawn.end <= p.start || drawn.start >= p.end { + out = append(out, p) + continue + } + if drawn.start > p.start { + out = append(out, interval{p.start, drawn.start}) + } + if drawn.end < p.end { + out = append(out, interval{drawn.end, p.end}) + } + } + return out +} + +// tableOf turns a block of lines and the boundaries they share into a table, +// or reports that the block is not one. +func tableOf(block [][]Run, seps []interval, page int, o TableOptions) (Table, bool) { + if len(block) < o.MinRows { + return Table{}, false + } + rows := make([][]string, len(block)) + for i, line := range block { + rows[i] = cellsOf(line, seps) + } + rows = dropEmptyColumns(rows) + if len(rows[0]) > 1 && markerColumn(rows) { + for i := range rows { + rows[i] = rows[i][1:] + } + } + if len(rows[0]) < o.MinColumns { + return Table{}, false + } + t := Table{Page: page, Rows: rows} + t.X, t.Y, t.Width, t.Height = boxOf(block) + return t, true +} + +// cellsOf puts a line's pieces of text into the columns the boundaries make. +func cellsOf(line []Run, seps []interval) []string { + in := make([][]Run, len(seps)+1) + for _, r := range line { + c := columnOf(seps, r) + in[c] = append(in[c], r) + } + row := make([]string, len(in)) + for c, runs := range in { + row[c] = joinLine(runs) + } + return row +} + +// dropEmptyColumns removes the columns that no row said anything in. +// +// ⛔ Measured: a 213-page document gave blocks of six and fifteen rows whose +// second and third columns were empty the whole way down. A column nothing was +// drawn in is not a column — it is a stretch of page nothing was drawn in, +// which is what a boundary already is — and counting it is how one column of +// left-aligned paragraphs passes for a table. +func dropEmptyColumns(rows [][]string) [][]string { + keep := make([]int, 0, len(rows[0])) + for c := range rows[0] { + for _, row := range rows { + if row[c] != "" { + keep = append(keep, c) + break + } + } + } + if len(keep) == len(rows[0]) { + return rows + } + out := make([][]string, len(rows)) + for i, row := range rows { + cells := make([]string, len(keep)) + for j, c := range keep { + cells[j] = row[c] + } + out[i] = cells + } + return out +} + +// markerColumn reports whether the first column is a list's bullet rather than +// data: one short mark, the same one wherever it appears, and nothing else. +// +// ⛔ Measured, on a 213-page document of the author's: 588 "tables" of two +// columns, and the first column of nearly all of them held "•" or U+F0A7. A +// bulleted list IS a two-column grid, and no amount of looking at the +// alignment will say otherwise — the thing that distinguishes it is that one +// of its columns says the same thing wherever it says anything, and a column +// like that distinguishes nothing. Dropping it leaves a list with one column, +// which is not a table, and leaves a real table with a repeated mark in it one +// column shorter and no worse informed. +// +// ⛔ The empty cells are why this is not simply "every row says the same": a +// bullet's text wraps, and the lines it wraps onto have no bullet of their +// own. Insisting on every row let the wrapped lists back in — which is how +// this read as a fix while a document of nothing but bullet lists still came +// back as fourteen rows of table. +func markerColumn(rows [][]string) bool { + mark := "" + for _, row := range rows { + switch { + case row[0] == "": + continue + case mark == "": + mark = row[0] + if len([]rune(mark)) > markerRunes { + return false + } + case row[0] != mark: + return false + } + } + // mark is never still empty here: [dropEmptyColumns] has already taken + // away the columns nothing was drawn in, so the first column has something + // in it somewhere. + return true +} + +// boxOf is the box the block's text occupies. The block has at least one line +// and a line has at least one run — [groupIntoLines] makes lines out of runs — +// so there is always a first run to start from. +func boxOf(block [][]Run) (x, y, w, h float64) { + first := block[0][0] + left, bottom := first.X, first.Y + right, top := first.X+first.Width, first.Y+first.Size + for _, line := range block { + for _, r := range line { + left, bottom = min(left, r.X), min(bottom, r.Y) + right, top = max(right, r.X+r.Width), max(top, r.Y+r.Size) + } + } + return left, bottom, right - left, top - bottom +} + +// columnOf is which column a run sits in: the number of boundaries that are +// wholly to its left. A run cannot straddle a boundary — a boundary is a +// stretch where no line in the block draws anything — so this is exact rather +// than nearest. +func columnOf(seps []interval, r Run) int { + n := 0 + for _, s := range seps { + if s.end <= r.X { + n++ + } + } + return n +} diff --git a/tables_test.go b/tables_test.go new file mode 100644 index 0000000..5e40813 --- /dev/null +++ b/tables_test.go @@ -0,0 +1,565 @@ +package extract + +import ( + "reflect" + "strings" + "testing" + + "github.com/go-pdfkit/reader" +) + +// The test font gives every character half an em, so at ten points a letter is +// five points wide and a space is five points too. The default boundary is one +// and a half spaces, so 7.5 points: the fixtures below put columns forty +// points apart and words five. + +func TestATableIsTheGapEveryRowShares(t *testing.T) { + d := pageWith(t, strings.Join([]string{ + "BT /F1 10 Tf 20 150 Td (Name) Tj 40 0 Td (Age) Tj ET", + "BT /F1 10 Tf 20 130 Td (Ada) Tj 40 0 Td (36) Tj ET", + }, "\n"), nil) + tables, err := Tables(d, 1) + if err != nil { + t.Fatal(err) + } + if len(tables) != 1 { + t.Fatalf("%d tables, want 1: %+v", len(tables), tables) + } + want := [][]string{{"Name", "Age"}, {"Ada", "36"}} + if !reflect.DeepEqual(tables[0].Rows, want) { + t.Errorf("the table reads %q, want %q", tables[0].Rows, want) + } + if tables[0].Page != 1 { + t.Errorf("the table says it is on page %d", tables[0].Page) + } +} + +// ⛔ THE CONTROL. Prose is text at places too, and a detector that cannot say +// no to a paragraph says yes to every page in the corpus. The word gaps here +// are one space wide, and one space is a word break rather than a column. +func TestAParagraphIsNotATable(t *testing.T) { + d := pageWith(t, strings.Join([]string{ + "BT /F1 10 Tf 20 150 Td (one) Tj 20 0 Td (two) Tj 20 0 Td (three) Tj ET", + "BT /F1 10 Tf 20 130 Td (four) Tj 25 0 Td (five) Tj 25 0 Td (six) Tj ET", + }, "\n"), nil) + tables, err := Tables(d, 1) + if err != nil { + t.Fatal(err) + } + if len(tables) != 0 { + t.Errorf("a paragraph came back as %d table(s): %+v", len(tables), tables) + } +} + +func TestAPageWithNothingWrittenOnItHasNoTables(t *testing.T) { + d := pageWith(t, "", nil) + tables, err := Tables(d, 1) + if err != nil { + t.Fatal(err) + } + if tables != nil { + t.Errorf("an empty page came back as %+v", tables) + } +} + +func TestATableOnAPageThatIsNotThere(t *testing.T) { + d := pageWith(t, "BT /F1 10 Tf 20 150 Td (x) Tj ET", nil) + if _, err := Tables(d, 9); err == nil { + t.Error("page nine of a one-page document was read") + } +} + +// A line with nothing but spaces on it has no gaps of its own, so taking it +// seriously would cut the table in two at the whitespace its author drew. +func TestALineOfSpacesDoesNotCutATableInHalf(t *testing.T) { + d := pageWith(t, strings.Join([]string{ + "BT /F1 10 Tf 20 150 Td (Name) Tj 40 0 Td (Age) Tj ET", + "BT /F1 10 Tf 20 140 Td ( ) Tj ET", + "BT /F1 10 Tf 20 130 Td (Ada) Tj 40 0 Td (36) Tj ET", + }, "\n"), nil) + tables, _ := Tables(d, 1) + if len(tables) != 1 { + t.Fatalf("%d tables, want 1: %+v", len(tables), tables) + } + want := [][]string{{"Name", "Age"}, {"Ada", "36"}} + if !reflect.DeepEqual(tables[0].Rows, want) { + t.Errorf("the table reads %q, want %q", tables[0].Rows, want) + } +} + +// A line that shares no gap closes the block it is in and starts the next one, +// which is how a page holding two tables with a sentence between them gives +// two tables and not one. +func TestTwoTablesWithASentenceBetweenThem(t *testing.T) { + d := pageWith(t, strings.Join([]string{ + "BT /F1 10 Tf 20 180 Td (Name) Tj 40 0 Td (Age) Tj ET", + "BT /F1 10 Tf 20 165 Td (Ada) Tj 40 0 Td (36) Tj ET", + "BT /F1 10 Tf 20 150 Td (and then the second one) Tj ET", + "BT /F1 10 Tf 20 135 Td (City) Tj 40 0 Td (Code) Tj ET", + "BT /F1 10 Tf 20 120 Td (Caen) Tj 40 0 Td (14000) Tj ET", + }, "\n"), nil) + tables, _ := Tables(d, 1) + if len(tables) != 2 { + t.Fatalf("%d tables, want 2: %+v", len(tables), tables) + } + if got := tables[0].Rows[0][0]; got != "Name" { + t.Errorf("the first table starts with %q", got) + } + if got := tables[1].Rows[0][0]; got != "City" { + t.Errorf("the second table starts with %q", got) + } +} + +// ⛔ The known failure, written down rather than hidden: a cell wide enough to +// reach across a boundary takes that boundary away from EVERY row, because a +// boundary is what all of them share. Two columns become one, and one column +// is not a table. +func TestACellThatReachesAcrossABoundaryTakesItAway(t *testing.T) { + d := pageWith(t, strings.Join([]string{ + "BT /F1 10 Tf 20 150 Td (Name) Tj 40 0 Td (Age) Tj ET", + "BT /F1 10 Tf 20 130 Td (a very long note indeed) Tj ET", + "BT /F1 10 Tf 20 110 Td (Ada) Tj 40 0 Td (36) Tj ET", + }, "\n"), nil) + tables, _ := Tables(d, 1) + if len(tables) != 0 { + t.Errorf("want none, got %d: %+v", len(tables), tables) + } +} + +// A row that says nothing in the middle column says it with an empty string, +// so that every row of a table has the same number of cells. +func TestACellTheRowLeftEmpty(t *testing.T) { + d := pageWith(t, strings.Join([]string{ + "BT /F1 10 Tf 20 150 Td (Name) Tj 40 0 Td (Age) Tj 40 0 Td (City) Tj ET", + "BT /F1 10 Tf 20 130 Td (Ada) Tj 80 0 Td (Caen) Tj ET", + }, "\n"), nil) + tables, _ := Tables(d, 1) + if len(tables) != 1 { + t.Fatalf("%d tables, want 1: %+v", len(tables), tables) + } + want := [][]string{{"Name", "Age", "City"}, {"Ada", "", "Caen"}} + if !reflect.DeepEqual(tables[0].Rows, want) { + t.Errorf("the table reads %q, want %q", tables[0].Rows, want) + } +} + +// Several pieces of text in one cell are joined the way a line is, which is +// what puts the space back between two words a generator drew separately. +func TestTwoPiecesOfTextInOneCell(t *testing.T) { + d := pageWith(t, strings.Join([]string{ + "BT /F1 10 Tf 20 150 Td (Ada) Tj 20 0 Td (King) Tj 40 0 Td (36) Tj ET", + "BT /F1 10 Tf 20 130 Td (Bob) Tj 60 0 Td (41) Tj ET", + }, "\n"), nil) + tables, _ := Tables(d, 1) + if len(tables) != 1 { + t.Fatalf("%d tables, want 1: %+v", len(tables), tables) + } + if got := tables[0].Rows[0][0]; got != "Ada King" { + t.Errorf("the first cell reads %q, want %q", got, "Ada King") + } +} + +func TestTheBoxIsWhereTheWordsAre(t *testing.T) { + d := pageWith(t, strings.Join([]string{ + "BT /F1 10 Tf 20 150 Td (Name) Tj 40 0 Td (Age) Tj ET", + "BT /F1 10 Tf 20 130 Td (Ada) Tj 40 0 Td (36) Tj ET", + }, "\n"), nil) + tables, _ := Tables(d, 1) + if len(tables) != 1 { + t.Fatalf("%d tables: %+v", len(tables), tables) + } + // Leftmost word at 20, rightmost ends at 60+15; lowest baseline 130, and + // the top line is ten points tall above 150. + got := tables[0] + if got.X != 20 || got.Y != 130 || got.Width != 55 || got.Height != 30 { + t.Errorf("the box is (%v,%v) %v by %v, want (20,130) 55 by 30", + got.X, got.Y, got.Width, got.Height) + } +} + +// ⛔ Runs are ordered by where they START, so the right-hand edge of what has +// been drawn so far is a running maximum. Read it off the previous run instead +// and a wide cell with a short one inside it opens a gap that is not there. +func TestAShortRunInsideAWideOne(t *testing.T) { + d := pageWith(t, strings.Join([]string{ + // Ten letters from 20 reach 70; the (x) at 30 sits inside them. + "BT /F1 10 Tf 20 150 Td (AAAAAAAAAA) Tj ET", + "BT /F1 10 Tf 30 150 Td (x) Tj ET", + "BT /F1 10 Tf 100 150 Td (right) Tj ET", + // A row whose own gaps are [35,50] and [65,100]. + "BT /F1 10 Tf 20 130 Td (Ada) Tj 30 0 Td (bbb) Tj 50 0 Td (ccc) Tj ET", + }, "\n"), nil) + tables, _ := Tables(d, 1) + if len(tables) != 1 { + t.Fatalf("%d tables, want 1: %+v", len(tables), tables) + } + // The only boundary both rows share is [70,100]: two columns, not three. + if n := len(tables[0].Rows[0]); n != 2 { + t.Errorf("%d columns, want 2: %q", n, tables[0].Rows) + } +} + +func TestTheSmallestGridWorthReporting(t *testing.T) { + content := strings.Join([]string{ + "BT /F1 10 Tf 20 150 Td (Name) Tj 40 0 Td (Age) Tj ET", + "BT /F1 10 Tf 20 130 Td (Ada) Tj 40 0 Td (36) Tj ET", + }, "\n") + d := pageWith(t, content, nil) + if tables, _ := TablesWithOptions(d, 1, TableOptions{MinRows: 3}); len(tables) != 0 { + t.Errorf("two rows passed a three-row minimum: %+v", tables) + } + if tables, _ := TablesWithOptions(d, 1, TableOptions{MinColumns: 3}); len(tables) != 0 { + t.Errorf("two columns passed a three-column minimum: %+v", tables) + } + if tables, _ := TablesWithOptions(d, 1, TableOptions{MinRows: 2, MinColumns: 2}); len(tables) != 1 { + t.Errorf("the grid it is exactly big enough for was refused: %+v", tables) + } +} + +// ⛔ A bulleted list IS a two-column grid, and 588 of them in one 213-page +// document was what this detector first reported. What tells them apart is +// that one column says the same thing on every row — and a column that says +// the same thing on every row distinguishes nothing. +func TestABulletedListIsNotATable(t *testing.T) { + d := pageWith(t, strings.Join([]string{ + "BT /F1 10 Tf 20 150 Td (\225) Tj 40 0 Td (conduct the project) Tj ET", + "BT /F1 10 Tf 20 130 Td (\225) Tj 40 0 Td (inform the trustees) Tj ET", + "BT /F1 10 Tf 20 110 Td (\225) Tj 40 0 Td (federate the staff) Tj ET", + }, "\n"), nil) + tables, _ := Tables(d, 1) + if len(tables) != 0 { + t.Errorf("a bulleted list came back as %d table(s): %+v", len(tables), tables) + } +} + +// The mark has to be a MARK. A first column that repeats a word is a column +// that repeats a word, and throwing it away would throw away the data. +func TestAFirstColumnThatRepeatsAWordIsStillAColumn(t *testing.T) { + d := pageWith(t, strings.Join([]string{ + "BT /F1 10 Tf 20 150 Td (CNRS) Tj 40 0 Td (one) Tj ET", + "BT /F1 10 Tf 20 130 Td (CNRS) Tj 40 0 Td (two) Tj ET", + }, "\n"), nil) + tables, _ := Tables(d, 1) + if len(tables) != 1 { + t.Fatalf("%d tables, want 1: %+v", len(tables), tables) + } + want := [][]string{{"CNRS", "one"}, {"CNRS", "two"}} + if !reflect.DeepEqual(tables[0].Rows, want) { + t.Errorf("the table reads %q, want %q", tables[0].Rows, want) + } +} + +// ⛔ THE DEFECT THAT CHANGED THE ALGORITHM. A cell whose text wraps leaves a +// line with one piece of text on it and so with no gaps of its own. Asking +// what gaps two lines SHARE ends the table there, and a four-row table with +// one wrapped cell came back as two tables of two rows — measured on a real +// document, not on a fixture. Asking instead what either line DRAWS in the +// boundary keeps it: a wrapped line sits inside a column and draws nothing in +// the boundary beside it. +func TestAWrappedCellDoesNotSplitATableInTwo(t *testing.T) { + d := pageWith(t, strings.Join([]string{ + "BT /F1 10 Tf 20 180 Td (Activity) Tj 90 0 Td (Outcome) Tj ET", + "BT /F1 10 Tf 20 160 Td (design) Tj 90 0 Td (ten petabytes) Tj ET", + // The wrapped remainder of the cell above: one run, no gaps of its + // own, and nothing drawn in the boundary at 85. + "BT /F1 10 Tf 110 145 Td (of flash) Tj ET", + "BT /F1 10 Tf 20 125 Td (operate) Tj 90 0 Td (a mesocentre) Tj ET", + }, "\n"), nil) + tables, _ := Tables(d, 1) + if len(tables) != 1 { + t.Fatalf("%d tables, want 1: %+v", len(tables), tables) + } + want := [][]string{ + {"Activity", "Outcome"}, + {"design", "ten petabytes"}, + {"", "of flash"}, + {"operate", "a mesocentre"}, + } + if !reflect.DeepEqual(tables[0].Rows, want) { + t.Errorf("the table reads %q, want %q", tables[0].Rows, want) + } +} + +// A boundary a run lands inside survives as the pieces either side of it, so +// that run gets a column of its own and the table goes on. Dropping the +// boundary instead would end the table at the first value anybody centred. +func TestARunInsideABoundaryMakesAColumnOfItsOwn(t *testing.T) { + d := pageWith(t, strings.Join([]string{ + // A boundary from 60 to 140. + "BT /F1 10 Tf 20 150 Td (Name) Tj 120 0 Td (Age) Tj ET", + // A run from 70 to 100 leaves 60 to 70 and 100 to 140 of it. + "BT /F1 10 Tf 20 130 Td (Ada) Tj 50 0 Td (xxxxxx) Tj ET", + }, "\n"), nil) + tables, _ := Tables(d, 1) + if len(tables) != 1 { + t.Fatalf("%d tables, want 1: %+v", len(tables), tables) + } + want := [][]string{{"Name", "", "Age"}, {"Ada", "xxxxxx", ""}} + if !reflect.DeepEqual(tables[0].Rows, want) { + t.Errorf("the table reads %q, want %q", tables[0].Rows, want) + } +} + +// ⛔ A page number sits in the left-hand column and draws nothing in any +// boundary, which is precisely what a row of a table does. What says it is not +// one is that it is a long way below the last row. +func TestAPageNumberIsNotTheLastRow(t *testing.T) { + d := pageWith(t, strings.Join([]string{ + "BT /F1 10 Tf 20 150 Td (Name) Tj 40 0 Td (Age) Tj ET", + "BT /F1 10 Tf 20 130 Td (Ada) Tj 40 0 Td (36) Tj ET", + "BT /F1 10 Tf 20 40 Td (7) Tj ET", + }, "\n"), nil) + tables, _ := Tables(d, 1) + if len(tables) != 1 { + t.Fatalf("%d tables, want 1: %+v", len(tables), tables) + } + if n := len(tables[0].Rows); n != 2 { + t.Errorf("%d rows, want 2: %q", n, tables[0].Rows) + } +} + +// ⛔ A column nothing was drawn in is not a column — it is a stretch of page +// nothing was drawn in, which is what a boundary already is. +// +// This is how one turns up: a run of nothing but spaces sits in the middle of +// a boundary and splits it in two, and the column it makes between the halves +// reads empty because that is what it says. +func TestAColumnNothingWasDrawnInIsNotAColumn(t *testing.T) { + d := pageWith(t, strings.Join([]string{ + "BT /F1 10 Tf 20 150 Td (Name) Tj 80 0 Td (Age) Tj ET", + "BT /F1 10 Tf 20 130 Td (Ada) Tj 40 0 Td ( ) Tj 40 0 Td (36) Tj ET", + }, "\n"), nil) + tables, _ := Tables(d, 1) + if len(tables) != 1 { + t.Fatalf("%d tables, want 1: %+v", len(tables), tables) + } + want := [][]string{{"Name", "Age"}, {"Ada", "36"}} + if !reflect.DeepEqual(tables[0].Rows, want) { + t.Errorf("the table reads %q, want %q", tables[0].Rows, want) + } +} + +// A bullet list whose text wraps: the lines it wraps onto carry no bullet, so +// the mark is in some rows of its column and not others. It is still a list. +func TestABulletedListWhoseTextWraps(t *testing.T) { + d := pageWith(t, strings.Join([]string{ + "BT /F1 10 Tf 20 150 Td (\225) Tj 40 0 Td (conduct the project) Tj ET", + "BT /F1 10 Tf 60 135 Td (and animate it) Tj ET", + "BT /F1 10 Tf 20 120 Td (\225) Tj 40 0 Td (inform the trustees) Tj ET", + "BT /F1 10 Tf 60 105 Td (of what changes) Tj ET", + }, "\n"), nil) + tables, _ := Tables(d, 1) + if len(tables) != 0 { + t.Errorf("a wrapped bulleted list came back as %d table(s): %+v", len(tables), tables) + } +} + +// The row above is measured by its TALLEST piece of text, not its first: a row +// whose first cell is small and whose second is a heading is as far from the +// next row as that heading is tall. +func TestHowFarIsMeasuredByTheTallestPieceOfTheRow(t *testing.T) { + d := pageWith(t, strings.Join([]string{ + "BT /F1 6 Tf 20 150 Td (no) Tj ET BT /F1 20 Tf 60 150 Td (Heading) Tj ET", + // Fifty points below: more than three of the small size, less than + // three of the large one. + "BT /F1 10 Tf 20 100 Td (Ada) Tj 40 0 Td (36) Tj ET", + }, "\n"), nil) + tables, _ := Tables(d, 1) + if len(tables) != 1 { + t.Errorf("the heading's own height did not reach the row below: %+v", tables) + } +} + +// ⛔ A caller who asks for one-row tables is the only one who can see the +// difference between a gap and a boundary on a SINGLE line, because a block of +// two lines or more has a boundary by construction: it grew only for as long +// as the lines shared one. Both the width a gap needs and the two columns a +// table needs are invisible until somebody lowers MinRows. +func TestOneRowIsNotATableEvenWhenTheCallerAllowsOne(t *testing.T) { + d := pageWith(t, strings.Join([]string{ + "BT /F1 10 Tf 20 150 Td (one) Tj 20 0 Td (two) Tj 20 0 Td (three) Tj ET", + "BT /F1 10 Tf 20 130 Td (four) Tj 25 0 Td (five) Tj 25 0 Td (six) Tj ET", + }, "\n"), nil) + tables, _ := TablesWithOptions(d, 1, TableOptions{MinRows: 1}) + if len(tables) != 0 { + t.Errorf("a line of prose came back as %d table(s): %+v", len(tables), tables) + } +} + +// Two rows whose gaps overlap by less than a boundary's width do not share a +// boundary: what is shared is the OVERLAP, and five points of it is a word +// break however wide each row's own gap was. +func TestTwoRowsWhoseGapsBarelyOverlap(t *testing.T) { + d := pageWith(t, strings.Join([]string{ + // A gap from 30 to 45, and ... + "BT /F1 10 Tf 20 150 Td (AA) Tj 25 0 Td (BB) Tj ET", + // ... one from 40 to 55: five points in common. + "BT /F1 10 Tf 20 130 Td (AAAA) Tj 35 0 Td (D) Tj ET", + }, "\n"), nil) + tables, _ := Tables(d, 1) + if len(tables) != 0 { + t.Errorf("a five-point overlap was a column: %+v", tables) + } +} + +// The width of a space is the MIDDLE one on the page, which means a line in +// an unusual size cannot decide what a column boundary is for everything else. +// Both ends of that matter and they fail in opposite directions: the narrowest +// makes every word gap in the body text into a column, the widest makes every +// real column into a word gap. +func TestOneLineInAnUnusualSizeDoesNotDecideForThePage(t *testing.T) { + fine := pageWith(t, strings.Join([]string{ + "BT /F1 10 Tf 20 150 Td (one) Tj 20 0 Td (two) Tj ET", + "BT /F1 10 Tf 20 130 Td (six) Tj 20 0 Td (ten) Tj ET", + "BT /F1 2 Tf 20 60 Td (tiny) Tj ET", + }, "\n"), nil) + if tables, _ := Tables(fine, 1); len(tables) != 0 { + t.Errorf("two-point type made the word gaps above into columns: %+v", tables) + } + large := pageWith(t, strings.Join([]string{ + "BT /F1 10 Tf 20 150 Td (Name) Tj 40 0 Td (Age) Tj ET", + "BT /F1 10 Tf 20 130 Td (Ada) Tj 40 0 Td (36) Tj ET", + "BT /F1 40 Tf 20 60 Td (X) Tj ET", + }, "\n"), nil) + if tables, _ := Tables(large, 1); len(tables) != 1 { + t.Errorf("forty-point type made a twenty-point column into a word gap: %+v", tables) + } +} + +// The boundary a sizeless page gets is not zero. Two runs that touch have a +// gap of nothing between them, and a boundary of nothing would make every one +// of them into a column. +func TestTwoSizelessRunsThatTouchAreNotTwoColumns(t *testing.T) { + d := type3Page(t, 0, strings.Join([]string{ + "BT /F3 0 Tf 20 150 Td (a) Tj (b) Tj ET", + "BT /F3 0 Tf 20 130 Td (a) Tj (b) Tj ET", + }, "\n")) + tables, _ := Tables(d, 1) + if len(tables) != 0 { + t.Errorf("runs that touch came back as %d table(s): %+v", len(tables), tables) + } +} + +// A block's boundaries narrow as it grows: they are what EVERY line in it +// shares, so the third row takes away what the first two agreed on. +func TestTheThirdRowTakesABoundaryAway(t *testing.T) { + d := pageWith(t, strings.Join([]string{ + "BT /F1 10 Tf 20 150 Td (A) Tj 40 0 Td (B) Tj 40 0 Td (C) Tj ET", + "BT /F1 10 Tf 20 130 Td (D) Tj 40 0 Td (E) Tj 40 0 Td (F) Tj ET", + // Sixty points of G reach from 60 to 120, across the second boundary. + "BT /F1 10 Tf 20 110 Td (G) Tj 40 0 Td (HHHHHHHHHHHH) Tj ET", + }, "\n"), nil) + tables, _ := Tables(d, 1) + if len(tables) != 1 { + t.Fatalf("%d tables, want 1: %+v", len(tables), tables) + } + if n := len(tables[0].Rows[0]); n != 2 { + t.Errorf("%d columns, want 2: %q", n, tables[0].Rows) + } +} + +// The line that closed a block is the first line of the next one. Skip it and +// the second of two tables that touch loses its header row — and with it the +// two rows that make it a table at all. +func TestTwoTablesWithNothingBetweenThem(t *testing.T) { + d := pageWith(t, strings.Join([]string{ + "BT /F1 10 Tf 20 180 Td (A) Tj 40 0 Td (B) Tj ET", + "BT /F1 10 Tf 20 160 Td (E) Tj 40 0 Td (F) Tj ET", + "BT /F1 10 Tf 20 140 Td (CCCCCCCCCC) Tj 80 0 Td (D) Tj ET", + "BT /F1 10 Tf 20 120 Td (GGGGGGGGGG) Tj 80 0 Td (H) Tj ET", + }, "\n"), nil) + tables, _ := Tables(d, 1) + if len(tables) != 2 { + t.Fatalf("%d tables, want 2: %+v", len(tables), tables) + } + if got := tables[1].Rows[0][0]; got != "CCCCCCCCCC" { + t.Errorf("the second table starts with %q", got) + } +} + +// The boundary is counted in spaces, not in points, so the same number works +// for a footnote and for a title. +func TestHowWideAGapHasToBe(t *testing.T) { + content := strings.Join([]string{ + "BT /F1 10 Tf 20 150 Td (one) Tj 20 0 Td (two) Tj ET", + "BT /F1 10 Tf 20 130 Td (six) Tj 20 0 Td (ten) Tj ET", + }, "\n") + d := pageWith(t, content, nil) + // The gap is one space. At the default of one and a half, it is a word + // break; asked for less than one, it is a column. + if tables, _ := TablesWithOptions(d, 1, TableOptions{}); len(tables) != 0 { + t.Errorf("a one-space gap was a column at the default: %+v", tables) + } + tables, _ := TablesWithOptions(d, 1, TableOptions{ColumnGapSpaces: 0.5}) + if len(tables) != 1 { + t.Fatalf("%d tables at half a space: %+v", len(tables), tables) + } + want := [][]string{{"one", "two"}, {"six", "ten"}} + if !reflect.DeepEqual(tables[0].Rows, want) { + t.Errorf("the table reads %q, want %q", tables[0].Rows, want) + } +} + +// A font that gives no width to the space it does not draw: the gap has to be +// measured from the size instead, as [wantsSpace] does for the same reason. +func TestAFontThatSaysNothingAboutItsSpace(t *testing.T) { + d := type3Page(t, 10, strings.Join([]string{ + "BT /F3 10 Tf 20 150 Td (a) Tj 10 0 Td (b) Tj ET", + "BT /F3 10 Tf 20 130 Td (b) Tj 10 0 Td (a) Tj ET", + }, "\n")) + // A quarter of ten points is 2.5, so the boundary is 3.75 and the ten + // point gap is well over it. + tables, _ := Tables(d, 1) + if len(tables) != 1 { + t.Fatalf("%d tables, want 1: %+v", len(tables), tables) + } + if want := [][]string{{"a", "b"}, {"b", "a"}}; !reflect.DeepEqual(tables[0].Rows, want) { + t.Errorf("the table reads %q, want %q", tables[0].Rows, want) + } +} + +// Text set at no size at all: nothing on the page has a width to compare a +// gap with. What matters is that the boundary is not zero, or every pair of +// touching runs would be two columns. +func TestTextSetAtNoSizeAtAll(t *testing.T) { + // Three pieces a point apart and then nineteen: at a boundary of nothing + // the first two are two columns, at a boundary of anything they are one. + d := type3Page(t, 0, strings.Join([]string{ + "BT /F3 0 Tf 20 150 Td (a) Tj 1 0 Td (b) Tj 19 0 Td (c) Tj ET", + "BT /F3 0 Tf 20 130 Td (a) Tj 1 0 Td (b) Tj 19 0 Td (c) Tj ET", + }, "\n")) + runs, err := Runs(d, 1) + if err != nil { + t.Fatal(err) + } + for _, r := range runs { + if r.Size != 0 || r.Space != 0 { + t.Fatalf("the fixture is not sizeless: %+v", r) + } + } + tables, _ := Tables(d, 1) + if len(tables) != 1 { + t.Fatalf("%d tables, want 1: %+v", len(tables), tables) + } + if want := [][]string{{"a b", "c"}, {"a b", "c"}}; !reflect.DeepEqual(tables[0].Rows, want) { + t.Errorf("the table reads %q, want %q", tables[0].Rows, want) + } +} + +// type3Page builds a page set in a Type 3 font with no character procedures, +// which is a font that gives no width to anything it is asked to draw. size is +// only here to be named in the test that reads it back. +func type3Page(t *testing.T, size int, content string) *reader.Document { + t.Helper() + _ = size + return pageWith(t, content, func(w *reader.Writer, res reader.Dict) { + res["Font"].(reader.Dict)["F3"] = w.Add(reader.Dict{ + "Type": reader.Name("Font"), "Subtype": reader.Name("Type3"), + "FontMatrix": reader.Array{reader.Real(0.001), reader.Integer(0), reader.Integer(0), + reader.Real(0.001), reader.Integer(0), reader.Integer(0)}, + "Encoding": w.Add(reader.Dict{"Differences": reader.Array{ + reader.Integer(97), reader.Name("a"), reader.Name("b")}}), + "CharProcs": reader.Dict{}, + }) + }) +}