A mark, not the same mark twice

tables.go

@@ -3,6 +3,7 @@ package extract
 import (
 	"sort"
 	"strings"
+	"unicode"
 
 	"github.com/go-pdfkit/reader"
 )
@@ -356,41 +357,59 @@ func dropEmptyColumns(rows [][]string) [][]string {
 	return out
 }
 
-// markerColumn reports whether the first column is a list's bullet rather than
-// data: one short mark, the same one wherever it appears, and nothing else.
+// markerColumn reports whether the first column holds a list's bullets rather
+// than data: short marks, and nothing that is a word or a number.
 //
 // ⛔ Measured, on a 213-page document of the author's: 588 "tables" of two
 // columns, and the first column of nearly all of them held "•" or U+F0A7. A
 // bulleted list IS a two-column grid, and no amount of looking at the
-// alignment will say otherwise — the thing that distinguishes it is that one
-// of its columns says the same thing wherever it says anything, and a column
-// like that distinguishes nothing. Dropping it leaves a list with one column,
-// which is not a table, and leaves a real table with a repeated mark in it one
-// column shorter and no worse informed.
+// alignment will say otherwise. What distinguishes it is that one of its
+// columns holds no DATA — a mark of one or two characters with no letter and
+// no digit in it is an ornament, and dropping it leaves a list with one
+// column, which is not a table.
 //
-// ⛔ The empty cells are why this is not simply "every row says the same": a
+// ⛔ A MARK, not the same mark twice. Insisting on one repeated mark read
+// "• • - -" as a table of four rows, because the bullets were not all the
+// same character — and a reader sees two lists there. The question is whether
+// the column carries information, and `-` carries exactly as little as `•`.
+//
+// ⛔ A letter or a digit is information, which is what keeps a real table.
+// `A` and `B` down a first column are labels, `1.` and `2.` are a numbering
+// somebody may want, and `1990` is a date. Taking those would be deciding
+// that a table of years is a list.
+//
+// ⛔ The empty cells are why this is not simply "every row holds one": a
 // bullet's text wraps, and the lines it wraps onto have no bullet of their
 // own. Insisting on every row let the wrapped lists back in — which is how
 // this read as a fix while a document of nothing but bullet lists still came
 // back as fourteen rows of table.
 func markerColumn(rows [][]string) bool {
-	mark := ""
 	for _, row := range rows {
-		switch {
-		case row[0] == "":
+		if row[0] == "" {
 			continue
-		case mark == "":
-			mark = row[0]
-			if len([]rune(mark)) > markerRunes {
-				return false
-			}
-		case row[0] != mark:
+		}
+		if !isMark(row[0]) {
+			return false
+		}
+	}
+	// Reaching here means every cell of the column was empty or a mark, and
+	// they cannot all have been empty: [dropEmptyColumns] has already taken
+	// away the columns nothing was drawn in. So at least one was a mark, and
+	// there is no "found nothing" case to report.
+	return true
+}
+
+// isMark reports whether a cell is an ornament rather than data.
+func isMark(cell string) bool {
+	runes := []rune(cell)
+	if len(runes) == 0 || len(runes) > markerRunes {
+		return false
+	}
+	for _, r := range runes {
+		if unicode.IsLetter(r) || unicode.IsDigit(r) {
 			return false
 		}
 	}
-	// mark is never still empty here: [dropEmptyColumns] has already taken
-	// away the columns nothing was drawn in, so the first column has something
-	// in it somewhere.
 	return true
 }
 

tables_test.go

@@ -232,7 +232,50 @@ func TestABulletedListIsNotATable(t *testing.T) {
 	}
 }
 
-// The mark has to be a MARK. A first column that repeats a word is a column
+// ⛔ A MARK, not the same mark twice. Insisting on one repeated mark read
+// "• • - -" as a table of four rows, because the bullets were not all the
+// same character — and a reader sees two lists there.
+func TestTwoDifferentMarksAreStillMarks(t *testing.T) {
+	d := pageWith(t, strings.Join([]string{
+		"BT /F1 10 Tf 20 150 Td (\225) Tj 40 0 Td (conduct the project) Tj ET",
+		"BT /F1 10 Tf 20 130 Td (\225) Tj 40 0 Td (inform the trustees) Tj ET",
+		"BT /F1 10 Tf 20 110 Td (-) Tj 40 0 Td (federate the staff) Tj ET",
+		"BT /F1 10 Tf 20 90 Td (-) Tj 40 0 Td (and animate it) Tj ET",
+	}, "\n"), nil)
+	tables, _ := Tables(d, 1)
+	if len(tables) != 0 {
+		t.Errorf("two kinds of bullet came back as %d table(s): %+v", len(tables), tables)
+	}
+}
+
+// ⛔ A letter or a digit is information, which is what keeps a real table. A
+// numbering somebody may want is not an ornament, and a table of years is
+// nothing but digits.
+func TestAFirstColumnOfLabelsOrNumbersIsStillAColumn(t *testing.T) {
+	for _, first := range []struct{ a, b string }{
+		{"A", "B"},   // labels
+		{"1.", "2."}, // a numbering
+		{"90", "91"}, // years
+		// Three characters is not short. A mark is one or two; anything
+		// longer was drawn to be read.
+		{"---", "---"},
+	} {
+		d := pageWith(t, strings.Join([]string{
+			"BT /F1 10 Tf 20 150 Td (" + first.a + ") Tj 40 0 Td (one) Tj ET",
+			"BT /F1 10 Tf 20 130 Td (" + first.b + ") Tj 40 0 Td (two) Tj ET",
+		}, "\n"), nil)
+		tables, _ := Tables(d, 1)
+		if len(tables) != 1 {
+			t.Fatalf("%q/%q: %d tables, want 1: %+v", first.a, first.b, len(tables), tables)
+		}
+		want := [][]string{{first.a, "one"}, {first.b, "two"}}
+		if !reflect.DeepEqual(tables[0].Rows, want) {
+			t.Errorf("%q/%q: the table reads %q, want %q", first.a, first.b, tables[0].Rows, want)
+		}
+	}
+}
+
+// The mark has to be SHORT. A first column that repeats a word is a column
 // that repeats a word, and throwing it away would throw away the data.
 func TestAFirstColumnThatRepeatsAWordIsStillAColumn(t *testing.T) {
 	d := pageWith(t, strings.Join([]string{