From 69faf5c39a5c33ee320d8fe5d13d38c0ca15b802 Mon Sep 17 00:00:00 2001 From: Niklas Hauser Date: Sun, 4 Oct 2026 21:38:50 +0200 Subject: [PATCH 1/2] [pdf] Compute the character boxes from font size and advance width The layout heuristics need the font size and the advance width of a character, so compute a box from these and place it with the text matrix instead of relying on the loose box of pdfium. This also works for rotated text, which removes the need to fix up their boxes. Lines of the same font size are now merged into the longer line, since comparing the line heights for equality depended on rounding errors. --- src/modm_data/pdf/character.py | 68 +++++++++++++++++----------------- src/modm_data/pdf/document.py | 2 - src/modm_data/pdf/page.py | 34 +---------------- src/modm_data/pdf2html/page.py | 14 ++++--- 4 files changed, 45 insertions(+), 73 deletions(-) diff --git a/src/modm_data/pdf/character.py b/src/modm_data/pdf/character.py index 7cd5b30..e2705f3 100644 --- a/src/modm_data/pdf/character.py +++ b/src/modm_data/pdf/character.py @@ -55,48 +55,50 @@ def __init__(self, page: "modm_data.pdf.page.Page", index: int): # noqa: F821 self.weblink: "modm_data.pdf.link.WebLink" = None # noqa: F821 """The web link of this character or `None`""" - bbox = Rectangle(*self._loose_charbox()) + bbox = Rectangle(*self._em_box()) if self._page.rotation: bbox = Rectangle(bbox.p0.y, self._page.height - bbox.p1.x, bbox.p1.y, self._page.height - bbox.p0.x) self._bbox = bbox - def _loose_charbox(self) -> tuple[float, float, float, float]: - # Newer pdfium computes the loose box from the real font ascent and - # descent, which makes it ~10% taller than the font size. All layout - # heuristics in pdf2html are tuned to the previous definition, where - # the box is exactly one font size tall and one advance width wide, and - # collapses for rotated characters (see `Page._fix_bboxes()`). - # The single precision rounding is part of that definition, since line - # heights are compared for equality when merging lines. - left, bottom, right, top = self._text.get_charbox(self._index, loose=True) + def _em_box(self) -> tuple[float, float, float, float]: + # The layout heuristics need the font size and the advance width of a + # character, not the extent of its glyph. We therefore compute our own + # box, which is exactly one font size tall, one advance width wide and + # placed by the text matrix, so that it also works for rotated text. + # The loose box of pdfium is not suitable, since its definition changed + # between versions and its height depends on the font metrics. + tight = self._text.get_charbox(self._index) obj = pp.raw.FPDFText_GetTextObject(self._text, self._index) size = pp.raw.FPDFText_GetFontSize(self._text, self._index) - if not obj or not size: - return left, bottom, right, top + if not obj or not size or pp.raw.FPDFText_IsGenerated(self._text, self._index): + return tight font = pp.raw.FPDFTextObj_GetFont(obj) ascent, descent, width = ctypes.c_float(), ctypes.c_float(), ctypes.c_float() - pp.raw.FPDFFont_GetAscent(font, 1000, ascent) - pp.raw.FPDFFont_GetDescent(font, 1000, descent) + pp.raw.FPDFFont_GetAscent(font, size, ascent) + pp.raw.FPDFFont_GetDescent(font, size, descent) if ascent.value == descent.value: - return left, bottom, right, top - - def _f32(value: float) -> float: - return ctypes.c_float(value).value - - matrix = pp.raw.FS_MATRIX() - x, y = ctypes.c_double(), ctypes.c_double() - assert pp.raw.FPDFText_GetMatrix(self._text, self._index, matrix) - assert pp.raw.FPDFText_GetCharOrigin(self._text, self._index, x, y) - scale = _f32(_f32(matrix.a * size) / (ascent.value - descent.value)) - bottom = _f32(y.value + _f32(descent.value * scale)) - top = _f32(y.value + _f32(ascent.value * scale)) - if pp.raw.FPDFText_IsGenerated(self._text, self._index): - return x.value, bottom, x.value, top - # The width lookup fails for characters without a reverse unicode - # mapping, in which case the new left and right bounds are close enough. - if pp.raw.FPDFFont_GetGlyphWidth(font, self.unicode, size, width) and width.value: - left, right = x.value, _f32(x.value + _f32(matrix.a * width.value)) - return left, bottom, right, top + return tight + + m = pp.raw.FS_MATRIX() + ox, oy = ctypes.c_double(), ctypes.c_double() + assert pp.raw.FPDFText_GetMatrix(self._text, self._index, m) + assert pp.raw.FPDFText_GetCharOrigin(self._text, self._index, ox, oy) + ox, oy = ox.value, oy.value + + # Split the font size above and below the baseline like the font does + scale = size / (ascent.value - descent.value) + y0, y1 = descent.value * scale, ascent.value * scale + x1 = 0 + if pp.raw.FPDFFont_GetGlyphWidth(font, self.unicode, size, width): + x1 = width.value + if not x1 and (norm := m.a * m.a + m.b * m.b): + # The width lookup fails for characters without a reverse unicode + # mapping, so we use the extent of the glyph along the baseline. + x1 = max(((x - ox) * m.a + (y - oy) * m.b) / norm for x in tight[::2] for y in tight[1::2]) + + xs = [ox + m.a * x + m.c * y for x in (0, x1) for y in (y0, y1)] + ys = [oy + m.b * x + m.d * y for x in (0, x1) for y in (y0, y1)] + return min(xs), min(ys), max(xs), max(ys) def generated_space(self) -> "Character": """ diff --git a/src/modm_data/pdf/document.py b/src/modm_data/pdf/document.py index 66aed78..4e270d6 100644 --- a/src/modm_data/pdf/document.py +++ b/src/modm_data/pdf/document.py @@ -7,7 +7,6 @@ from typing import Iterator, Iterable, NamedTuple from pathlib import Path from functools import cached_property, cache -from collections import defaultdict from .page import Page _LOGGER = logging.getLogger(__name__) @@ -52,7 +51,6 @@ def __init__(self, path: Path, autoclose: bool = False): """Stem of the document file name""" super().__init__(path, autoclose=autoclose) self._path = path - self._bbox_cache = defaultdict(dict) _LOGGER.debug(f"Loading: {path}") @cached_property diff --git a/src/modm_data/pdf/page.py b/src/modm_data/pdf/page.py index 3fdcefd..ad47dd2 100644 --- a/src/modm_data/pdf/page.py +++ b/src/modm_data/pdf/page.py @@ -22,9 +22,8 @@ class Page(pp.PdfPage): """ - This class provides low-level access to graphics and characters of the page. - It also fixes missing bounding boxes for rotates characters on page load, - as well as allow searching for characters in an area instead of just text. + This class provides low-level access to graphics and characters of the page + and allows searching for characters in an area instead of just text. """ def __init__(self, document: "modm_data.pdf.Document", index: int): # noqa: F821 @@ -52,8 +51,6 @@ def __init__(self, document: "modm_data.pdf.Document", index: int): # noqa: F82 weakref.finalize(self, pp.raw.FPDF_StructTree_Close, self._structtree) weakref.finalize(self, pp.raw.FPDFLink_CloseWebLinks, self._linkpage) - self._fix_bboxes() - @cached_property def label(self) -> str: """The page label.""" @@ -256,30 +253,3 @@ def _charlines(self): orderedchars[ypos] = sorted(chars, key=lambda c: c.bbox.midpoint.x) return orderedchars - - def _fix_bboxes(self): - def _key(char): - height = round(char.tbbox.height, 1) - width = round(char.tbbox.width, 1) - return f"{char.font} {char.unicode} {height} {width}" - - fix_chars = [] - for char in self.chars: - if not char._bbox.width or not char._bbox.height: - if char._rotation: - fix_chars.append(char) - elif char.unicode not in {0xA, 0xD}: - fix_chars.append(char) - elif char.unicode not in {0xA, 0xD} and not char._rotation and _key(char) not in self.pdf._bbox_cache: - bbox = char._bbox.translated(-char.origin).rotated(self.rotation + char._rotation) - self.pdf._bbox_cache[_key(char)] = (char, bbox) - # print("->", _key(char), char.descr(), char.height, char.rotation, char._rotation, self.rotation) - for char in fix_chars: - bbox = self.pdf._bbox_cache.get(_key(char)) - if bbox is not None: - # print("<-", char.descr(), char._rotation, char.rotation, char.height) - _, bbox = bbox - bbox = bbox.rotated(-self.rotation - char._rotation).translated(char.origin) - char._bbox = bbox - elif char.unicode not in {0x20, 0xA, 0xD}: - _LOGGER.debug(f"Unable to fix bbox for {char.descr()}!") diff --git a/src/modm_data/pdf2html/page.py b/src/modm_data/pdf2html/page.py index 0644a9d..3033c50 100644 --- a/src/modm_data/pdf2html/page.py +++ b/src/modm_data/pdf2html/page.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: MPL-2.0 import logging +import math import statistics from typing import Callable from functools import cached_property @@ -219,12 +220,13 @@ def charlines_in_area( height = max(current_line.height, next_line.height) # Calculate overlap via normalize origin (increasing with line index) if (current_line._sort_origin + rtol * height) > (next_line._sort_origin - rtol * height): - # if line.rotation or self.rotation: - # # The next line overlaps this one, we merge the shorter line - # # (typically super- and subscript) into taller line - # use_current = len(current_line.chars) >= len(next_line.chars) - # else: - use_current = current_line.height >= next_line.height + # The next line overlaps this one, we merge the shorter line + # (typically super- and subscript) into the taller line. Lines + # of the same font size are merged into the longer line. + if math.isclose(current_line.height, next_line.height, rel_tol=0.01): + use_current = len(current_line.chars) >= len(next_line.chars) + else: + use_current = current_line.height > next_line.height line = current_line if use_current else next_line current_line = CharLine( self, From e073270e95490ae3a49622bd6a52dc39ae558d27 Mon Sep 17 00:00:00 2001 From: Niklas Hauser Date: Sun, 4 Oct 2026 22:10:26 +0200 Subject: [PATCH 2/2] [pdf2html] Keep stacked lines of the same font size separate Lines of the same font size, whose characters are printed on top of each other, are not sub- or superscript but separate lines, like the numerator and denominator of a formula or overlapping index entries. --- src/modm_data/pdf2html/page.py | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/src/modm_data/pdf2html/page.py b/src/modm_data/pdf2html/page.py index 3033c50..4de16b9 100644 --- a/src/modm_data/pdf2html/page.py +++ b/src/modm_data/pdf2html/page.py @@ -212,18 +212,41 @@ def charlines_in_area( if not bbox_lines: return [] + def _stacked(line0, line1) -> bool: + # Characters of the same font size, that overlap along the line by + # more than half their width, are printed on top of each other. + horizontal = not line0.rotation + + def _spans(line): + chars = (c for c in line.chars if c.unicode not in {0x20, 0xA, 0xD}) + chars = (c for c in chars if math.isclose(c.height, line.height, rel_tol=0.01)) + return [(c.bbox.left, c.bbox.right) if horizontal else (c.bbox.bottom, c.bbox.top) for c in chars] + + spans1 = _spans(line1) + for start0, stop0 in _spans(line0): + for start1, stop1 in spans1: + overlap = min(stop0, stop1) - max(start0, start1) + if overlap > 0.5 * min(stop0 - start0, stop1 - start1): + return True + return False + # Merge lines that have overlapping bbox_lines - # FIXME: This merges lines that "collide" vertically like in formulas merged_lines = [] current_line = bbox_lines[0] for next_line in bbox_lines[1:]: height = max(current_line.height, next_line.height) + same_height = math.isclose(current_line.height, next_line.height, rel_tol=0.01) # Calculate overlap via normalize origin (increasing with line index) - if (current_line._sort_origin + rtol * height) > (next_line._sort_origin - rtol * height): + if (current_line._sort_origin + rtol * height) > (next_line._sort_origin - rtol * height) and not ( + # Lines of the same font size printed on top of each other are + # separate lines, like the numerator and denominator of a formula + same_height and current_line.rotation == next_line.rotation and _stacked(current_line, next_line) + ): # The next line overlaps this one, we merge the shorter line # (typically super- and subscript) into the taller line. Lines - # of the same font size are merged into the longer line. - if math.isclose(current_line.height, next_line.height, rel_tol=0.01): + # of the same font size continue each other with a slightly + # shifted baseline, so we merge them into the longer line. + if same_height: use_current = len(current_line.chars) >= len(next_line.chars) else: use_current = current_line.height > next_line.height