[pdf] Upgrade to pypdfium2 v5

CHANGELOG.md

@@ -6,6 +6,8 @@
   set of binary, source and similar compatible variants and publish them with
   the Peripheral Variant Explorer.
 - Add the register access to the SVD files reconstructed from the CMSIS headers.
+- Upgrade to pypdfium2 v5 while keeping the HTML output identical by computing
+  the loose character bounding boxes the way the previous pdfium did.
 
 ## 0.0.4
 

pyproject.toml

@@ -32,7 +32,7 @@ dependencies = [
     "jinja2>=3.1.3,<4",
     "lxml>=5.2.0,<6",
     "kuzu>=0.11.3,<0.12",
-    "pypdfium2==4.30.0",
+    "pypdfium2==5.14.0",
     "tqdm>=4.66.2,<5",
 ]
 dynamic = ["version"]

src/modm_data/pdf/character.py

@@ -2,6 +2,7 @@
 # SPDX-License-Identifier: MPL-2.0
 
 import math
+import copy
 import ctypes
 from functools import cached_property
 from enum import Enum
@@ -54,11 +55,61 @@ def __init__(self, page: "modm_data.pdf.page.Page", index: int):  # noqa: F821
         self.weblink: "modm_data.pdf.link.WebLink" = None  # noqa: F821
         """The web link of this character or `None`"""
 
-        bbox = Rectangle(*self._text.get_charbox(self._index, loose=True))
+        bbox = Rectangle(*self._loose_charbox())
         if self._page.rotation:
             bbox = Rectangle(bbox.p0.y, self._page.height - bbox.p1.x, bbox.p1.y, self._page.height - bbox.p0.x)
         self._bbox = bbox
 
+    def _loose_charbox(self) -> tuple[float, float, float, float]:
+        # Newer pdfium computes the loose box from the real font ascent and
+        # descent, which makes it ~10% taller than the font size. All layout
+        # heuristics in pdf2html are tuned to the previous definition, where
+        # the box is exactly one font size tall and one advance width wide, and
+        # collapses for rotated characters (see `Page._fix_bboxes()`).
+        # The single precision rounding is part of that definition, since line
+        # heights are compared for equality when merging lines.
+        left, bottom, right, top = self._text.get_charbox(self._index, loose=True)
+        obj = pp.raw.FPDFText_GetTextObject(self._text, self._index)
+        size = pp.raw.FPDFText_GetFontSize(self._text, self._index)
+        if not obj or not size:
+            return left, bottom, right, top
+        font = pp.raw.FPDFTextObj_GetFont(obj)
+        ascent, descent, width = ctypes.c_float(), ctypes.c_float(), ctypes.c_float()
+        pp.raw.FPDFFont_GetAscent(font, 1000, ascent)
+        pp.raw.FPDFFont_GetDescent(font, 1000, descent)
+        if ascent.value == descent.value:
+            return left, bottom, right, top
+
+        def _f32(value: float) -> float:
+            return ctypes.c_float(value).value
+
+        matrix = pp.raw.FS_MATRIX()
+        x, y = ctypes.c_double(), ctypes.c_double()
+        assert pp.raw.FPDFText_GetMatrix(self._text, self._index, matrix)
+        assert pp.raw.FPDFText_GetCharOrigin(self._text, self._index, x, y)
+        scale = _f32(_f32(matrix.a * size) / (ascent.value - descent.value))
+        bottom = _f32(y.value + _f32(descent.value * scale))
+        top = _f32(y.value + _f32(ascent.value * scale))
+        if pp.raw.FPDFText_IsGenerated(self._text, self._index):
+            return x.value, bottom, x.value, top
+        # The width lookup fails for characters without a reverse unicode
+        # mapping, in which case the new left and right bounds are close enough.
+        if pp.raw.FPDFFont_GetGlyphWidth(font, self.unicode, size, width) and width.value:
+            left, right = x.value, _f32(x.value + _f32(matrix.a * width.value))
+        return left, bottom, right, top
+
+    def generated_space(self) -> "Character":
+        """
+        :return: A zero-sized space character located at the origin of this
+                 character, like the ones pdfium generates for visual gaps.
+        """
+        space = copy.copy(self)
+        space.__dict__ = {k: v for k, v in self.__dict__.items() if k.startswith("_") or k.endswith("link")}
+        space.unicode = 0x20
+        space._bbox = space.tbbox = Rectangle(self.origin.x, self.origin.y, self.origin.x, self.origin.y)
+        space.origin = self.origin
+        return space
+
     def _font_flags(self) -> tuple[str, int]:
         if self._font is None:
             font = ctypes.create_string_buffer(255)
@@ -129,7 +180,9 @@ def theight(self) -> float:
     @cached_property
     def render_mode(self) -> RenderMode:
         """The render mode of the character."""
-        return Character.RenderMode(pp.raw.FPDFText_GetTextRenderMode(self._text, self._index))
+        return Character.RenderMode(
+            pp.raw.FPDFTextObj_GetTextRenderMode(pp.raw.FPDFText_GetTextObject(self._text, self._index))
+        )
 
     @cached_property
     def rotation(self) -> int:

src/modm_data/pdf/document.py

@@ -4,7 +4,7 @@
 import ctypes
 import logging
 import pypdfium2 as pp
-from typing import Iterator, Iterable
+from typing import Iterator, Iterable, NamedTuple
 from pathlib import Path
 from functools import cached_property, cache
 from collections import defaultdict
@@ -13,8 +13,11 @@
 _LOGGER = logging.getLogger(__name__)
 
 
-# We cannot monkey patch this class, since it's a named tuple. :-(
-class _OutlineItem(pp.PdfOutlineItem):
+class _OutlineItem(NamedTuple):
+    level: int
+    title: str
+    page_index: int
+
     def __hash__(self) -> int:
         return hash(f"{self.page_index}+{self.title}")
 
@@ -61,16 +64,16 @@ def metadata(self) -> dict[str, str]:
     def destinations(self) -> Iterator[tuple[int, str]]:
         """Yields (page 0-index, named destination) of the whole document."""
         for ii in range(pp.raw.FPDF_CountNamedDests(self)):
-            length = pp.raw.FPDF_GetNamedDest(self, ii, 0, 0)
-            clength = ctypes.c_long(length)
-            cbuffer = ctypes.create_string_buffer(length * 2)
+            clength = ctypes.c_long()
+            pp.raw.FPDF_GetNamedDest(self, ii, None, clength)
+            cbuffer = ctypes.create_string_buffer(clength.value)
             dest = pp.raw.FPDF_GetNamedDest(self, ii, cbuffer, clength)
-            name = cbuffer.raw[: clength.value * 2].decode("utf-16-le").rstrip("\x00")
+            name = cbuffer.raw[: clength.value].decode("utf-16-le").rstrip("\x00")
             page = pp.raw.FPDFDest_GetDestPageIndex(self, dest)
             yield (page, name)
 
     @cached_property
-    def toc(self) -> list[pp.PdfOutlineItem]:
+    def toc(self) -> list[_OutlineItem]:
         """
         The table of content as a sorted list of outline items ensuring item has
         a page index by reusing the last one.
@@ -79,17 +82,9 @@ def toc(self) -> list[pp.PdfOutlineItem]:
         # Sometimes the TOC contains duplicates so we must use a set
         last_page_index = 0
         for toc in self.get_toc():
-            outline = _OutlineItem(
-                toc.level,
-                toc.title,
-                toc.is_closed,
-                toc.n_kids,
-                toc.page_index or last_page_index,
-                toc.view_mode,
-                toc.view_pos,
-            )
-            last_page_index = toc.page_index or last_page_index
-            tocs.add(outline)
+            dest = toc.get_dest()
+            last_page_index = (dest and dest.get_index()) or last_page_index
+            tocs.add(_OutlineItem(toc.level, toc.get_title(), last_page_index))
         return list(sorted(list(tocs), key=lambda o: (o.page_index, o.level, o.title)))
 
     @cached_property

src/modm_data/pdf/image.py

@@ -25,7 +25,7 @@ def __init__(self, obj):
         """
         :param obj: Page object of the image.
         """
-        super().__init__(obj.raw, obj.page, obj.pdf, obj.level)
+        super().__init__(obj.raw, obj.page, obj.pdf, obj.container, obj.level)
         assert pp.raw.FPDFPageObj_GetType(obj.raw) == pp.raw.FPDF_PAGEOBJ_IMAGE
         self.type = pp.raw.FPDF_PAGEOBJ_IMAGE
 
@@ -50,7 +50,7 @@ def matrix(self) -> pp.PdfMatrix:
     @cached_property
     def bbox(self) -> Rectangle:
         """The bounding box of the image."""
-        bbox = Rectangle(*self.get_pos())
+        bbox = Rectangle(*self.get_bounds())
         if self.page.rotation:
             bbox = Rectangle(bbox.p0.y, self.page.height - bbox.p1.x, bbox.p1.y, self.page.height - bbox.p0.x)
         return bbox

src/modm_data/pdf/page.py

@@ -38,6 +38,7 @@ def __init__(self, document: "modm_data.pdf.Document", index: int):  # noqa: F82
         """1-index page number."""
 
         super().__init__(pp.raw.FPDF_LoadPage(document, index), document, document.formenv)
+        document._add_kid(self)
         self._links = None
         self._weblinks = None
         self._linked = False

src/modm_data/pdf/path.py

@@ -49,7 +49,7 @@ def __init__(self, obj):
         """
         :param obj: PDF object of the path.
         """
-        super().__init__(obj.raw, obj.page, obj.pdf, obj.level)
+        super().__init__(obj.raw, obj.page, obj.pdf, obj.container, obj.level)
         assert pp.raw.FPDFPageObj_GetType(obj.raw) == pp.raw.FPDF_PAGEOBJ_PATH
         self.type = pp.raw.FPDF_PAGEOBJ_PATH
 

src/modm_data/pdf2html/page.py

@@ -265,10 +265,13 @@ def sort_key(char):
                         return char.origin.x + 1e9
                     return char.origin.x
 
+            chars = sorted(line.chars, key=sort_key)
+            if not line.rotation:
+                chars = self._chars_with_bullet_space(chars)
             sorted_lines.append(
                 CharLine(
                     self,
-                    sorted(line.chars, key=sort_key),
+                    chars,
                     line.bottom,
                     line.origin,
                     line.top,
@@ -281,6 +284,22 @@ def sort_key(char):
 
         return sorted_lines
 
+    @staticmethod
+    def _chars_with_bullet_space(chars: list) -> list:
+        # Newer pdfium does not generate a space for every visual gap anymore,
+        # but the list detection relies on the one between bullet and text.
+        for ii, (bullet, char) in enumerate(zip(chars, chars[1:]), 1):
+            if bullet.unicode == 0x20:
+                continue
+            if (
+                bullet.char in {"•", "–"}
+                and char.unicode not in {0x20, 0xA, 0xD}
+                and char.origin.x - bullet.bbox.right >= 0.25 * bullet.height
+            ):
+                return chars[:ii] + [char.generated_space()] + chars[ii:]
+            break
+        return chars
+
     def graphic_bboxes_in_area(
         self, area: Rectangle, with_graphics: bool = True
     ) -> list[tuple[Rectangle, Table | Figure | None]]:

tools/requirements.txt

@@ -6,6 +6,7 @@ charset-normalizer==3.4.4
 click==8.3.1
 colorama==0.4.6
 CppHeaderParser==2.7.4
+cxxheaderparser==1.9.2
 ghp-import==2.1.0
 idna==3.11
 Jinja2==3.1.6
@@ -26,7 +27,7 @@ platformdirs==4.5.1
 ply==3.11
 Pygments==2.19.2
 pymdown-extensions==10.20
-pypdfium2==4.30.0
+pypdfium2==5.14.0
 python-dateutil==2.9.0.post0
 PyYAML==6.0.3
 pyyaml_env_tag==1.1