From ff72b4b198b6f703df02479eae41a7e3a70b1c80 Mon Sep 17 00:00:00 2001 From: Niklas Hauser Date: Sun, 4 Oct 2026 20:24:17 +0200 Subject: [PATCH 1/2] [py] Freeze the missing cxxheaderparser requirement --- tools/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/requirements.txt b/tools/requirements.txt index 2fc9eea..0e5bdc9 100644 --- a/tools/requirements.txt +++ b/tools/requirements.txt @@ -6,6 +6,7 @@ charset-normalizer==3.4.4 click==8.3.1 colorama==0.4.6 CppHeaderParser==2.7.4 +cxxheaderparser==1.9.2 ghp-import==2.1.0 idna==3.11 Jinja2==3.1.6 From 041fdb7518716bec735eebd421ef972b458d1034 Mon Sep 17 00:00:00 2001 From: Niklas Hauser Date: Sun, 4 Oct 2026 20:38:53 +0200 Subject: [PATCH 2/2] [pdf] Upgrade to pypdfium2 v5 Adapt to the API changes: the outline items are now bookmarks, page objects take a container argument, get_pos() became get_bounds(), the text render mode moved to the text object and pages must be registered with their document. Newer pdfium computes the loose character box from the real font ascent and descent, which breaks the layout heuristics. Compute the previous definition instead, including its single precision rounding. It also generates fewer spaces, so add back the one between a bullet point and its text for the list detection. Fix the named destination lookup, which passed the wrong ctypes types. --- CHANGELOG.md | 2 ++ pyproject.toml | 2 +- src/modm_data/pdf/character.py | 57 ++++++++++++++++++++++++++++++++-- src/modm_data/pdf/document.py | 33 +++++++++----------- src/modm_data/pdf/image.py | 4 +-- src/modm_data/pdf/page.py | 1 + src/modm_data/pdf/path.py | 2 +- src/modm_data/pdf2html/page.py | 21 ++++++++++++- tools/requirements.txt | 2 +- 9 files changed, 97 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a7acc32..70b7190 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ set of binary, source and similar compatible variants and publish them with the Peripheral Variant Explorer. - Add the register access to the SVD files reconstructed from the CMSIS headers. +- Upgrade to pypdfium2 v5 while keeping the HTML output identical by computing + the loose character bounding boxes the way the previous pdfium did. ## 0.0.4 diff --git a/pyproject.toml b/pyproject.toml index e0e0fe5..5de4206 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,7 +32,7 @@ dependencies = [ "jinja2>=3.1.3,<4", "lxml>=5.2.0,<6", "kuzu>=0.11.3,<0.12", - "pypdfium2==4.30.0", + "pypdfium2==5.14.0", "tqdm>=4.66.2,<5", ] dynamic = ["version"] diff --git a/src/modm_data/pdf/character.py b/src/modm_data/pdf/character.py index e99cd1f..7cd5b30 100644 --- a/src/modm_data/pdf/character.py +++ b/src/modm_data/pdf/character.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: MPL-2.0 import math +import copy import ctypes from functools import cached_property from enum import Enum @@ -54,11 +55,61 @@ def __init__(self, page: "modm_data.pdf.page.Page", index: int): # noqa: F821 self.weblink: "modm_data.pdf.link.WebLink" = None # noqa: F821 """The web link of this character or `None`""" - bbox = Rectangle(*self._text.get_charbox(self._index, loose=True)) + bbox = Rectangle(*self._loose_charbox()) if self._page.rotation: bbox = Rectangle(bbox.p0.y, self._page.height - bbox.p1.x, bbox.p1.y, self._page.height - bbox.p0.x) self._bbox = bbox + def _loose_charbox(self) -> tuple[float, float, float, float]: + # Newer pdfium computes the loose box from the real font ascent and + # descent, which makes it ~10% taller than the font size. All layout + # heuristics in pdf2html are tuned to the previous definition, where + # the box is exactly one font size tall and one advance width wide, and + # collapses for rotated characters (see `Page._fix_bboxes()`). + # The single precision rounding is part of that definition, since line + # heights are compared for equality when merging lines. + left, bottom, right, top = self._text.get_charbox(self._index, loose=True) + obj = pp.raw.FPDFText_GetTextObject(self._text, self._index) + size = pp.raw.FPDFText_GetFontSize(self._text, self._index) + if not obj or not size: + return left, bottom, right, top + font = pp.raw.FPDFTextObj_GetFont(obj) + ascent, descent, width = ctypes.c_float(), ctypes.c_float(), ctypes.c_float() + pp.raw.FPDFFont_GetAscent(font, 1000, ascent) + pp.raw.FPDFFont_GetDescent(font, 1000, descent) + if ascent.value == descent.value: + return left, bottom, right, top + + def _f32(value: float) -> float: + return ctypes.c_float(value).value + + matrix = pp.raw.FS_MATRIX() + x, y = ctypes.c_double(), ctypes.c_double() + assert pp.raw.FPDFText_GetMatrix(self._text, self._index, matrix) + assert pp.raw.FPDFText_GetCharOrigin(self._text, self._index, x, y) + scale = _f32(_f32(matrix.a * size) / (ascent.value - descent.value)) + bottom = _f32(y.value + _f32(descent.value * scale)) + top = _f32(y.value + _f32(ascent.value * scale)) + if pp.raw.FPDFText_IsGenerated(self._text, self._index): + return x.value, bottom, x.value, top + # The width lookup fails for characters without a reverse unicode + # mapping, in which case the new left and right bounds are close enough. + if pp.raw.FPDFFont_GetGlyphWidth(font, self.unicode, size, width) and width.value: + left, right = x.value, _f32(x.value + _f32(matrix.a * width.value)) + return left, bottom, right, top + + def generated_space(self) -> "Character": + """ + :return: A zero-sized space character located at the origin of this + character, like the ones pdfium generates for visual gaps. + """ + space = copy.copy(self) + space.__dict__ = {k: v for k, v in self.__dict__.items() if k.startswith("_") or k.endswith("link")} + space.unicode = 0x20 + space._bbox = space.tbbox = Rectangle(self.origin.x, self.origin.y, self.origin.x, self.origin.y) + space.origin = self.origin + return space + def _font_flags(self) -> tuple[str, int]: if self._font is None: font = ctypes.create_string_buffer(255) @@ -129,7 +180,9 @@ def theight(self) -> float: @cached_property def render_mode(self) -> RenderMode: """The render mode of the character.""" - return Character.RenderMode(pp.raw.FPDFText_GetTextRenderMode(self._text, self._index)) + return Character.RenderMode( + pp.raw.FPDFTextObj_GetTextRenderMode(pp.raw.FPDFText_GetTextObject(self._text, self._index)) + ) @cached_property def rotation(self) -> int: diff --git a/src/modm_data/pdf/document.py b/src/modm_data/pdf/document.py index a5b725b..66aed78 100644 --- a/src/modm_data/pdf/document.py +++ b/src/modm_data/pdf/document.py @@ -4,7 +4,7 @@ import ctypes import logging import pypdfium2 as pp -from typing import Iterator, Iterable +from typing import Iterator, Iterable, NamedTuple from pathlib import Path from functools import cached_property, cache from collections import defaultdict @@ -13,8 +13,11 @@ _LOGGER = logging.getLogger(__name__) -# We cannot monkey patch this class, since it's a named tuple. :-( -class _OutlineItem(pp.PdfOutlineItem): +class _OutlineItem(NamedTuple): + level: int + title: str + page_index: int + def __hash__(self) -> int: return hash(f"{self.page_index}+{self.title}") @@ -61,16 +64,16 @@ def metadata(self) -> dict[str, str]: def destinations(self) -> Iterator[tuple[int, str]]: """Yields (page 0-index, named destination) of the whole document.""" for ii in range(pp.raw.FPDF_CountNamedDests(self)): - length = pp.raw.FPDF_GetNamedDest(self, ii, 0, 0) - clength = ctypes.c_long(length) - cbuffer = ctypes.create_string_buffer(length * 2) + clength = ctypes.c_long() + pp.raw.FPDF_GetNamedDest(self, ii, None, clength) + cbuffer = ctypes.create_string_buffer(clength.value) dest = pp.raw.FPDF_GetNamedDest(self, ii, cbuffer, clength) - name = cbuffer.raw[: clength.value * 2].decode("utf-16-le").rstrip("\x00") + name = cbuffer.raw[: clength.value].decode("utf-16-le").rstrip("\x00") page = pp.raw.FPDFDest_GetDestPageIndex(self, dest) yield (page, name) @cached_property - def toc(self) -> list[pp.PdfOutlineItem]: + def toc(self) -> list[_OutlineItem]: """ The table of content as a sorted list of outline items ensuring item has a page index by reusing the last one. @@ -79,17 +82,9 @@ def toc(self) -> list[pp.PdfOutlineItem]: # Sometimes the TOC contains duplicates so we must use a set last_page_index = 0 for toc in self.get_toc(): - outline = _OutlineItem( - toc.level, - toc.title, - toc.is_closed, - toc.n_kids, - toc.page_index or last_page_index, - toc.view_mode, - toc.view_pos, - ) - last_page_index = toc.page_index or last_page_index - tocs.add(outline) + dest = toc.get_dest() + last_page_index = (dest and dest.get_index()) or last_page_index + tocs.add(_OutlineItem(toc.level, toc.get_title(), last_page_index)) return list(sorted(list(tocs), key=lambda o: (o.page_index, o.level, o.title))) @cached_property diff --git a/src/modm_data/pdf/image.py b/src/modm_data/pdf/image.py index a01b24d..1e814f9 100644 --- a/src/modm_data/pdf/image.py +++ b/src/modm_data/pdf/image.py @@ -25,7 +25,7 @@ def __init__(self, obj): """ :param obj: Page object of the image. """ - super().__init__(obj.raw, obj.page, obj.pdf, obj.level) + super().__init__(obj.raw, obj.page, obj.pdf, obj.container, obj.level) assert pp.raw.FPDFPageObj_GetType(obj.raw) == pp.raw.FPDF_PAGEOBJ_IMAGE self.type = pp.raw.FPDF_PAGEOBJ_IMAGE @@ -50,7 +50,7 @@ def matrix(self) -> pp.PdfMatrix: @cached_property def bbox(self) -> Rectangle: """The bounding box of the image.""" - bbox = Rectangle(*self.get_pos()) + bbox = Rectangle(*self.get_bounds()) if self.page.rotation: bbox = Rectangle(bbox.p0.y, self.page.height - bbox.p1.x, bbox.p1.y, self.page.height - bbox.p0.x) return bbox diff --git a/src/modm_data/pdf/page.py b/src/modm_data/pdf/page.py index 1caafda..3fdcefd 100644 --- a/src/modm_data/pdf/page.py +++ b/src/modm_data/pdf/page.py @@ -38,6 +38,7 @@ def __init__(self, document: "modm_data.pdf.Document", index: int): # noqa: F82 """1-index page number.""" super().__init__(pp.raw.FPDF_LoadPage(document, index), document, document.formenv) + document._add_kid(self) self._links = None self._weblinks = None self._linked = False diff --git a/src/modm_data/pdf/path.py b/src/modm_data/pdf/path.py index fefbbc8..94f9e72 100644 --- a/src/modm_data/pdf/path.py +++ b/src/modm_data/pdf/path.py @@ -49,7 +49,7 @@ def __init__(self, obj): """ :param obj: PDF object of the path. """ - super().__init__(obj.raw, obj.page, obj.pdf, obj.level) + super().__init__(obj.raw, obj.page, obj.pdf, obj.container, obj.level) assert pp.raw.FPDFPageObj_GetType(obj.raw) == pp.raw.FPDF_PAGEOBJ_PATH self.type = pp.raw.FPDF_PAGEOBJ_PATH diff --git a/src/modm_data/pdf2html/page.py b/src/modm_data/pdf2html/page.py index b926be9..0644a9d 100644 --- a/src/modm_data/pdf2html/page.py +++ b/src/modm_data/pdf2html/page.py @@ -265,10 +265,13 @@ def sort_key(char): return char.origin.x + 1e9 return char.origin.x + chars = sorted(line.chars, key=sort_key) + if not line.rotation: + chars = self._chars_with_bullet_space(chars) sorted_lines.append( CharLine( self, - sorted(line.chars, key=sort_key), + chars, line.bottom, line.origin, line.top, @@ -281,6 +284,22 @@ def sort_key(char): return sorted_lines + @staticmethod + def _chars_with_bullet_space(chars: list) -> list: + # Newer pdfium does not generate a space for every visual gap anymore, + # but the list detection relies on the one between bullet and text. + for ii, (bullet, char) in enumerate(zip(chars, chars[1:]), 1): + if bullet.unicode == 0x20: + continue + if ( + bullet.char in {"•", "–"} + and char.unicode not in {0x20, 0xA, 0xD} + and char.origin.x - bullet.bbox.right >= 0.25 * bullet.height + ): + return chars[:ii] + [char.generated_space()] + chars[ii:] + break + return chars + def graphic_bboxes_in_area( self, area: Rectangle, with_graphics: bool = True ) -> list[tuple[Rectangle, Table | Figure | None]]: diff --git a/tools/requirements.txt b/tools/requirements.txt index 0e5bdc9..452e23c 100644 --- a/tools/requirements.txt +++ b/tools/requirements.txt @@ -27,7 +27,7 @@ platformdirs==4.5.1 ply==3.11 Pygments==2.19.2 pymdown-extensions==10.20 -pypdfium2==4.30.0 +pypdfium2==5.14.0 python-dateutil==2.9.0.post0 PyYAML==6.0.3 pyyaml_env_tag==1.1