diff --git a/CHANGELOG.md b/CHANGELOG.md index a7acc32..70b7190 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ set of binary, source and similar compatible variants and publish them with the Peripheral Variant Explorer. - Add the register access to the SVD files reconstructed from the CMSIS headers. +- Upgrade to pypdfium2 v5 while keeping the HTML output identical by computing + the loose character bounding boxes the way the previous pdfium did. ## 0.0.4 diff --git a/pyproject.toml b/pyproject.toml index e0e0fe5..5de4206 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,7 +32,7 @@ dependencies = [ "jinja2>=3.1.3,<4", "lxml>=5.2.0,<6", "kuzu>=0.11.3,<0.12", - "pypdfium2==4.30.0", + "pypdfium2==5.14.0", "tqdm>=4.66.2,<5", ] dynamic = ["version"] diff --git a/src/modm_data/pdf/character.py b/src/modm_data/pdf/character.py index e99cd1f..7cd5b30 100644 --- a/src/modm_data/pdf/character.py +++ b/src/modm_data/pdf/character.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: MPL-2.0 import math +import copy import ctypes from functools import cached_property from enum import Enum @@ -54,11 +55,61 @@ def __init__(self, page: "modm_data.pdf.page.Page", index: int): # noqa: F821 self.weblink: "modm_data.pdf.link.WebLink" = None # noqa: F821 """The web link of this character or `None`""" - bbox = Rectangle(*self._text.get_charbox(self._index, loose=True)) + bbox = Rectangle(*self._loose_charbox()) if self._page.rotation: bbox = Rectangle(bbox.p0.y, self._page.height - bbox.p1.x, bbox.p1.y, self._page.height - bbox.p0.x) self._bbox = bbox + def _loose_charbox(self) -> tuple[float, float, float, float]: + # Newer pdfium computes the loose box from the real font ascent and + # descent, which makes it ~10% taller than the font size. All layout + # heuristics in pdf2html are tuned to the previous definition, where + # the box is exactly one font size tall and one advance width wide, and + # collapses for rotated characters (see `Page._fix_bboxes()`). + # The single precision rounding is part of that definition, since line + # heights are compared for equality when merging lines. + left, bottom, right, top = self._text.get_charbox(self._index, loose=True) + obj = pp.raw.FPDFText_GetTextObject(self._text, self._index) + size = pp.raw.FPDFText_GetFontSize(self._text, self._index) + if not obj or not size: + return left, bottom, right, top + font = pp.raw.FPDFTextObj_GetFont(obj) + ascent, descent, width = ctypes.c_float(), ctypes.c_float(), ctypes.c_float() + pp.raw.FPDFFont_GetAscent(font, 1000, ascent) + pp.raw.FPDFFont_GetDescent(font, 1000, descent) + if ascent.value == descent.value: + return left, bottom, right, top + + def _f32(value: float) -> float: + return ctypes.c_float(value).value + + matrix = pp.raw.FS_MATRIX() + x, y = ctypes.c_double(), ctypes.c_double() + assert pp.raw.FPDFText_GetMatrix(self._text, self._index, matrix) + assert pp.raw.FPDFText_GetCharOrigin(self._text, self._index, x, y) + scale = _f32(_f32(matrix.a * size) / (ascent.value - descent.value)) + bottom = _f32(y.value + _f32(descent.value * scale)) + top = _f32(y.value + _f32(ascent.value * scale)) + if pp.raw.FPDFText_IsGenerated(self._text, self._index): + return x.value, bottom, x.value, top + # The width lookup fails for characters without a reverse unicode + # mapping, in which case the new left and right bounds are close enough. + if pp.raw.FPDFFont_GetGlyphWidth(font, self.unicode, size, width) and width.value: + left, right = x.value, _f32(x.value + _f32(matrix.a * width.value)) + return left, bottom, right, top + + def generated_space(self) -> "Character": + """ + :return: A zero-sized space character located at the origin of this + character, like the ones pdfium generates for visual gaps. + """ + space = copy.copy(self) + space.__dict__ = {k: v for k, v in self.__dict__.items() if k.startswith("_") or k.endswith("link")} + space.unicode = 0x20 + space._bbox = space.tbbox = Rectangle(self.origin.x, self.origin.y, self.origin.x, self.origin.y) + space.origin = self.origin + return space + def _font_flags(self) -> tuple[str, int]: if self._font is None: font = ctypes.create_string_buffer(255) @@ -129,7 +180,9 @@ def theight(self) -> float: @cached_property def render_mode(self) -> RenderMode: """The render mode of the character.""" - return Character.RenderMode(pp.raw.FPDFText_GetTextRenderMode(self._text, self._index)) + return Character.RenderMode( + pp.raw.FPDFTextObj_GetTextRenderMode(pp.raw.FPDFText_GetTextObject(self._text, self._index)) + ) @cached_property def rotation(self) -> int: diff --git a/src/modm_data/pdf/document.py b/src/modm_data/pdf/document.py index a5b725b..66aed78 100644 --- a/src/modm_data/pdf/document.py +++ b/src/modm_data/pdf/document.py @@ -4,7 +4,7 @@ import ctypes import logging import pypdfium2 as pp -from typing import Iterator, Iterable +from typing import Iterator, Iterable, NamedTuple from pathlib import Path from functools import cached_property, cache from collections import defaultdict @@ -13,8 +13,11 @@ _LOGGER = logging.getLogger(__name__) -# We cannot monkey patch this class, since it's a named tuple. :-( -class _OutlineItem(pp.PdfOutlineItem): +class _OutlineItem(NamedTuple): + level: int + title: str + page_index: int + def __hash__(self) -> int: return hash(f"{self.page_index}+{self.title}") @@ -61,16 +64,16 @@ def metadata(self) -> dict[str, str]: def destinations(self) -> Iterator[tuple[int, str]]: """Yields (page 0-index, named destination) of the whole document.""" for ii in range(pp.raw.FPDF_CountNamedDests(self)): - length = pp.raw.FPDF_GetNamedDest(self, ii, 0, 0) - clength = ctypes.c_long(length) - cbuffer = ctypes.create_string_buffer(length * 2) + clength = ctypes.c_long() + pp.raw.FPDF_GetNamedDest(self, ii, None, clength) + cbuffer = ctypes.create_string_buffer(clength.value) dest = pp.raw.FPDF_GetNamedDest(self, ii, cbuffer, clength) - name = cbuffer.raw[: clength.value * 2].decode("utf-16-le").rstrip("\x00") + name = cbuffer.raw[: clength.value].decode("utf-16-le").rstrip("\x00") page = pp.raw.FPDFDest_GetDestPageIndex(self, dest) yield (page, name) @cached_property - def toc(self) -> list[pp.PdfOutlineItem]: + def toc(self) -> list[_OutlineItem]: """ The table of content as a sorted list of outline items ensuring item has a page index by reusing the last one. @@ -79,17 +82,9 @@ def toc(self) -> list[pp.PdfOutlineItem]: # Sometimes the TOC contains duplicates so we must use a set last_page_index = 0 for toc in self.get_toc(): - outline = _OutlineItem( - toc.level, - toc.title, - toc.is_closed, - toc.n_kids, - toc.page_index or last_page_index, - toc.view_mode, - toc.view_pos, - ) - last_page_index = toc.page_index or last_page_index - tocs.add(outline) + dest = toc.get_dest() + last_page_index = (dest and dest.get_index()) or last_page_index + tocs.add(_OutlineItem(toc.level, toc.get_title(), last_page_index)) return list(sorted(list(tocs), key=lambda o: (o.page_index, o.level, o.title))) @cached_property diff --git a/src/modm_data/pdf/image.py b/src/modm_data/pdf/image.py index a01b24d..1e814f9 100644 --- a/src/modm_data/pdf/image.py +++ b/src/modm_data/pdf/image.py @@ -25,7 +25,7 @@ def __init__(self, obj): """ :param obj: Page object of the image. """ - super().__init__(obj.raw, obj.page, obj.pdf, obj.level) + super().__init__(obj.raw, obj.page, obj.pdf, obj.container, obj.level) assert pp.raw.FPDFPageObj_GetType(obj.raw) == pp.raw.FPDF_PAGEOBJ_IMAGE self.type = pp.raw.FPDF_PAGEOBJ_IMAGE @@ -50,7 +50,7 @@ def matrix(self) -> pp.PdfMatrix: @cached_property def bbox(self) -> Rectangle: """The bounding box of the image.""" - bbox = Rectangle(*self.get_pos()) + bbox = Rectangle(*self.get_bounds()) if self.page.rotation: bbox = Rectangle(bbox.p0.y, self.page.height - bbox.p1.x, bbox.p1.y, self.page.height - bbox.p0.x) return bbox diff --git a/src/modm_data/pdf/page.py b/src/modm_data/pdf/page.py index 1caafda..3fdcefd 100644 --- a/src/modm_data/pdf/page.py +++ b/src/modm_data/pdf/page.py @@ -38,6 +38,7 @@ def __init__(self, document: "modm_data.pdf.Document", index: int): # noqa: F82 """1-index page number.""" super().__init__(pp.raw.FPDF_LoadPage(document, index), document, document.formenv) + document._add_kid(self) self._links = None self._weblinks = None self._linked = False diff --git a/src/modm_data/pdf/path.py b/src/modm_data/pdf/path.py index fefbbc8..94f9e72 100644 --- a/src/modm_data/pdf/path.py +++ b/src/modm_data/pdf/path.py @@ -49,7 +49,7 @@ def __init__(self, obj): """ :param obj: PDF object of the path. """ - super().__init__(obj.raw, obj.page, obj.pdf, obj.level) + super().__init__(obj.raw, obj.page, obj.pdf, obj.container, obj.level) assert pp.raw.FPDFPageObj_GetType(obj.raw) == pp.raw.FPDF_PAGEOBJ_PATH self.type = pp.raw.FPDF_PAGEOBJ_PATH diff --git a/src/modm_data/pdf2html/page.py b/src/modm_data/pdf2html/page.py index b926be9..0644a9d 100644 --- a/src/modm_data/pdf2html/page.py +++ b/src/modm_data/pdf2html/page.py @@ -265,10 +265,13 @@ def sort_key(char): return char.origin.x + 1e9 return char.origin.x + chars = sorted(line.chars, key=sort_key) + if not line.rotation: + chars = self._chars_with_bullet_space(chars) sorted_lines.append( CharLine( self, - sorted(line.chars, key=sort_key), + chars, line.bottom, line.origin, line.top, @@ -281,6 +284,22 @@ def sort_key(char): return sorted_lines + @staticmethod + def _chars_with_bullet_space(chars: list) -> list: + # Newer pdfium does not generate a space for every visual gap anymore, + # but the list detection relies on the one between bullet and text. + for ii, (bullet, char) in enumerate(zip(chars, chars[1:]), 1): + if bullet.unicode == 0x20: + continue + if ( + bullet.char in {"•", "–"} + and char.unicode not in {0x20, 0xA, 0xD} + and char.origin.x - bullet.bbox.right >= 0.25 * bullet.height + ): + return chars[:ii] + [char.generated_space()] + chars[ii:] + break + return chars + def graphic_bboxes_in_area( self, area: Rectangle, with_graphics: bool = True ) -> list[tuple[Rectangle, Table | Figure | None]]: diff --git a/tools/requirements.txt b/tools/requirements.txt index 2fc9eea..452e23c 100644 --- a/tools/requirements.txt +++ b/tools/requirements.txt @@ -6,6 +6,7 @@ charset-normalizer==3.4.4 click==8.3.1 colorama==0.4.6 CppHeaderParser==2.7.4 +cxxheaderparser==1.9.2 ghp-import==2.1.0 idna==3.11 Jinja2==3.1.6 @@ -26,7 +27,7 @@ platformdirs==4.5.1 ply==3.11 Pygments==2.19.2 pymdown-extensions==10.20 -pypdfium2==4.30.0 +pypdfium2==5.14.0 python-dateutil==2.9.0.post0 PyYAML==6.0.3 pyyaml_env_tag==1.1