From cfce5aba9576e822f55de9519a53cd1045deab8d Mon Sep 17 00:00:00 2001 From: David Straub Date: Wed, 12 Aug 2026 16:26:11 +0200 Subject: [PATCH 1/3] Add write support --- README.md | 76 ++------------- gedcom7/__init__.py | 12 ++- gedcom7/exceptions.py | 4 + gedcom7/serializer.py | 163 +++++++++++++++++++++++++++++++ gedcom7/types.py | 5 +- test/test_serializer.py | 211 ++++++++++++++++++++++++++++++++++++++++ 6 files changed, 402 insertions(+), 69 deletions(-) create mode 100644 gedcom7/serializer.py create mode 100644 test/test_serializer.py diff --git a/README.md b/README.md index abd0def..2d71567 100644 --- a/README.md +++ b/README.md @@ -5,19 +5,11 @@ [![Python versions](https://img.shields.io/pypi/pyversions/gedcom7)](https://pypi.org/project/gedcom7/) [![License: MIT](https://img.shields.io/badge/license-MIT-blue)](LICENSE) -A [GEDCOM 7](https://gedcom.io/) parser for Python. +A [GEDCOM 7](https://gedcom.io/) parser and serializer for Python. ## Background -The parser is based on regular expressions generated directly from the ABNF -grammar via [`abnf-to-regexp`](https://github.com/aas-core-works/abnf-to-regexp), -and on the structure and payload tables extracted from the specification. It -targets FamilySearch GEDCOM -[7.0.18](https://github.com/FamilySearch/GEDCOM/blob/main/specification/). - -It does not attempt to parse files that are not standards compliant: a data -stream that violates the specification raises `GedcomParseError` rather than -being partially or silently misparsed. +The parser is based on regular expressions generated directly from the ABNF grammar via [`abnf-to-regexp`](https://github.com/aas-core-works/abnf-to-regexp), and on the structure and payload tables extracted from the specification. It targets FamilySearch GEDCOM [7.0.18](https://github.com/FamilySearch/GEDCOM/blob/main/specification/) and does not attempt to parse files that are not standards compliant: a data stream that violates the specification raises `GedcomParseError` rather than being silently misparsed. ## Installation @@ -30,73 +22,25 @@ python -m pip install gedcom7 ```python import gedcom7 -with open("my_gedcom.ged", "r", encoding="utf-8") as f: - string = f.read() - -records = gedcom7.loads(string) -``` - -Each record is a `GedcomStructure` with a `tag`, an optional `xref` and -`pointer`, the raw `text` payload, and `children`. The `value` property casts the -payload to the data type the specification gives that structure type: +with open("my_gedcom.ged", encoding="utf-8") as f: + records = gedcom7.loads(f.read()) -```python -indi = records[1] # 0 @I1@ INDI -birt = indi.children[0] # 1 BIRT -date = birt.children[0] # 2 DATE 1 JAN 2000 -date.type_id # 'https://gedcom.io/terms/v7/DATE' -date.value # Date(calendar=None, day=1, month='JAN', ...) +with open("out.ged", "w", encoding="utf-8") as f: + gedcom7.dump(records, f) ``` -`value` is `None` where a structure has no payload, and the payload is returned -uninterpreted where the structure type is defined by an extension rather than by -the specification. - -### Errors - -`loads` raises `GedcomParseError` (a subclass of `ValueError`) for any data -stream that does not conform to the specification, reporting where the problem -is: - -```python -from gedcom7 import GedcomParseError - -try: - records = gedcom7.loads(string) -except GedcomParseError as exc: - print(exc) # line 42: malformed line: '1 NOTE @invalid' - print(exc.line_number) # 42 - print(exc.line) # "1 NOTE @invalid" -``` +Each record is a `GedcomStructure` with a `tag`, an optional `xref` and `pointer`, the raw `text` payload, and `children`. Its `value` property casts the payload to the data type the specification gives that structure type, and `type_id` is the structure type URI, or `None` where the type is defined by an extension. -This covers malformed lines, prohibited level sequences, banned characters, -misplaced `CONT` continuations, duplicate or misplaced cross-reference -identifiers, pointers that resolve to nothing, and a missing header or trailer. +Serializing is the exact inverse of parsing, so `dumps(loads(text)) == text` holds byte for byte for a conforming file. `GedcomParseError` and `GedcomSerializeError` both subclass `ValueError`; parse errors carry `line_number` and `line`. ## Development -Install the package together with the development dependencies: - ``` python -m pip install --group dev --editable . +pytest && mypy && ruff check . ``` -Then run the checks: - -``` -pytest # tests -mypy # static type check (strict) -ruff check . # lint -ruff format . # format -``` - -## Releasing - -The version is derived from git tags by -[setuptools-scm](https://setuptools-scm.readthedocs.io/); there is no version -string in the source tree. To release, push a `vX.Y.Z` tag and publish a GitHub -release for it — the `PyPI deploy` workflow builds the distributions and -uploads them via PyPI trusted publishing. +The version is derived from git tags by [setuptools-scm](https://setuptools-scm.readthedocs.io/). To release, push a `vX.Y.Z` tag and publish a GitHub release for it. ## Credits diff --git a/gedcom7/__init__.py b/gedcom7/__init__.py index 4f719ee..805d12c 100644 --- a/gedcom7/__init__.py +++ b/gedcom7/__init__.py @@ -2,10 +2,18 @@ from importlib.metadata import PackageNotFoundError, version -from .exceptions import GedcomError, GedcomParseError +from .exceptions import GedcomError, GedcomParseError, GedcomSerializeError from .parser import loads +from .serializer import dump, dumps -__all__ = ["GedcomError", "GedcomParseError", "loads"] +__all__ = [ + "GedcomError", + "GedcomParseError", + "GedcomSerializeError", + "dump", + "dumps", + "loads", +] try: __version__ = version("gedcom7") diff --git a/gedcom7/exceptions.py b/gedcom7/exceptions.py index 1811ed9..7602849 100644 --- a/gedcom7/exceptions.py +++ b/gedcom7/exceptions.py @@ -7,6 +7,10 @@ class GedcomError(Exception): """Base class for all errors raised by this package.""" +class GedcomSerializeError(GedcomError, ValueError): + """Raised when structures cannot be encoded as a conforming data stream.""" + + class GedcomParseError(GedcomError, ValueError): """Raised when a data stream does not conform to the GEDCOM 7 grammar. diff --git a/gedcom7/serializer.py b/gedcom7/serializer.py new file mode 100644 index 0000000..c6ef5c1 --- /dev/null +++ b/gedcom7/serializer.py @@ -0,0 +1,163 @@ +"""GEDCOM 7 serializer.""" + +from __future__ import annotations + +import re +from typing import TYPE_CHECKING + +from . import const, grammar +from .exceptions import GedcomSerializeError + +if TYPE_CHECKING: + from collections.abc import Iterable + from typing import IO + + from .types import GedcomStructure + +_EOL = re.compile(r"\r\n|\r|\n") +_TAG = re.compile(grammar.tag) +_XREF = re.compile(grammar.xref) +_POINTER = re.compile(grammar.pointer) +_BANNED = re.compile(grammar.banned) +_TAGDEF = re.compile(grammar.tagdef) + +_BOM = "" + + +def _escape(linestr: str) -> str: + """Escape a line string's leading "@" by doubling it. + + The inverse of the parser's unescaping: only a leading ``@`` is doubled, so + ``@@@@`` is written as ``@@@@@``. + """ + return "@" + linestr if linestr.startswith("@") else linestr + + +def _schema(records: Iterable[GedcomStructure]) -> dict[str, str]: + """Map each URI declared by the header schema back to its extension tag.""" + uris: dict[str, str] = {} + for record in records: + if record.tag != const.HEAD: + continue + for schema in record.children: + if schema.tag != const.SCHMA: + continue + for definition in schema.children: + if definition.tag != const.TAG: + continue + match = _TAGDEF.fullmatch(definition.text) + # A schema should map only one tag to each URI; keep the first. + if match is not None: + uris.setdefault(match.group("uri"), match.group("exttag")) + return uris + + +def _lines( + structure: GedcomStructure, level: int, uris: dict[str, str] +) -> Iterable[str]: + """Yield the lines encoding a structure and everything below it.""" + tag = uris.get(structure.tag, structure.tag) + if _TAG.fullmatch(tag) is None: + hint = ( + " Add a matching HEAD.SCHMA.TAG definition to abbreviate it." + if "://" in tag + else "" + ) + raise GedcomSerializeError(f"{tag!r} is not a valid tag.{hint}") + + parts = [str(level)] + if structure.xref is not None and structure.xref != "": + if level != 0: + raise GedcomSerializeError( + f"only records may have a cross-reference identifier, but {tag} " + f"at level {level} has {structure.xref}" + ) + if _XREF.fullmatch(structure.xref) is None or structure.xref == const.VOIDPTR: + raise GedcomSerializeError( + f"{structure.xref!r} is not a valid cross-reference identifier" + ) + parts.append(structure.xref) + parts.append(tag) + + if structure.pointer is not None and structure.pointer != "": + if structure.text: + raise GedcomSerializeError( + f"{tag} has both a pointer and a text payload; a line value is " + "one or the other" + ) + if _POINTER.fullmatch(structure.pointer) is None: + raise GedcomSerializeError(f"{structure.pointer!r} is not a valid pointer") + parts.append(structure.pointer) + yield " ".join(parts) + else: + # A payload containing line terminators is split across the structure's + # own line and one CONT pseudo-structure per subsequent line. + payload = _EOL.split(structure.text) + if payload[0]: + parts.append(_escape(payload[0])) + yield " ".join(parts) + for continuation in payload[1:]: + yield ( + f"{level + 1} {const.CONT} {_escape(continuation)}" + if continuation + else f"{level + 1} {const.CONT}" + ) + + for child in structure.children: + yield from _lines(child, level + 1, uris) + + +def dumps( + records: Iterable[GedcomStructure], + *, + line_terminator: str = "\n", + byte_order_mark: bool = True, +) -> str: + """Serialize structures to a GEDCOM 7 data stream. + + Levels are derived from the structure tree, payloads containing line + terminators are split into CONT pseudo-structures, and a leading "@" in a + payload is escaped. Extension tags stored as URIs are abbreviated using the + tag definitions in the header schema. + + The specification says a data stream should begin with U+FEFF, so a byte + order mark is included by default; pass ``byte_order_mark=False`` to omit it. + Write the result with ``encoding="utf-8"``, not ``"utf-8-sig"``. + + Raises :class:`~gedcom7.exceptions.GedcomSerializeError` if the structures + cannot be encoded as conforming lines. + """ + if line_terminator not in ("\n", "\r\n", "\r"): + raise GedcomSerializeError( + f"{line_terminator!r} is not a valid line terminator; " + "use '\\n', '\\r\\n' or '\\r'" + ) + + records = list(records) + uris = _schema(records) + lines = [line for record in records for line in _lines(record, 0, uris)] + + string = "".join(line + line_terminator for line in lines) + banned = _BANNED.search(string) + if banned: + raise GedcomSerializeError( + f"banned character U+{ord(banned.group()):04X} in payload" + ) + return (_BOM if byte_order_mark else "") + string + + +def dump( + records: Iterable[GedcomStructure], + fp: IO[str], + *, + line_terminator: str = "\n", + byte_order_mark: bool = True, +) -> None: + """Serialize structures to a text file opened with ``encoding="utf-8"``.""" + fp.write( + dumps( + records, + line_terminator=line_terminator, + byte_order_mark=byte_order_mark, + ) + ) diff --git a/gedcom7/types.py b/gedcom7/types.py index 5b7275d..c0c11da 100644 --- a/gedcom7/types.py +++ b/gedcom7/types.py @@ -18,7 +18,10 @@ class GedcomStructure: text: str xref: str | None children: list[GedcomStructure] = field(default_factory=list) - parent: GedcomStructure | None = None + # Excluded from comparison and repr: it points back up the tree, so including + # it would make __eq__ recurse endlessly and __repr__ print every ancestor. + # A structure's superstructure is implied by its position in the tree. + parent: GedcomStructure | None = field(default=None, compare=False, repr=False) @property def type_id(self) -> str | None: diff --git a/test/test_serializer.py b/test/test_serializer.py new file mode 100644 index 0000000..81942d4 --- /dev/null +++ b/test/test_serializer.py @@ -0,0 +1,211 @@ +"""Tests for serializing structures back to a GEDCOM 7 data stream.""" + +import io +import pathlib + +import pytest + +import gedcom7 +from gedcom7 import GedcomSerializeError, types + +HEAD = "0 HEAD\n1 GEDC\n2 VERS 7.0\n" +TRLR = "0 TRLR\n" + + +def roundtrip(text: str) -> str: + """Parse and re-serialize, without a byte order mark.""" + return gedcom7.dumps(gedcom7.loads(text), byte_order_mark=False) + + +# -------------------------------------------------------------------------- +# Round-tripping +# -------------------------------------------------------------------------- + + +def test_maximal_roundtrips_byte_for_byte() -> None: + """The official maximal70.ged must survive a parse and re-serialize exactly.""" + filename = pathlib.Path(__file__).parent / "data" / "maximal70.ged" + original = filename.read_text(encoding="utf-8") + assert gedcom7.dumps(gedcom7.loads(original)) == original + + +def test_maximal_roundtrips_as_a_tree() -> None: + """Re-parsing the serialized form yields an equal structure tree.""" + filename = pathlib.Path(__file__).parent / "data" / "maximal70.ged" + records = gedcom7.loads(filename.read_text(encoding="utf-8")) + assert gedcom7.loads(gedcom7.dumps(records)) == records + + +@pytest.mark.parametrize( + "body", + [ + "0 @I1@ INDI\n1 SEX M\n", + "0 @I1@ INDI\n1 BIRT\n2 DATE 1 JAN 2000\n3 TIME 14:30\n", + "0 @I1@ INDI\n1 ALIA @VOID@\n", + "0 @I1@ INDI\n1 NOTE line one\n2 CONT line two\n2 CONT\n2 CONT line four\n", + "0 @I1@ INDI\n1 NOTE @@leading at\n", + "0 @I1@ INDI\n1 NOTE leading space\n", + "0 @I1@ INDI\n1 NOTE trailing space \n", + "0 @N1@ SNOTE shared note\n1 LANG en\n", + ], +) +def test_body_roundtrips(body: str) -> None: + """Serializing is the exact inverse of parsing.""" + text = HEAD + body + TRLR + assert roundtrip(text) == text + + +def test_extension_tag_roundtrips_through_the_schema() -> None: + """A tag stored as a URI is abbreviated using the header schema.""" + text = ( + "0 HEAD\n1 SCHMA\n2 TAG _FOO http://example.com/foo\n1 GEDC\n2 VERS 7.0\n" + "0 @I1@ INDI\n1 _FOO 23\n" + TRLR + ) + records = gedcom7.loads(text) + # the parser resolved the tag to its URI + assert records[1].children[0].tag == "http://example.com/foo" + assert roundtrip(text) == text + + +# -------------------------------------------------------------------------- +# Encoding details +# -------------------------------------------------------------------------- + + +def test_byte_order_mark_included_by_default() -> None: + """The specification says a data stream should begin with U+FEFF.""" + out = gedcom7.dumps(gedcom7.loads(HEAD + TRLR)) + assert out.startswith("") + assert gedcom7.dumps(gedcom7.loads(HEAD + TRLR), byte_order_mark=False) == ( + HEAD + TRLR + ) + + +@pytest.mark.parametrize("eol", ["\n", "\r\n", "\r"]) +def test_line_terminator(eol: str) -> None: + """Any of the three permitted terminators may be written.""" + records = gedcom7.loads(HEAD + "0 @I1@ INDI\n1 SEX M\n" + TRLR) + out = gedcom7.dumps(records, line_terminator=eol, byte_order_mark=False) + assert out == (HEAD + "0 @I1@ INDI\n1 SEX M\n" + TRLR).replace("\n", eol) + # and the result is still readable + assert gedcom7.loads(out) == records + + +def test_invalid_line_terminator_rejected() -> None: + """Only CR, LF and CR-LF are line terminators.""" + with pytest.raises(GedcomSerializeError, match="not a valid line terminator"): + gedcom7.dumps(gedcom7.loads(HEAD + TRLR), line_terminator="\n\n") + + +def test_multiline_payload_is_split_into_cont() -> None: + """A payload containing line terminators becomes CONT continuations.""" + records = gedcom7.loads(HEAD + "0 @I1@ INDI\n1 NOTE a\n" + TRLR) + records[1].children[0].text = "a\nb\n\nc" + out = gedcom7.dumps(records, byte_order_mark=False) + assert "1 NOTE a\n2 CONT b\n2 CONT\n2 CONT c\n" in out + assert gedcom7.loads(out)[1].children[0].text == "a\nb\n\nc" + + +def test_empty_payload_writes_no_line_value() -> None: + """Empty and missing payloads are both written with no trailing space.""" + records = gedcom7.loads(HEAD + "0 @O1@ OBJE\n1 FILE\n2 FORM image/jpeg\n" + TRLR) + out = gedcom7.dumps(records, byte_order_mark=False) + assert "\n1 FILE\n" in out + + +def test_leading_at_is_escaped() -> None: + """A leading "@" must be doubled; later ones must not.""" + records = gedcom7.loads(HEAD + "0 @I1@ INDI\n1 NOTE x\n" + TRLR) + note = records[1].children[0] + + note.text = "@me and @I" + assert "1 NOTE @@me and @I\n" in gedcom7.dumps(records, byte_order_mark=False) + + note.text = "@@@@" + out = gedcom7.dumps(records, byte_order_mark=False) + assert "1 NOTE @@@@@\n" in out + assert gedcom7.loads(out)[1].children[0].text == "@@@@" + + note.text = "a@b" + assert "1 NOTE a@b\n" in gedcom7.dumps(records, byte_order_mark=False) + + +def test_dump_writes_to_a_file_object() -> None: + """dump() mirrors dumps() but writes to a stream.""" + text = HEAD + "0 @I1@ INDI\n1 SEX M\n" + TRLR + records = gedcom7.loads(text) + buffer = io.StringIO() + gedcom7.dump(records, buffer, byte_order_mark=False) + assert buffer.getvalue() == text + + +def test_levels_derive_from_the_tree() -> None: + """Levels come from the structure tree, not from any stored value.""" + head = types.GedcomStructure(tag="HEAD", pointer=None, text="", xref=None) + gedc = types.GedcomStructure(tag="GEDC", pointer=None, text="", xref=None) + vers = types.GedcomStructure(tag="VERS", pointer=None, text="7.0", xref=None) + head.append_child(gedc) + gedc.append_child(vers) + trlr = types.GedcomStructure(tag="TRLR", pointer=None, text="", xref=None) + assert gedcom7.dumps([head, trlr], byte_order_mark=False) == HEAD + TRLR + + +# -------------------------------------------------------------------------- +# Rejected input +# -------------------------------------------------------------------------- + + +def test_unabbreviated_uri_tag_rejected() -> None: + """A URI tag with no schema definition cannot be written as a tag.""" + records = gedcom7.loads(HEAD + "0 @I1@ INDI\n1 SEX M\n" + TRLR) + records[1].children[0].tag = "http://example.com/unknown" + with pytest.raises(GedcomSerializeError, match="HEAD.SCHMA.TAG"): + gedcom7.dumps(records) + + +def test_invalid_tag_rejected() -> None: + """Tags must match stdTag or extTag.""" + records = gedcom7.loads(HEAD + "0 @I1@ INDI\n1 SEX M\n" + TRLR) + records[1].children[0].tag = "lower" + with pytest.raises(GedcomSerializeError, match="not a valid tag"): + gedcom7.dumps(records) + + +def test_xref_on_substructure_rejected() -> None: + """Only records may carry a cross-reference identifier.""" + records = gedcom7.loads(HEAD + "0 @I1@ INDI\n1 SEX M\n" + TRLR) + records[1].children[0].xref = "@X1@" + with pytest.raises(GedcomSerializeError, match="only records may have"): + gedcom7.dumps(records) + + +def test_voidptr_as_xref_rejected() -> None: + """@VOID@ is not a cross-reference identifier.""" + records = gedcom7.loads(HEAD + "0 @I1@ INDI\n1 SEX M\n" + TRLR) + records[1].xref = "@VOID@" + with pytest.raises(GedcomSerializeError, match="not a valid cross-reference"): + gedcom7.dumps(records) + + +def test_pointer_and_text_together_rejected() -> None: + """A line value is a pointer or a line string, not both.""" + records = gedcom7.loads(HEAD + "0 @I1@ INDI\n1 ALIA @I1@\n" + TRLR) + records[1].children[0].text = "oops" + with pytest.raises(GedcomSerializeError, match="both a pointer and a text"): + gedcom7.dumps(records) + + +def test_invalid_pointer_rejected() -> None: + """Pointers must match the pointer production.""" + records = gedcom7.loads(HEAD + "0 @I1@ INDI\n1 ALIA @I1@\n" + TRLR) + records[1].children[0].pointer = "@not a pointer@" + with pytest.raises(GedcomSerializeError, match="not a valid pointer"): + gedcom7.dumps(records) + + +def test_banned_character_in_payload_rejected() -> None: + """Banned characters must not be written into a data stream.""" + records = gedcom7.loads(HEAD + "0 @I1@ INDI\n1 NOTE fine\n" + TRLR) + records[1].children[0].text = "bad\x7fchar" + with pytest.raises(GedcomSerializeError, match="banned character"): + gedcom7.dumps(records) From dc4dc9035fe13852eb25fc716a31e1af1e43a09c Mon Sep 17 00:00:00 2001 From: David Straub Date: Wed, 12 Aug 2026 21:38:00 +0200 Subject: [PATCH 2/3] Add load and dump --- README.md | 12 +++---- gedcom7/__init__.py | 3 +- gedcom7/parser.py | 23 +++++++++++++ gedcom7/serializer.py | 29 +++++++++++------ test/test_serializer.py | 71 ++++++++++++++++++++++++++++++++++++++--- 5 files changed, 117 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 2d71567..2d988e2 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ A [GEDCOM 7](https://gedcom.io/) parser and serializer for Python. ## Background -The parser is based on regular expressions generated directly from the ABNF grammar via [`abnf-to-regexp`](https://github.com/aas-core-works/abnf-to-regexp), and on the structure and payload tables extracted from the specification. It targets FamilySearch GEDCOM [7.0.18](https://github.com/FamilySearch/GEDCOM/blob/main/specification/) and does not attempt to parse files that are not standards compliant: a data stream that violates the specification raises `GedcomParseError` rather than being silently misparsed. +The parser is based on regular expressions generated directly from the ABNF grammar via [`abnf-to-regexp`](https://github.com/aas-core-works/abnf-to-regexp), and on the structure and payload tables extracted from the specification. It targets FamilySearch GEDCOM [7.0.18](https://github.com/FamilySearch/GEDCOM/blob/main/specification/) and does not attempt to parse files that are not standards compliant. ## Installation @@ -22,16 +22,16 @@ python -m pip install gedcom7 ```python import gedcom7 -with open("my_gedcom.ged", encoding="utf-8") as f: - records = gedcom7.loads(f.read()) +with open("my_gedcom.ged", "rb") as f: + records = gedcom7.load(f) -with open("out.ged", "w", encoding="utf-8") as f: +with open("out.ged", "wb") as f: gedcom7.dump(records, f) ``` -Each record is a `GedcomStructure` with a `tag`, an optional `xref` and `pointer`, the raw `text` payload, and `children`. Its `value` property casts the payload to the data type the specification gives that structure type, and `type_id` is the structure type URI, or `None` where the type is defined by an extension. +Each record is a `GedcomStructure` with a `tag`, an optional `xref` and `pointer`, the raw `text` payload, and `children`. Its `value` property casts the payload to the data type the specification gives that structure type. -Serializing is the exact inverse of parsing, so `dumps(loads(text)) == text` holds byte for byte for a conforming file. `GedcomParseError` and `GedcomSerializeError` both subclass `ValueError`; parse errors carry `line_number` and `line`. +`loads` and `dumps` are the string equivalents. Non-conforming input raises `GedcomParseError`, a `ValueError` carrying `line_number`. ## Development diff --git a/gedcom7/__init__.py b/gedcom7/__init__.py index 805d12c..572ae07 100644 --- a/gedcom7/__init__.py +++ b/gedcom7/__init__.py @@ -3,7 +3,7 @@ from importlib.metadata import PackageNotFoundError, version from .exceptions import GedcomError, GedcomParseError, GedcomSerializeError -from .parser import loads +from .parser import load, loads from .serializer import dump, dumps __all__ = [ @@ -12,6 +12,7 @@ "GedcomSerializeError", "dump", "dumps", + "load", "loads", ] diff --git a/gedcom7/parser.py b/gedcom7/parser.py index 767a423..d6106bc 100644 --- a/gedcom7/parser.py +++ b/gedcom7/parser.py @@ -3,11 +3,15 @@ from __future__ import annotations import re +from typing import TYPE_CHECKING from . import const, grammar from .exceptions import GedcomParseError from .types import GedcomStructure +if TYPE_CHECKING: + from typing import BinaryIO + # EOL = %x0D [%x0A] / %x0A -- CR-LF, CR, or LF _EOL = re.compile(r"\r\n|\r|\n") _LINE = re.compile(grammar.line) @@ -28,6 +32,25 @@ def _unescape(linestr: str) -> str: return linestr[1:] if linestr.startswith("@@") else linestr +def load(fp: BinaryIO) -> list[GedcomStructure]: + """Load a GEDCOM 7 dataset from a binary file object. + + The file must be opened in binary mode, e.g. ``open(path, "rb")``. GEDCOM 7 + data streams are always UTF-8, and reading the bytes directly avoids the + encoding guesswork and newline translation that text mode would apply. + """ + data = fp.read() + try: + string = data.decode("utf-8") + except AttributeError: + raise TypeError( + 'File must be opened in binary mode, e.g. use `open("my.ged", "rb")`' + ) from None + except UnicodeDecodeError as exc: + raise GedcomParseError(f"data stream is not valid UTF-8: {exc}") from exc + return loads(string) + + def loads(string: str) -> list[GedcomStructure]: """Load a GEDCOM 7 dataset from a string. diff --git a/gedcom7/serializer.py b/gedcom7/serializer.py index c6ef5c1..9ed6716 100644 --- a/gedcom7/serializer.py +++ b/gedcom7/serializer.py @@ -10,7 +10,7 @@ if TYPE_CHECKING: from collections.abc import Iterable - from typing import IO + from typing import BinaryIO from .types import GedcomStructure @@ -148,16 +148,25 @@ def dumps( def dump( records: Iterable[GedcomStructure], - fp: IO[str], + fp: BinaryIO, *, line_terminator: str = "\n", byte_order_mark: bool = True, ) -> None: - """Serialize structures to a text file opened with ``encoding="utf-8"``.""" - fp.write( - dumps( - records, - line_terminator=line_terminator, - byte_order_mark=byte_order_mark, - ) - ) + """Serialize structures to a binary file object. + + The file must be opened in binary mode, e.g. ``open(path, "wb")``. GEDCOM 7 + data streams are always UTF-8, and writing the bytes directly keeps text + mode from re-encoding them or rewriting the line terminators. + """ + data = dumps( + records, + line_terminator=line_terminator, + byte_order_mark=byte_order_mark, + ).encode("utf-8") + try: + fp.write(data) + except TypeError: + raise TypeError( + 'File must be opened in binary mode, e.g. use `open("my.ged", "wb")`' + ) from None diff --git a/test/test_serializer.py b/test/test_serializer.py index 81942d4..150341e 100644 --- a/test/test_serializer.py +++ b/test/test_serializer.py @@ -130,13 +130,76 @@ def test_leading_at_is_escaped() -> None: assert "1 NOTE a@b\n" in gedcom7.dumps(records, byte_order_mark=False) -def test_dump_writes_to_a_file_object() -> None: - """dump() mirrors dumps() but writes to a stream.""" +def test_dump_writes_to_a_binary_file_object() -> None: + """dump() mirrors dumps() but writes UTF-8 bytes to a stream.""" text = HEAD + "0 @I1@ INDI\n1 SEX M\n" + TRLR records = gedcom7.loads(text) - buffer = io.StringIO() + buffer = io.BytesIO() gedcom7.dump(records, buffer, byte_order_mark=False) - assert buffer.getvalue() == text + assert buffer.getvalue() == text.encode("utf-8") + + +def test_load_reads_a_binary_file_object() -> None: + """load() mirrors loads() but reads UTF-8 bytes from a stream.""" + text = HEAD + "0 @I1@ INDI\n1 SEX M\n" + TRLR + buffer = io.BytesIO(text.encode("utf-8")) + assert gedcom7.load(buffer) == gedcom7.loads(text) + + +def test_text_mode_is_rejected() -> None: + """Text streams would re-encode and rewrite terminators, so they are refused.""" + records = gedcom7.loads(HEAD + TRLR) + with pytest.raises(TypeError, match="binary mode"): + gedcom7.dump(records, io.StringIO()) # type: ignore[arg-type] + with pytest.raises(TypeError, match="binary mode"): + gedcom7.load(io.StringIO(HEAD + TRLR)) # type: ignore[arg-type] + + +def test_dump_writes_nothing_to_a_text_stream() -> None: + """The type error must arrive before any output is produced.""" + buffer = io.StringIO() + with pytest.raises(TypeError): + gedcom7.dump(gedcom7.loads(HEAD + TRLR), buffer) # type: ignore[arg-type] + assert buffer.getvalue() == "" + + +def test_invalid_utf8_rejected() -> None: + """GEDCOM 7 data streams are always UTF-8.""" + with pytest.raises(gedcom7.GedcomParseError, match="not valid UTF-8"): + gedcom7.load(io.BytesIO(b"0 HEAD\n1 NOTE \xff\xfe\n0 TRLR\n")) + + +@pytest.mark.parametrize("eol", ["\n", "\r\n", "\r"]) +def test_file_roundtrip_preserves_terminators(eol: str, tmp_path: pathlib.Path) -> None: + """Going through the filesystem must not rewrite the line terminators. + + Text mode would translate "\\n" to os.linesep on Windows, turning an + explicit "\\r\\n" into "\\r\\r\\n" and producing a file this library rejects. + """ + source = (HEAD + "0 @I1@ INDI\n1 SEX M\n" + TRLR).replace("\n", eol) + path = tmp_path / "out.ged" + + with open(path, "wb") as f: + gedcom7.dump( + gedcom7.loads(source), f, line_terminator=eol, byte_order_mark=False + ) + + assert path.read_bytes() == source.encode("utf-8") + with open(path, "rb") as f: + assert gedcom7.load(f) == gedcom7.loads(source) + + +def test_official_file_roundtrips_through_the_filesystem( + tmp_path: pathlib.Path, +) -> None: + """load() and dump() reproduce a real file byte for byte.""" + source = pathlib.Path(__file__).parent / "data" / "maximal70.ged" + path = tmp_path / "out.ged" + with open(source, "rb") as f: + records = gedcom7.load(f) + with open(path, "wb") as f: + gedcom7.dump(records, f) + assert path.read_bytes() == source.read_bytes() def test_levels_derive_from_the_tree() -> None: From 8ec2241eeabe0b99efb265955b77520fee2acf86 Mon Sep 17 00:00:00 2001 From: David Straub Date: Wed, 12 Aug 2026 21:55:53 +0200 Subject: [PATCH 3/3] Address comments --- README.md | 2 +- gedcom7/parser.py | 2 +- gedcom7/serializer.py | 5 +++-- test/test_conformance.py | 2 +- test/test_serializer.py | 2 +- 5 files changed, 7 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 2d988e2..a78a0d6 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ Each record is a `GedcomStructure` with a `tag`, an optional `xref` and `pointer ``` python -m pip install --group dev --editable . -pytest && mypy && ruff check . +pytest && mypy && ruff check . && ruff format --check . ``` The version is derived from git tags by [setuptools-scm](https://setuptools-scm.readthedocs.io/). To release, push a `vX.Y.Z` tag and publish a GitHub release for it. diff --git a/gedcom7/parser.py b/gedcom7/parser.py index d6106bc..96b99ac 100644 --- a/gedcom7/parser.py +++ b/gedcom7/parser.py @@ -19,7 +19,7 @@ _TAGDEF = re.compile(grammar.tagdef) # U+FEFF, the byte-order mark, may open a data stream and carries no meaning -_BOM = "" +_BOM = "\ufeff" def _unescape(linestr: str) -> str: diff --git a/gedcom7/serializer.py b/gedcom7/serializer.py index 9ed6716..f9218e0 100644 --- a/gedcom7/serializer.py +++ b/gedcom7/serializer.py @@ -21,7 +21,7 @@ _BANNED = re.compile(grammar.banned) _TAGDEF = re.compile(grammar.tagdef) -_BOM = "" +_BOM = "\ufeff" def _escape(linestr: str) -> str: @@ -122,7 +122,8 @@ def dumps( The specification says a data stream should begin with U+FEFF, so a byte order mark is included by default; pass ``byte_order_mark=False`` to omit it. - Write the result with ``encoding="utf-8"``, not ``"utf-8-sig"``. + Use :func:`dump` to write to a file, so that the encoding and the line + terminators are not altered on the way out. Raises :class:`~gedcom7.exceptions.GedcomSerializeError` if the structures cannot be encoded as conforming lines. diff --git a/test/test_conformance.py b/test/test_conformance.py index 1987a42..071ea17 100644 --- a/test/test_conformance.py +++ b/test/test_conformance.py @@ -36,7 +36,7 @@ def test_line_terminators(eol: str) -> None: def test_byte_order_mark_is_ignored() -> None: """The data stream should begin with U+FEFF, which carries no meaning.""" - records = gedcom7.loads("" + HEAD + TRLR) + records = gedcom7.loads("\ufeff" + HEAD + TRLR) assert records[0].tag == "HEAD" diff --git a/test/test_serializer.py b/test/test_serializer.py index 150341e..91d6e2c 100644 --- a/test/test_serializer.py +++ b/test/test_serializer.py @@ -75,7 +75,7 @@ def test_extension_tag_roundtrips_through_the_schema() -> None: def test_byte_order_mark_included_by_default() -> None: """The specification says a data stream should begin with U+FEFF.""" out = gedcom7.dumps(gedcom7.loads(HEAD + TRLR)) - assert out.startswith("") + assert out.startswith("\ufeff") assert gedcom7.dumps(gedcom7.loads(HEAD + TRLR), byte_order_mark=False) == ( HEAD + TRLR )