From 79c540eb615735a71f9fe652af015492943af866 Mon Sep 17 00:00:00 2001 From: vthorsteinsson Date: Thu, 15 Oct 2026 14:59:27 +0000 Subject: [PATCH] Keep original text and origin spans on month range tokens MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Commit 14c566c (review fixes for the month range tokens of PR #63) built the DATEREL, hyphen and DATEREL tokens of e.g. 'mars-apríl' from plain strings instead of splitting the source token. The new tokens had original=None and no origin spans, so with original=True the range vanished from the reconstructed text and its character offsets were lost. Split the token again, keeping the walrus refactor. The regression is unreleased (3.6.4 predates PR #63). With the fix, the original text of test/toktest_large.txt is reconstructed exactly from its tokens, except for the final newline: trailing whitespace at the end of the input belongs to no token, as in 3.6.4. Co-Authored-By: Claude Opus 5.5 --- src/tokenizer/tokenizer.py | 10 +++++++--- test/test_tokenizer.py | 22 ++++++++++++++++++++++ 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/src/tokenizer/tokenizer.py b/src/tokenizer/tokenizer.py index 84b0e95..7aaba13 100644 --- a/src/tokenizer/tokenizer.py +++ b/src/tokenizer/tokenizer.py @@ -3010,9 +3010,13 @@ def parse_date_and_time(token_stream: Iterator[Tok]) -> Iterator[Tok]: m1 = MONTHS.get(a0.lower()) m2 = MONTHS.get(a1.lower()) if m1 is not None and m2 is not None: - yield TOK.Daterel(a0, y=0, m=m1, d=0) - yield TOK.Punctuation(hchar) - token = TOK.Daterel(a1, y=0, m=m2, d=0) + # Split the token itself, so that the parts keep + # their original text and origin spans + first, rest = token.split(len(a0)) + hyphen, second = rest.split(1) + yield TOK.Daterel(first, y=0, m=m1, d=0) + yield TOK.Punctuation(hyphen) + token = TOK.Daterel(second, y=0, m=m2, d=0) # Check for a single month, change to DATEREL if token.kind == TOK.WORD: diff --git a/test/test_tokenizer.py b/test/test_tokenizer.py index d3bd911..25f8da4 100644 --- a/test/test_tokenizer.py +++ b/test/test_tokenizer.py @@ -2446,6 +2446,28 @@ def test_abbr_at_eos() -> None: assert len([tok for tok in toklist if tok.kind == TOK.S_END]) == 2 +def test_month_range_original() -> None: + """Test that a month range keeps the original text of its parts""" + text = "Hér komum við mánaðamótin mars-apríl og júlí–ágúst." + toklist = list(t.tokenize(text, original=True)) + assert "".join(tok.original or "" for tok in toklist) == text + rng = [(tok.kind, tok.txt, tok.original) for tok in toklist[5:8]] + assert rng == [ + (TOK.DATEREL, "mars", " mars"), + (TOK.PUNCTUATION, "-", "-"), + (TOK.DATEREL, "apríl", "apríl"), + ] + rng = [(tok.kind, tok.txt, tok.original) for tok in toklist[9:12]] + assert rng == [ + (TOK.DATEREL, "júlí", " júlí"), + (TOK.PUNCTUATION, "–", "–"), + (TOK.DATEREL, "ágúst", "ágúst"), + ] + for tok in toklist[5:12]: + assert tok.original is not None + assert len(tok.origin_spans) == len(tok.txt) + + def test_time_token() -> None: toklist = list( t.tokenize("2.55pm - Síðasta tilraun setur Knights fram klukkan hálf")