From 34d7ca6e41670be30277b2e03d18e66025de85d5 Mon Sep 17 00:00:00 2001 From: Vilhjalmur Thorsteinsson Date: Wed, 2 Sep 2026 18:08:36 +0000 Subject: [PATCH 1/2] Recognize month ranges fused into a single word MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A tight month-month range such as "ágúst-september" (common in phrases like "um mánaðamótin ágúst-september") was glued into a single WORD token by hyphenated-compound handling before date recognition could see it, so downstream parsers never received the month names. parse_date_and_time() now splits such a word into DATEREL, hyphen and DATEREL, for both the ASCII hyphen and the en dash. The check is guarded cheaply: only WORD tokens containing a hyphen character are examined further, and the month lookup only runs on a two-part split whose halves are not ambiguous month names ("mar-apr" and capitalized "Ágúst-..." are left alone, consistent with single-month behavior). Co-Authored-By: Claude Fable 5 --- src/tokenizer/tokenizer.py | 24 ++++++++++++++++++++++++ test/test_tokenizer.py | 19 +++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/src/tokenizer/tokenizer.py b/src/tokenizer/tokenizer.py index 0e7f71f..2613693 100644 --- a/src/tokenizer/tokenizer.py +++ b/src/tokenizer/tokenizer.py @@ -2992,6 +2992,30 @@ def parse_date_and_time(token_stream: Iterator[Tok]) -> Iterator[Tok]: # Eat the year token next_token = next(token_stream) + # Check for a [month name]-[month name] range fused into + # a single word token, such as 'ágúst-september': split it + # into DATEREL, hyphen and DATEREL. The guard is deliberately + # cheap: only WORD tokens containing a hyphen or an en dash + # are examined any further. + if token.kind == TOK.WORD and ( + HYPHEN in token.txt or EN_DASH in token.txt + ): + hchar = HYPHEN if HYPHEN in token.txt else EN_DASH + a = token.txt.split(hchar) + if ( + len(a) == 2 + and a[0] not in AMBIGUOUS_MONTH_NAMES + and a[1] not in AMBIGUOUS_MONTH_NAMES + ): + m1 = MONTHS.get(a[0].lower()) + m2 = MONTHS.get(a[1].lower()) + if m1 is not None and m2 is not None: + first, rest = token.split(len(a[0])) + hyphen, second = rest.split(1) + yield TOK.Daterel(first, y=0, m=m1, d=0) + yield TOK.Punctuation(hyphen) + token = TOK.Daterel(second, y=0, m=m2, d=0) + # Check for a single month, change to DATEREL if token.kind == TOK.WORD: month = month_for_token(token) diff --git a/test/test_tokenizer.py b/test/test_tokenizer.py index fafb36b..d3bd911 100644 --- a/test/test_tokenizer.py +++ b/test/test_tokenizer.py @@ -130,6 +130,25 @@ def test_single_tokens() -> None: ("3. maí", [Tok(TOK.DATEREL, "3. maí", (0, 5, 3))]), ("Ágúst", TOK.WORD), # Not month name if capitalized ("13. ágúst", [Tok(TOK.DATEREL, "13. ágúst", (0, 8, 13))]), + ( + "ágúst-september", + [ + Tok(TOK.DATEREL, "ágúst", (0, 8, 0)), + Tok(TOK.PUNCTUATION, "-", None), + Tok(TOK.DATEREL, "september", (0, 9, 0)), + ], + ), + ( + "júlí–ágúst", + [ + Tok(TOK.DATEREL, "júlí", (0, 7, 0)), + Tok(TOK.PUNCTUATION, "–", None), + Tok(TOK.DATEREL, "ágúst", (0, 8, 0)), + ], + ), + ("mar-apr", TOK.WORD), # Ambiguous month names: left alone + ("Ágúst-september", TOK.WORD), # Blacklisted capitalized Ágúst + ("vestur-þýska", TOK.WORD), # Ordinary hyphenated word ("nóvember 1918", [Tok(TOK.DATEREL, "nóvember 1918", (1918, 11, 0))]), ( "nóvember 19180", From 14c566cc7b221454409c8ecf0846b8a9c82207dd Mon Sep 17 00:00:00 2001 From: Vilhjalmur Thorsteinsson Date: Wed, 2 Sep 2026 18:22:25 +0000 Subject: [PATCH 2/2] Address review comments Capture the two halves with walrus assignments, reuse them for the month lookups, and construct the emitted tokens directly from the strings instead of slicing the original token. Co-Authored-By: Claude Fable 5 --- src/tokenizer/tokenizer.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/src/tokenizer/tokenizer.py b/src/tokenizer/tokenizer.py index 2613693..84b0e95 100644 --- a/src/tokenizer/tokenizer.py +++ b/src/tokenizer/tokenizer.py @@ -3004,17 +3004,15 @@ def parse_date_and_time(token_stream: Iterator[Tok]) -> Iterator[Tok]: a = token.txt.split(hchar) if ( len(a) == 2 - and a[0] not in AMBIGUOUS_MONTH_NAMES - and a[1] not in AMBIGUOUS_MONTH_NAMES + and (a0 := a[0]) not in AMBIGUOUS_MONTH_NAMES + and (a1 := a[1]) not in AMBIGUOUS_MONTH_NAMES ): - m1 = MONTHS.get(a[0].lower()) - m2 = MONTHS.get(a[1].lower()) + m1 = MONTHS.get(a0.lower()) + m2 = MONTHS.get(a1.lower()) if m1 is not None and m2 is not None: - first, rest = token.split(len(a[0])) - hyphen, second = rest.split(1) - yield TOK.Daterel(first, y=0, m=m1, d=0) - yield TOK.Punctuation(hyphen) - token = TOK.Daterel(second, y=0, m=m2, d=0) + yield TOK.Daterel(a0, y=0, m=m1, d=0) + yield TOK.Punctuation(hchar) + token = TOK.Daterel(a1, y=0, m=m2, d=0) # Check for a single month, change to DATEREL if token.kind == TOK.WORD: