diff --git a/src/tokenizer/tokenizer.py b/src/tokenizer/tokenizer.py index 0e7f71f..84b0e95 100644 --- a/src/tokenizer/tokenizer.py +++ b/src/tokenizer/tokenizer.py @@ -2992,6 +2992,28 @@ def parse_date_and_time(token_stream: Iterator[Tok]) -> Iterator[Tok]: # Eat the year token next_token = next(token_stream) + # Check for a [month name]-[month name] range fused into + # a single word token, such as 'ágúst-september': split it + # into DATEREL, hyphen and DATEREL. The guard is deliberately + # cheap: only WORD tokens containing a hyphen or an en dash + # are examined any further. + if token.kind == TOK.WORD and ( + HYPHEN in token.txt or EN_DASH in token.txt + ): + hchar = HYPHEN if HYPHEN in token.txt else EN_DASH + a = token.txt.split(hchar) + if ( + len(a) == 2 + and (a0 := a[0]) not in AMBIGUOUS_MONTH_NAMES + and (a1 := a[1]) not in AMBIGUOUS_MONTH_NAMES + ): + m1 = MONTHS.get(a0.lower()) + m2 = MONTHS.get(a1.lower()) + if m1 is not None and m2 is not None: + yield TOK.Daterel(a0, y=0, m=m1, d=0) + yield TOK.Punctuation(hchar) + token = TOK.Daterel(a1, y=0, m=m2, d=0) + # Check for a single month, change to DATEREL if token.kind == TOK.WORD: month = month_for_token(token) diff --git a/test/test_tokenizer.py b/test/test_tokenizer.py index fafb36b..d3bd911 100644 --- a/test/test_tokenizer.py +++ b/test/test_tokenizer.py @@ -130,6 +130,25 @@ def test_single_tokens() -> None: ("3. maí", [Tok(TOK.DATEREL, "3. maí", (0, 5, 3))]), ("Ágúst", TOK.WORD), # Not month name if capitalized ("13. ágúst", [Tok(TOK.DATEREL, "13. ágúst", (0, 8, 13))]), + ( + "ágúst-september", + [ + Tok(TOK.DATEREL, "ágúst", (0, 8, 0)), + Tok(TOK.PUNCTUATION, "-", None), + Tok(TOK.DATEREL, "september", (0, 9, 0)), + ], + ), + ( + "júlí–ágúst", + [ + Tok(TOK.DATEREL, "júlí", (0, 7, 0)), + Tok(TOK.PUNCTUATION, "–", None), + Tok(TOK.DATEREL, "ágúst", (0, 8, 0)), + ], + ), + ("mar-apr", TOK.WORD), # Ambiguous month names: left alone + ("Ágúst-september", TOK.WORD), # Blacklisted capitalized Ágúst + ("vestur-þýska", TOK.WORD), # Ordinary hyphenated word ("nóvember 1918", [Tok(TOK.DATEREL, "nóvember 1918", (1918, 11, 0))]), ( "nóvember 19180",