diff --git a/src/tokenizer/tokenizer.py b/src/tokenizer/tokenizer.py index 84b0e95..7aaba13 100644 --- a/src/tokenizer/tokenizer.py +++ b/src/tokenizer/tokenizer.py @@ -3010,9 +3010,13 @@ def parse_date_and_time(token_stream: Iterator[Tok]) -> Iterator[Tok]: m1 = MONTHS.get(a0.lower()) m2 = MONTHS.get(a1.lower()) if m1 is not None and m2 is not None: - yield TOK.Daterel(a0, y=0, m=m1, d=0) - yield TOK.Punctuation(hchar) - token = TOK.Daterel(a1, y=0, m=m2, d=0) + # Split the token itself, so that the parts keep + # their original text and origin spans + first, rest = token.split(len(a0)) + hyphen, second = rest.split(1) + yield TOK.Daterel(first, y=0, m=m1, d=0) + yield TOK.Punctuation(hyphen) + token = TOK.Daterel(second, y=0, m=m2, d=0) # Check for a single month, change to DATEREL if token.kind == TOK.WORD: diff --git a/test/test_tokenizer.py b/test/test_tokenizer.py index d3bd911..25f8da4 100644 --- a/test/test_tokenizer.py +++ b/test/test_tokenizer.py @@ -2446,6 +2446,28 @@ def test_abbr_at_eos() -> None: assert len([tok for tok in toklist if tok.kind == TOK.S_END]) == 2 +def test_month_range_original() -> None: + """Test that a month range keeps the original text of its parts""" + text = "Hér komum við mánaðamótin mars-apríl og júlí–ágúst." + toklist = list(t.tokenize(text, original=True)) + assert "".join(tok.original or "" for tok in toklist) == text + rng = [(tok.kind, tok.txt, tok.original) for tok in toklist[5:8]] + assert rng == [ + (TOK.DATEREL, "mars", " mars"), + (TOK.PUNCTUATION, "-", "-"), + (TOK.DATEREL, "apríl", "apríl"), + ] + rng = [(tok.kind, tok.txt, tok.original) for tok in toklist[9:12]] + assert rng == [ + (TOK.DATEREL, "júlí", " júlí"), + (TOK.PUNCTUATION, "–", "–"), + (TOK.DATEREL, "ágúst", "ágúst"), + ] + for tok in toklist[5:12]: + assert tok.original is not None + assert len(tok.origin_spans) == len(tok.txt) + + def test_time_token() -> None: toklist = list( t.tokenize("2.55pm - Síðasta tilraun setur Knights fram klukkan hálf")