Keep original text and origin spans on month range tokens

src/tokenizer/tokenizer.py

@@ -3010,9 +3010,13 @@ def parse_date_and_time(token_stream: Iterator[Tok]) -> Iterator[Tok]:
                     m1 = MONTHS.get(a0.lower())
                     m2 = MONTHS.get(a1.lower())
                     if m1 is not None and m2 is not None:
-                        yield TOK.Daterel(a0, y=0, m=m1, d=0)
-                        yield TOK.Punctuation(hchar)
-                        token = TOK.Daterel(a1, y=0, m=m2, d=0)
+                        # Split the token itself, so that the parts keep
+                        # their original text and origin spans
+                        first, rest = token.split(len(a0))
+                        hyphen, second = rest.split(1)
+                        yield TOK.Daterel(first, y=0, m=m1, d=0)
+                        yield TOK.Punctuation(hyphen)
+                        token = TOK.Daterel(second, y=0, m=m2, d=0)
 
             # Check for a single month, change to DATEREL
             if token.kind == TOK.WORD:

test/test_tokenizer.py

@@ -2446,6 +2446,28 @@ def test_abbr_at_eos() -> None:
     assert len([tok for tok in toklist if tok.kind == TOK.S_END]) == 2
 
 
+def test_month_range_original() -> None:
+    """Test that a month range keeps the original text of its parts"""
+    text = "Hér komum við mánaðamótin mars-apríl og júlí–ágúst."
+    toklist = list(t.tokenize(text, original=True))
+    assert "".join(tok.original or "" for tok in toklist) == text
+    rng = [(tok.kind, tok.txt, tok.original) for tok in toklist[5:8]]
+    assert rng == [
+        (TOK.DATEREL, "mars", " mars"),
+        (TOK.PUNCTUATION, "-", "-"),
+        (TOK.DATEREL, "apríl", "apríl"),
+    ]
+    rng = [(tok.kind, tok.txt, tok.original) for tok in toklist[9:12]]
+    assert rng == [
+        (TOK.DATEREL, "júlí", " júlí"),
+        (TOK.PUNCTUATION, "–", "–"),
+        (TOK.DATEREL, "ágúst", "ágúst"),
+    ]
+    for tok in toklist[5:12]:
+        assert tok.original is not None
+        assert len(tok.origin_spans) == len(tok.txt)
+
+
 def test_time_token() -> None:
     toklist = list(
         t.tokenize("2.55pm - Síðasta tilraun setur Knights fram klukkan hálf")