Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 22 additions & 0 deletions src/tokenizer/tokenizer.py
Original file line number Diff line number Diff line change
Expand Up @@ -2992,6 +2992,28 @@ def parse_date_and_time(token_stream: Iterator[Tok]) -> Iterator[Tok]:
# Eat the year token
next_token = next(token_stream)

# Check for a [month name]-[month name] range fused into
# a single word token, such as 'ágúst-september': split it
# into DATEREL, hyphen and DATEREL. The guard is deliberately
# cheap: only WORD tokens containing a hyphen or an en dash
# are examined any further.
if token.kind == TOK.WORD and (
HYPHEN in token.txt or EN_DASH in token.txt
):
hchar = HYPHEN if HYPHEN in token.txt else EN_DASH
a = token.txt.split(hchar)
if (
len(a) == 2
and (a0 := a[0]) not in AMBIGUOUS_MONTH_NAMES
and (a1 := a[1]) not in AMBIGUOUS_MONTH_NAMES
):
m1 = MONTHS.get(a0.lower())
m2 = MONTHS.get(a1.lower())
if m1 is not None and m2 is not None:
yield TOK.Daterel(a0, y=0, m=m1, d=0)
yield TOK.Punctuation(hchar)
token = TOK.Daterel(a1, y=0, m=m2, d=0)

# Check for a single month, change to DATEREL
if token.kind == TOK.WORD:
month = month_for_token(token)
Expand Down
19 changes: 19 additions & 0 deletions test/test_tokenizer.py
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,25 @@ def test_single_tokens() -> None:
("3. maí", [Tok(TOK.DATEREL, "3. maí", (0, 5, 3))]),
("Ágúst", TOK.WORD), # Not month name if capitalized
("13. ágúst", [Tok(TOK.DATEREL, "13. ágúst", (0, 8, 13))]),
(
"ágúst-september",
[
Tok(TOK.DATEREL, "ágúst", (0, 8, 0)),
Tok(TOK.PUNCTUATION, "-", None),
Tok(TOK.DATEREL, "september", (0, 9, 0)),
],
),
(
"júlí–ágúst",
[
Tok(TOK.DATEREL, "júlí", (0, 7, 0)),
Tok(TOK.PUNCTUATION, "–", None),
Tok(TOK.DATEREL, "ágúst", (0, 8, 0)),
],
),
("mar-apr", TOK.WORD), # Ambiguous month names: left alone
("Ágúst-september", TOK.WORD), # Blacklisted capitalized Ágúst
("vestur-þýska", TOK.WORD), # Ordinary hyphenated word
("nóvember 1918", [Tok(TOK.DATEREL, "nóvember 1918", (1918, 11, 0))]),
(
"nóvember 19180",
Expand Down
Loading