diff --git a/HOW_TO_ADD_NEW_CAPABILITY.md b/HOW_TO_ADD_NEW_CAPABILITY.md index 4203de05..9f499747 100644 --- a/HOW_TO_ADD_NEW_CAPABILITY.md +++ b/HOW_TO_ADD_NEW_CAPABILITY.md @@ -260,14 +260,14 @@ Recognize by syntactic *shape*: fixed widths, delimiters, or character classes. Choose Regex when the representation has a distinctive, enumerable shape. All Date, Email, IP, ISBN, and Phone grammars, plus Country's `alpha2`/`alpha3`/`numeric` grammars, are Regex grammars. Three recurring sub-patterns: - **Compile once, iterate with `finditer()`** — never compile inside `recognize()` (it runs for every input). -- **Sanitize the matched token** — the notation value is a *cleaned* raw token, never a canonical value. Phone strips separators (`strip_separators` in `Phone/grammar/common.py`), Country uppercases, ISBN strips separators and guards length. `raw_text` is always the original span, so `len(raw_text) == end - start` keeps holding. +- **Sanitize the matched token** — the notation value is a *cleaned* raw token, never a canonical value. Phone strips separators (a local `strip_separators` in each Phone grammar), Country uppercases, ISBN strips separators and guards length. `raw_text` is always the original span, so `len(raw_text) == end - start` keeps holding. - **Guard boundaries against sibling grammars** — when two grammars could claim the same span, use lookbehind/lookahead so each claims only its own representation. See Phone's `national_recognition` (four stacked lookbehinds rejecting `+1…` and `tel:+…` prefixes) and `e164_recognition` (a `(? list[str]: return [name for name, active in grammar_rules.items() if active] ``` -**Benefit:** Adding a grammar = one dict entry, no method edit needed. +**Benefit:** Adding a grammar requires one dictionary entry in this property. --- @@ -173,7 +173,7 @@ All four architectural deepening candidates have been implemented: ### Architecture Layers -``` +```text paxman/core/ Domain objects, protocols, discovery paxman/capabilities/ Capability implementations (Email, Date, Country) paxman/engine/ Pipeline orchestrator @@ -182,7 +182,7 @@ paxman/api/ Public API entry points ### Import Boundaries (ADR-0001) -``` +```text paxman.core → (no imports from capabilities, engine, api) paxman.capabilities → (can import from core) paxman.engine → (can import from core, capabilities) @@ -191,7 +191,7 @@ paxman.api → (can import from everything) ### Hot Spots (Recent Commits) -``` +```text 05732ae refactor(core): address code review findings for Email capability 1a69f7e fix: quality gate fixes (ruff, pyright, import-linter, formatting) 5de47b7 feat(capabilities): add Email capability exports @@ -203,7 +203,7 @@ d0a6f2f test(integration): add ambiguity detection tests ### Test Structure -``` +```text tests/unit/ Domain object immutability and protocol compliance tests/capabilities/ Grammar recognition and rule normalization tests/integration/ Full pipeline flow, ambiguity, temporal filtering diff --git a/docs/development/report/recognition-handling-library-research.md b/docs/development/reports/recognition-handling-library-research.md similarity index 92% rename from docs/development/report/recognition-handling-library-research.md rename to docs/development/reports/recognition-handling-library-research.md index a91bbda8..3a7acf36 100644 --- a/docs/development/report/recognition-handling-library-research.md +++ b/docs/development/reports/recognition-handling-library-research.md @@ -170,12 +170,15 @@ from `ValidationError` (itself a `ValueError`): `InvalidFormat`, `InvalidChecksu normalization happens (separator stripping, Unicode folding, case folding). This is exactly the "grammars emit raw tokens; one shared syntax seam" shape Paxman's F3 resolution already gestured at with - `paxman/capabilities/Country/name_normalization.py`. + `paxman/capabilities/Country/notation.py:normalize_name` (lookup-key + normalization, distinct from the emitted notation which preserves the original + trimmed case). - **Validation/formatting separation (F4).** `validate()` returns the canonical - minimal representation; `format()` is purely presentation and is per-module - parameterized (`separator`, `add_check_digit`). This matches the - already-shipped `Capability.format_value()` seam: rules emit a default - canonical form, formatting is a separate seam. + minimal representation; `format()` is purely presentation only for separator + changes (`separator` parameter), while `add_check_digit=True` appends a Luhn + check digit to a 14-digit IMEI and therefore is not purely presentation. + This matches the already-shipped `Capability.format_value()` seam: rules emit + a default canonical form, formatting is a separate seam. ### What it does not solve @@ -453,7 +456,7 @@ return all trees, (3) visit the SPPF yourself | Paxman invariant | stdnum | phonenumbers | Lark | |------------------|--------|--------------|------| | Deterministic / replay-safe | Yes (pure functions) | Yes (pure functions) | Yes, but `dynamic_complete` cost is input-dependent | -| No guessing | Yes (`validate` is all-or-nothing) | **No** -- matcher picks best match per span | Default is best-derivation (**no**); `ambiguity='explicit'` is **yes** | +| No guessing | Yes (`validate` is all-or-nothing) | **No** -- matcher picks first candidate accepted by the configured leniency per span (POSSIBLE may accept possible-but-invalid numbers; VALID is default) | Default is best-derivation (**no**); `ambiguity='explicit'` is **yes** | | AMBIGUOUS preserved | N/A (no ambiguity model) | **No** -- collapses to one match | **Yes** -- retains all derivations (but syntactic) | | Provenance-first | No provenance concept | Implicit metadata only | No provenance concept | | No-raise rule policy | **No** -- raises typed exceptions | Returns bools | N/A | @@ -473,11 +476,14 @@ candidates for `01/02/2026`. Adopt a **span-first recognition contract** and a **shared syntax-normalization seam**, regularized by an engine-enforced precedence order. Concretely: -1. **Carry source spans in `RecognizedRep`** (add `start: int`, `end: int`, - `raw_text: str`, mirroring `PhoneNumberMatch`). Keep `Grammar.recognize()` - returning `list[NotationT]` but make the span available at the seam the - engine already owns (`RecognizedRep` construction). Addresses Tier 2 #2 and - #3; without spans neither uniform dedup nor uniform ordering is possible. +1. **Carry source spans via `RecognitionMatch`** — `Grammar.recognize()` + produces span-bearing `RecognitionMatch` values (`start`, `end`, `raw_text` + alongside `notation`, mirroring `PhoneNumberMatch`), and `RecognizedRep` + preserves `start`, `end`, and `raw_text` alongside the notation. Do not let + recognition reduce results to bare `NotationT` values before this metadata + reaches the engine, so uniform deduplication and document ordering remain + possible. Addresses Tier 2 #2 and #3; without spans neither uniform dedup nor + uniform ordering is possible. 2. **Unify dedup as per-grammar span-overlap, engine-enforced.** Replace the three divergent mechanisms (part-keyed, address-keyed, span-overlap) with one rule: within a single grammar's output, drop a recognition whose span is @@ -493,10 +499,18 @@ seam**, regularized by an engine-enforced precedence order. Concretely: construction. 4. **Standardize the recognition level at "raw tokens + shared syntax seam".** Grammars emit raw recognized tokens; a capability-level syntax-normalization - helper (the stdnum `clean()`/`compact()` idea, already partially realized as - `Country/name_normalization.py`) does case folding, Unicode cleanup, and - separator stripping; semantic mapping stays in rules (Tier 2 #1; reinforces - the F3 resolution). Optionally expose a phonenumbers-style lenient/authoritative + helper (the stdnum `clean()`/`compact()` idea) does case folding, Unicode + cleanup, and separator stripping — preserved in the notation layer as + `paxman/capabilities/Country/notation.py:normalize_name` (lookup-key + transform, distinct from the emitted notation which preserves the original + trimmed case), and in grammar layers as separator stripping in + `paxman/capabilities/Phone/grammar/_common.py:strip_separators` and digit + extraction in `paxman/capabilities/ISBN/grammar/*_recognition.py`; semantic + mapping stays in rules (Tier 2 #1; reinforces the F3 resolution). Country's + `normalize_name` remains in the notation layer (not moved into grammar + code); Phone/ISBN follow the same principle with syntax normalization in + grammar layers and semantic mapping in rules. + Optionally expose a phonenumbers-style lenient/authoritative two-tier validation as a *diagnostic* (reason codes), not a behavior change. 5. **Keep the existing `format_value` seam and the no-raise rule policy.** Validation/formatting separation is already shipped (F4, 2026-08-04) and diff --git a/paxman/capabilities/AGENTS.md b/paxman/capabilities/AGENTS.md index e0ce8a1c..9da41312 100644 --- a/paxman/capabilities/AGENTS.md +++ b/paxman/capabilities/AGENTS.md @@ -57,7 +57,6 @@ Every capability must conform to the same structural surface. `CapabilityContrac - **Quality gates before merge** — `ruff check`, `ruff format --check`, `pyright` (strict), `import-linter lint`, `pytest` (95% coverage). ## ANTI-PATTERNS & LEGACY EXCEPTIONS -- Don't imitate the one-off modules: `Country/name_normalization.py` and `Phone/grammar/common.py` predate the sanctioned strategies — not patterns to copy. - Don't force a representation into a regex that fights it — consult HOW_TO's recognition-strategy section (scanner, format-candidate, parser combinators, Unicode-property, automaton) before choosing. - Don't invert the two-locus gating model (e.g., gating recognition on authority features or gating validation on input-shape features) — it produces the wrong `Resolution` statuses. - Don't add `slots=True` to contracts. diff --git a/paxman/capabilities/Country/capability.py b/paxman/capabilities/Country/capability.py index 15fb9a3e..74d19a94 100644 --- a/paxman/capabilities/Country/capability.py +++ b/paxman/capabilities/Country/capability.py @@ -9,8 +9,7 @@ from paxman.capabilities.Country.grammar.alpha3_recognition import Alpha3Grammar from paxman.capabilities.Country.grammar.name_recognition import NameGrammar from paxman.capabilities.Country.grammar.numeric_recognition import NumericGrammar -from paxman.capabilities.Country.name_normalization import normalize_name -from paxman.capabilities.Country.notation import CountryNotation +from paxman.capabilities.Country.notation import CountryNotation, normalize_name from paxman.capabilities.Country.rules.cldr_localized_ed2025 import ( SectionLocalizedNames, ) diff --git a/paxman/capabilities/Country/grammar/alpha2_recognition.py b/paxman/capabilities/Country/grammar/alpha2_recognition.py index 2e5acce4..c97a88cf 100644 --- a/paxman/capabilities/Country/grammar/alpha2_recognition.py +++ b/paxman/capabilities/Country/grammar/alpha2_recognition.py @@ -1,16 +1,28 @@ -"""Alpha-2 country code recognition grammar.""" +"""Alpha-2 country code recognition grammar (staged pipeline). + +Recognizes exactly 2 ASCII letters as an alpha-2 country code shape. The word +boundary is supplied by BoundaryGuard.word_only() (ADR-0008 D5) so no +hard-coded lookaround literal remains in this file. Syntax only: the grammar +never resolves the code to a country. +""" from __future__ import annotations import re from paxman.capabilities.Country.notation import CountryNotation -from paxman.core.domain import Grammar, RecognitionMatch +from paxman.core.grammar import BoundaryGuard, PipelineGrammar, RegexStage, StandardPre + +_GUARD = BoundaryGuard.word_only() +_ALPHA2_PATTERN = _GUARD.lookbehind + r"[A-Za-z]{2}" + _GUARD.lookahead + -_ALPHA2_PATTERN = re.compile(r"\b[A-Za-z]{2}\b") +def _alpha2_notation(match: re.Match[str]) -> CountryNotation: + """Map an alpha-2 match to its upper-cased notation.""" + return CountryNotation(shape="alpha2", value=match.group(0).upper()) -class Alpha2Grammar(Grammar[CountryNotation]): +class Alpha2Grammar(PipelineGrammar[CountryNotation]): """Recognizes exactly 2 ASCII letters as alpha-2 country code shape. Examples: "US", "GB", "us", "gB" @@ -21,27 +33,7 @@ class Alpha2Grammar(Grammar[CountryNotation]): semantics = "alpha2_recognition" single_value = True - def recognize(self, text: str) -> list[RecognitionMatch[CountryNotation]]: - """Extract alpha-2 patterns from text. - - Args: - text: Raw input text. - - Returns: - List of span-bearing matches with shape="alpha2" notations. - """ - if not text.strip(): - return [] - matches: list[RecognitionMatch[CountryNotation]] = [] - for match in _ALPHA2_PATTERN.finditer(text): - matches.append( - RecognitionMatch( - notation=CountryNotation( - shape="alpha2", value=match.group(0).upper() - ), - start=match.start(), - end=match.end(), - raw_text=match.group(0), - ) - ) - return matches + pre = StandardPre[CountryNotation](empty_guard=True) + regex = RegexStage[CountryNotation]( + pattern=_ALPHA2_PATTERN, notation_fn=_alpha2_notation + ) diff --git a/paxman/capabilities/Country/grammar/alpha3_recognition.py b/paxman/capabilities/Country/grammar/alpha3_recognition.py index cdad16e7..9af93bab 100644 --- a/paxman/capabilities/Country/grammar/alpha3_recognition.py +++ b/paxman/capabilities/Country/grammar/alpha3_recognition.py @@ -1,16 +1,28 @@ -"""Alpha-3 country code recognition grammar.""" +"""Alpha-3 country code recognition grammar (staged pipeline). + +Recognizes exactly 3 ASCII letters as an alpha-3 country code shape. The word +boundary is supplied by BoundaryGuard.word_only() (ADR-0008 D5) so no +hard-coded lookaround literal remains in this file. Syntax only: the grammar +never resolves the code to a country. +""" from __future__ import annotations import re from paxman.capabilities.Country.notation import CountryNotation -from paxman.core.domain import Grammar, RecognitionMatch +from paxman.core.grammar import BoundaryGuard, PipelineGrammar, RegexStage, StandardPre + +_GUARD = BoundaryGuard.word_only() +_ALPHA3_PATTERN = _GUARD.lookbehind + r"[A-Za-z]{3}" + _GUARD.lookahead + -_ALPHA3_PATTERN = re.compile(r"\b[A-Za-z]{3}\b") +def _alpha3_notation(match: re.Match[str]) -> CountryNotation: + """Map an alpha-3 match to its upper-cased notation.""" + return CountryNotation(shape="alpha3", value=match.group(0).upper()) -class Alpha3Grammar(Grammar[CountryNotation]): +class Alpha3Grammar(PipelineGrammar[CountryNotation]): """Recognizes exactly 3 ASCII letters as alpha-3 country code shape. Examples: "USA", "GBR", "usa", "gbr" @@ -21,27 +33,7 @@ class Alpha3Grammar(Grammar[CountryNotation]): semantics = "alpha3_recognition" single_value = True - def recognize(self, text: str) -> list[RecognitionMatch[CountryNotation]]: - """Extract alpha-3 patterns from text. - - Args: - text: Raw input text. - - Returns: - List of span-bearing matches with shape="alpha3" notations. - """ - if not text.strip(): - return [] - matches: list[RecognitionMatch[CountryNotation]] = [] - for match in _ALPHA3_PATTERN.finditer(text): - matches.append( - RecognitionMatch( - notation=CountryNotation( - shape="alpha3", value=match.group(0).upper() - ), - start=match.start(), - end=match.end(), - raw_text=match.group(0), - ) - ) - return matches + pre = StandardPre[CountryNotation](empty_guard=True) + regex = RegexStage[CountryNotation]( + pattern=_ALPHA3_PATTERN, notation_fn=_alpha3_notation + ) diff --git a/paxman/capabilities/Country/grammar/data/chinese_names.py b/paxman/capabilities/Country/grammar/data/chinese_names.py index 46ac2d86..5b0a8b71 100644 --- a/paxman/capabilities/Country/grammar/data/chinese_names.py +++ b/paxman/capabilities/Country/grammar/data/chinese_names.py @@ -11,7 +11,7 @@ from __future__ import annotations -from paxman.capabilities.Country.name_normalization import normalize_name +from paxman.capabilities.Country.notation import normalize_name CHINESE_NAME_KEYS: frozenset[str] = frozenset( normalize_name(key) diff --git a/paxman/capabilities/Country/grammar/data/english_names.py b/paxman/capabilities/Country/grammar/data/english_names.py index 37f95a0b..bd21e83b 100644 --- a/paxman/capabilities/Country/grammar/data/english_names.py +++ b/paxman/capabilities/Country/grammar/data/english_names.py @@ -11,7 +11,7 @@ from __future__ import annotations -from paxman.capabilities.Country.name_normalization import normalize_name +from paxman.capabilities.Country.notation import normalize_name ENGLISH_NAME_KEYS: frozenset[str] = frozenset( normalize_name(key) diff --git a/paxman/capabilities/Country/grammar/data/historical_names.py b/paxman/capabilities/Country/grammar/data/historical_names.py index 8b3e7578..475e4595 100644 --- a/paxman/capabilities/Country/grammar/data/historical_names.py +++ b/paxman/capabilities/Country/grammar/data/historical_names.py @@ -11,7 +11,7 @@ from __future__ import annotations -from paxman.capabilities.Country.name_normalization import normalize_name +from paxman.capabilities.Country.notation import normalize_name HISTORICAL_NAME_KEYS: frozenset[str] = frozenset( normalize_name(key) diff --git a/paxman/capabilities/Country/grammar/data/localized_names.py b/paxman/capabilities/Country/grammar/data/localized_names.py index c00eccd9..af477755 100644 --- a/paxman/capabilities/Country/grammar/data/localized_names.py +++ b/paxman/capabilities/Country/grammar/data/localized_names.py @@ -18,7 +18,7 @@ from __future__ import annotations -from paxman.capabilities.Country.name_normalization import normalize_name +from paxman.capabilities.Country.notation import normalize_name LOCALIZED_NAME_KEYS: frozenset[str] = frozenset( normalize_name(key) diff --git a/paxman/capabilities/Country/grammar/name_recognition.py b/paxman/capabilities/Country/grammar/name_recognition.py index 28462406..e252c052 100644 --- a/paxman/capabilities/Country/grammar/name_recognition.py +++ b/paxman/capabilities/Country/grammar/name_recognition.py @@ -20,17 +20,16 @@ from paxman.capabilities.Country.grammar.data.localized_names import ( LOCALIZED_NAME_KEYS, ) -from paxman.capabilities.Country.name_normalization import normalize_name -from paxman.capabilities.Country.notation import CountryNotation -from paxman.core.domain import Grammar, RecognitionMatch +from paxman.capabilities.Country.notation import CountryNotation, normalize_name +from paxman.core.grammar import PipelineGrammar, StandardPre, WholeInputLookup # Union of every recognized name representation across locales. -_KNOWN_NAME_KEYS = ( +_KNOWN_NAME_KEYS = frozenset( ENGLISH_NAME_KEYS | HISTORICAL_NAME_KEYS | CHINESE_NAME_KEYS | LOCALIZED_NAME_KEYS ) -class NameGrammar(Grammar[CountryNotation]): +class NameGrammar(PipelineGrammar[CountryNotation]): """Recognizes country name representations from recognition key sets. Decides whether an input is a known country name representation and @@ -52,32 +51,9 @@ class NameGrammar(Grammar[CountryNotation]): semantics = "name_recognition" single_value = True - def recognize(self, text: str) -> list[RecognitionMatch[CountryNotation]]: - """Extract a country name representation from text. - - Args: - text: Raw input text. - - Returns: - A list with a single span-bearing match of shape="name" carrying - the trimmed input token when the token is a known name - representation, or an empty list for empty/unknown input. - """ - trimmed = text.strip() - if not trimmed: - return [] - - normalized = normalize_name(trimmed) - - if normalized in _KNOWN_NAME_KEYS: - start = len(text) - len(text.lstrip()) - return [ - RecognitionMatch( - notation=CountryNotation(shape="name", value=trimmed), - start=start, - end=start + len(trimmed), - raw_text=trimmed, - ) - ] - - return [] + pre = StandardPre[CountryNotation](empty_guard=True) + lexicon = WholeInputLookup[CountryNotation]( + keys=_KNOWN_NAME_KEYS, + normalizer=normalize_name, + notation_fn=lambda trimmed: CountryNotation(shape="name", value=trimmed), + ) diff --git a/paxman/capabilities/Country/grammar/numeric_recognition.py b/paxman/capabilities/Country/grammar/numeric_recognition.py index 82044ad2..39ebee29 100644 --- a/paxman/capabilities/Country/grammar/numeric_recognition.py +++ b/paxman/capabilities/Country/grammar/numeric_recognition.py @@ -1,16 +1,28 @@ -"""Numeric (M49) country code recognition grammar.""" +"""Numeric (M49) country code recognition grammar (staged pipeline). + +Recognizes 1-3 digits as a numeric country code shape. The word boundary is +supplied by BoundaryGuard.word_only() (ADR-0008 D5) so no hard-coded +lookaround literal remains in this file. Syntax only: the grammar never +resolves the code to a country. +""" from __future__ import annotations import re from paxman.capabilities.Country.notation import CountryNotation -from paxman.core.domain import Grammar, RecognitionMatch +from paxman.core.grammar import BoundaryGuard, PipelineGrammar, RegexStage, StandardPre + +_GUARD = BoundaryGuard.word_only() +_NUMERIC_PATTERN = _GUARD.lookbehind + r"\d{1,3}" + _GUARD.lookahead + -_NUMERIC_PATTERN = re.compile(r"\b\d{1,3}\b") +def _numeric_notation(match: re.Match[str]) -> CountryNotation: + """Map a numeric match to its verbatim notation.""" + return CountryNotation(shape="numeric", value=match.group(0)) -class NumericGrammar(Grammar[CountryNotation]): +class NumericGrammar(PipelineGrammar[CountryNotation]): """Recognizes 1-3 digits as numeric country code shape. Examples: "840", "4", "004" @@ -21,25 +33,7 @@ class NumericGrammar(Grammar[CountryNotation]): semantics = "numeric_recognition" single_value = True - def recognize(self, text: str) -> list[RecognitionMatch[CountryNotation]]: - """Extract numeric patterns from text. - - Args: - text: Raw input text. - - Returns: - List of span-bearing matches with shape="numeric" notations. - """ - if not text.strip(): - return [] - matches: list[RecognitionMatch[CountryNotation]] = [] - for match in _NUMERIC_PATTERN.finditer(text): - matches.append( - RecognitionMatch( - notation=CountryNotation(shape="numeric", value=match.group(0)), - start=match.start(), - end=match.end(), - raw_text=match.group(0), - ) - ) - return matches + pre = StandardPre[CountryNotation](empty_guard=True) + regex = RegexStage[CountryNotation]( + pattern=_NUMERIC_PATTERN, notation_fn=_numeric_notation + ) diff --git a/paxman/capabilities/Country/name_normalization.py b/paxman/capabilities/Country/name_normalization.py deleted file mode 100644 index 9b1a6490..00000000 --- a/paxman/capabilities/Country/name_normalization.py +++ /dev/null @@ -1,61 +0,0 @@ -"""Shared syntax-only normalization for Country name recognition. - -Country name grammars use ``normalize_name()`` to decide whether an input -is a known name representation, and Country validation rules use the same -function to look up notation values in their authority tables. Sharing one -normalizer keeps both layers in agreement on lookup keys. - -This is syntax normalization only: case folding, Unicode decomposition, -separator-to-word-boundary folding, punctuation removal, and whitespace -collapsing make no statement about what a name means. Transliteration, -fuzzy matching, synonym resolution, and canonical-value lookup are out of -scope. -""" - -from __future__ import annotations - -import re -import unicodedata - -# Compiled normalizer: keep only letters, digits, and whitespace -# (strip punctuation and other symbols). -_WHITESPACE_RE = re.compile(r"\s+") - -# Separator characters treated as word boundaries: hyphen, en dash, and -# slash. Mapping them to a space (rather than stripping them) makes -# "GUINEA-BISSAU", "GUINEA\u2013BISSAU", "GUINEA/BISSAU", and "GUINEA BISSAU" -# collapse to one lookup key while keeping punctuation removal intact. -_SEPARATOR_TO_SPACE = str.maketrans( - { - "-": " ", - "\u2013": " ", # en dash - "/": " ", - } -) - - -def normalize_name(text: str) -> str: - """Return a syntax-normalized Country name lookup key. - - NFKD-decomposes the input (splits accented chars like ô → o + combining - mark), folds hyphens, en dashes, and slashes into word boundaries, - strips combining marks and punctuation, collapses whitespace, uppercases, - and trims outer whitespace. Unicode letters (including CJK) and digits - are preserved. - - Args: - text: Raw input text. - - Returns: - Normalized key suitable for membership and lookup-table matching. - """ - # NFKD decompose: "ô" → "o" + combining circumflex - decomposed = unicodedata.normalize("NFKD", text) - # Fold separators into word boundaries so separated variants share a key - with_separators = decomposed.translate(_SEPARATOR_TO_SPACE) - # Keep only alphanumeric and whitespace (strips combining marks + punctuation) - cleaned = "".join(c for c in with_separators if c.isalnum() or c.isspace()) - # Collapse whitespace - cleaned = _WHITESPACE_RE.sub(" ", cleaned) - # Uppercase and trim - return cleaned.upper().strip() diff --git a/paxman/capabilities/Country/notation.py b/paxman/capabilities/Country/notation.py index 77d69705..42599bfe 100644 --- a/paxman/capabilities/Country/notation.py +++ b/paxman/capabilities/Country/notation.py @@ -2,8 +2,25 @@ from __future__ import annotations +import re +import unicodedata from dataclasses import dataclass +# Compiled normalizer: collapse whitespace runs. +_WHITESPACE_RE = re.compile(r"\s+") + +# Separator characters treated as word boundaries: hyphen, en dash, and +# slash. Mapping them to a space (rather than stripping them) makes +# "GUINEA-BISSAU", "GUINEA\u2013BISSAU", "GUINEA/BISSAU", and "GUINEA BISSAU" +# collapse to one lookup key while keeping punctuation removal intact. +_SEPARATOR_TO_SPACE = str.maketrans( + { + "-": " ", + "\u2013": " ", # en dash + "/": " ", + } +) + @dataclass(frozen=True, slots=True) class CountryNotation: @@ -16,3 +33,36 @@ class CountryNotation: shape: str value: str + + +def normalize_name(text: str) -> str: + """Return a syntax-normalized Country name lookup key. + + NFKD-decomposes the input (splits accented chars like ô → o + combining + mark), folds hyphens, en dashes, and slashes into word boundaries, + strips combining marks and punctuation, collapses whitespace, uppercases, + and trims outer whitespace. Unicode letters (including CJK) and digits + are preserved. + + This is syntax normalization only: case folding, Unicode decomposition, + separator-to-word-boundary folding, punctuation removal, and whitespace + collapsing make no statement about what a name means. Transliteration, + fuzzy matching, synonym resolution, and canonical-value lookup are out of + scope. + + Args: + text: Raw input text. + + Returns: + Normalized key suitable for membership and lookup-table matching. + """ + # NFKD decompose: "ô" → "o" + combining circumflex + decomposed = unicodedata.normalize("NFKD", text) + # Fold separators into word boundaries so separated variants share a key + with_separators = decomposed.translate(_SEPARATOR_TO_SPACE) + # Keep only alphanumeric and whitespace (strips combining marks + punctuation) + cleaned = "".join(c for c in with_separators if c.isalnum() or c.isspace()) + # Collapse whitespace + cleaned = _WHITESPACE_RE.sub(" ", cleaned) + # Uppercase and trim + return cleaned.upper().strip() diff --git a/paxman/capabilities/Country/rules/cldr_localized_ed2025.py b/paxman/capabilities/Country/rules/cldr_localized_ed2025.py index 5b1b977e..44f6aaab 100644 --- a/paxman/capabilities/Country/rules/cldr_localized_ed2025.py +++ b/paxman/capabilities/Country/rules/cldr_localized_ed2025.py @@ -2,8 +2,7 @@ from __future__ import annotations -from paxman.capabilities.Country.name_normalization import normalize_name -from paxman.capabilities.Country.notation import CountryNotation +from paxman.capabilities.Country.notation import CountryNotation, normalize_name from paxman.capabilities.Country.rules.data.cldr_ed2025 import LOCALIZED_TO_ALPHA2 from paxman.core.contract import Contract from paxman.core.domain import Provenance, Rule, RuleStrategy diff --git a/paxman/capabilities/Country/rules/iso_3166_ed2024.py b/paxman/capabilities/Country/rules/iso_3166_ed2024.py index 4edd79c5..7cb922a8 100644 --- a/paxman/capabilities/Country/rules/iso_3166_ed2024.py +++ b/paxman/capabilities/Country/rules/iso_3166_ed2024.py @@ -11,8 +11,7 @@ from __future__ import annotations -from paxman.capabilities.Country.name_normalization import normalize_name -from paxman.capabilities.Country.notation import CountryNotation +from paxman.capabilities.Country.notation import CountryNotation, normalize_name from paxman.capabilities.Country.rules.cldr_localized_ed2025 import ( LOCALIZED_TO_ALPHA2_NORMALIZED, ) diff --git a/paxman/capabilities/Country/rules/iso_3166_historical_ed2020.py b/paxman/capabilities/Country/rules/iso_3166_historical_ed2020.py index 2a7e4b72..03a1100a 100644 --- a/paxman/capabilities/Country/rules/iso_3166_historical_ed2020.py +++ b/paxman/capabilities/Country/rules/iso_3166_historical_ed2020.py @@ -15,8 +15,7 @@ from __future__ import annotations -from paxman.capabilities.Country.name_normalization import normalize_name -from paxman.capabilities.Country.notation import CountryNotation +from paxman.capabilities.Country.notation import CountryNotation, normalize_name from paxman.capabilities.Country.rules.data.iso_3166_ed2020_part3 import ( FORMER_ALPHA2_CODES, FORMER_NAME_TO_ALPHA2, diff --git a/paxman/capabilities/Currency/grammar/code_recognition.py b/paxman/capabilities/Currency/grammar/code_recognition.py index 8097c209..9175b73c 100644 --- a/paxman/capabilities/Currency/grammar/code_recognition.py +++ b/paxman/capabilities/Currency/grammar/code_recognition.py @@ -1,4 +1,4 @@ -"""ISO 4217 alpha-3 currency code recognition grammar. +"""ISO 4217 alpha-3 currency code recognition grammar (staged pipeline). Recognizes a standalone 3-letter ASCII code shape (case-insensitive) as one span-bearing token. Case folding is the grammar's concern (Country @@ -12,19 +12,26 @@ import re from paxman.capabilities.Currency.notation import CurrencyNotation -from paxman.core.domain import Grammar, RecognitionMatch +from paxman.core.grammar import BoundaryGuard, PipelineGrammar, RegexStage, StandardPre -# Sign characters ('-', U+2212, '+') are outside the identifier grammar; the -# boundary guards reject sign-adjacent tokens (mirrors Money's code grammar). -_CODE_PATTERN = re.compile(r"(?[A-Za-z]{3})(?![\w\-+\u2212])") +def _code_notation(match: re.Match[str]) -> CurrencyNotation: + """Fold the matched 3-letter code to uppercase at recognition.""" + return CurrencyNotation(text=match.group(0).upper(), shape="code") -class CodeRecognition(Grammar[CurrencyNotation]): + +_GUARD = BoundaryGuard.word_sign() +_CODE_PATTERN = _GUARD.lookbehind + r"[A-Za-z]{3}" + _GUARD.lookahead + + +class CodeRecognition(PipelineGrammar[CurrencyNotation]): """Recognizes standalone ISO 4217 alpha-3 code shapes. Matches a 3-letter ASCII code in any casing: "USD", "usd", "Gbp". The grammar folds the token to uppercase at recognition; the rule - validates against CURRENCY_CODES. + validates against CURRENCY_CODES. Sign characters ('-', U+2212, '+') + are outside the identifier grammar; the word_sign boundary guards + reject sign-adjacent tokens (mirrors Money's code grammar). Examples: "USD" -> text "USD", shape "code" "usd" -> text "USD", shape "code" @@ -36,28 +43,8 @@ class CodeRecognition(Grammar[CurrencyNotation]): semantics = "code_recognition" single_value = True - def recognize(self, text: str) -> list[RecognitionMatch[CurrencyNotation]]: - """Extract standalone 3-letter code tokens from text. - - Args: - text: Raw input text. - - Returns: - List of span-bearing matches with shape "code" notations. - """ - if not text.strip(): - return [] - matches: list[RecognitionMatch[CurrencyNotation]] = [] - for match in _CODE_PATTERN.finditer(text): - matches.append( - RecognitionMatch( - notation=CurrencyNotation( - text=match.group("code").upper(), - shape="code", - ), - start=match.start(), - end=match.end(), - raw_text=match.group(0), - ) - ) - return matches + pre = StandardPre[CurrencyNotation](empty_guard=True) + regex = RegexStage( + pattern=_CODE_PATTERN, + notation_fn=_code_notation, + ) diff --git a/paxman/capabilities/Currency/grammar/symbol_recognition.py b/paxman/capabilities/Currency/grammar/symbol_recognition.py index 054be738..0071250d 100644 --- a/paxman/capabilities/Currency/grammar/symbol_recognition.py +++ b/paxman/capabilities/Currency/grammar/symbol_recognition.py @@ -1,26 +1,22 @@ -"""CLDR currency symbol recognition grammar. +"""CLDR currency symbol recognition grammar (staged pipeline). Recognizes a standalone currency symbol token (qualified or bare) as one span-bearing token. The alternation is built from SYMBOL_TOKENS -(qualified-first, longest-first — D4). Syntax only: resolving the symbol -to a code is the rules' job. +(qualified-first, longest-first — D4) and guarded by word_sign boundaries; +the grammar emits the verbatim token with shape "symbol" or +"qualified_symbol". Syntax only: resolving the symbol to a code is the +rules' job. """ from __future__ import annotations -import re - from paxman.capabilities.Currency.grammar.data.currency_symbols import SYMBOL_TOKENS from paxman.capabilities.Currency.notation import CurrencyNotation -from paxman.core.domain import Grammar, RecognitionMatch - -_SYMBOL_ALTERNATION = "|".join(re.escape(token) for token in SYMBOL_TOKENS) -# Lookarounds, not \b: pure-symbol tokens ("$", "€") are non-word -# characters that \b would reject at string start, and the lookarounds -# still block matches inside a longer token. The sign block mirrors the -# Money symbol grammar. -_SYMBOL_PATTERN = re.compile( - rf"(? bool: return any(char.isascii() and char.isalpha() for char in token) -class SymbolRecognition(Grammar[CurrencyNotation]): +def _symbol_notation(token: str) -> CurrencyNotation: + """Map a matched symbol token to its qualified/bare notation.""" + return CurrencyNotation( + text=token, + shape="qualified_symbol" if _is_qualified(token) else "symbol", + ) + + +class SymbolRecognition(PipelineGrammar[CurrencyNotation]): """Recognizes standalone CLDR currency symbol tokens. A token is "qualified" when it embeds an ASCII letter ("US$", "A$", "R$") and "bare" otherwise ("$", "€", "¥"). Symbols are case-exact — - no case folding (symbols are arbitrary glyph strings). + no case folding (symbols are arbitrary glyph strings). The lexicon is + guarded by word_sign lookarounds so amount-glued ("US$5", "$500") and + inside-token ("x€") forms are rejected. Examples: "US$" -> text "US$", shape "qualified_symbol" "€" -> text "€", shape "symbol" - Non-examples: "US$5"/"$500" (amount-glued: the trailing digit is a - word character, blocked by the lookaround), "x€" (inside a - longer token). """ name = "symbol_recognition" semantics = "symbol_recognition" single_value = True - def recognize(self, text: str) -> list[RecognitionMatch[CurrencyNotation]]: - """Extract standalone symbol tokens from text. - - Args: - text: Raw input text. - - Returns: - List of span-bearing matches with shape "symbol" or - "qualified_symbol" notations. - """ - if not text.strip(): - return [] - matches: list[RecognitionMatch[CurrencyNotation]] = [] - for match in _SYMBOL_PATTERN.finditer(text): - token = match.group(0) - matches.append( - RecognitionMatch( - notation=CurrencyNotation( - text=token, - shape="qualified_symbol" if _is_qualified(token) else "symbol", - ), - start=match.start(), - end=match.end(), - raw_text=token, - ) - ) - return matches + pre = StandardPre[CurrencyNotation](empty_guard=True) + lexicon = LexiconStage( + tokens=SYMBOL_TOKENS, + boundary=BoundaryGuard.word_sign(), + longest_first=True, + notation_fn=_symbol_notation, + ) diff --git a/paxman/capabilities/Currency/grammar/word_recognition.py b/paxman/capabilities/Currency/grammar/word_recognition.py index d18cf96d..0eb65865 100644 --- a/paxman/capabilities/Currency/grammar/word_recognition.py +++ b/paxman/capabilities/Currency/grammar/word_recognition.py @@ -1,10 +1,9 @@ -"""CLDR currency display-name word recognition grammar. +"""CLDR currency display-name word recognition grammar (staged pipeline). Recognizes a standalone currency display-name word (case-insensitive) as one span-bearing token. The alternation is built from WORD_TOKENS -(longest-first). Case folding is the grammar's concern (Country/ISBN -precedent): the token is emitted lowercase so the rule is a pure -lowercase-key table lookup. Syntax only. +(longest-first) and guarded by word_sign boundaries; the token is emitted +lowercase so the rule is a pure lowercase-key table lookup. Syntax only. """ from __future__ import annotations @@ -13,22 +12,27 @@ from paxman.capabilities.Currency.grammar.data.currency_words import WORD_TOKENS from paxman.capabilities.Currency.notation import CurrencyNotation -from paxman.core.domain import Grammar, RecognitionMatch - -_WORD_ALTERNATION = "|".join(re.escape(token) for token in WORD_TOKENS) -_WORD_PATTERN = re.compile( - rf"(? CurrencyNotation: + """Fold the matched display-name word to lowercase at recognition.""" + return CurrencyNotation(text=token.lower(), shape="word") + + +class WordRecognition(PipelineGrammar[CurrencyNotation]): """Recognizes standalone CLDR currency display-name word tokens. - Matching is case-insensitive; the emitted text is the token folded to - lowercase so the rule's NAME_TO_CODES lookup is an exact lowercase-key - hit. "Euro"/"euro"/"EURO" all emit text "euro". Word boundaries keep - the match inside one token: "Dollars" does not match "Dollar". + Matching is case-insensitive (``re.IGNORECASE`` on the guarded + alternation); the emitted text is the token folded to lowercase so the + rule's NAME_TO_CODES lookup is an exact lowercase-key hit. "Euro"/ + "euro"/"EURO" all emit text "euro". Word boundaries keep the match + inside one token: "Dollars" does not match "Dollar". Examples: "Euro" -> text "euro", shape "word" "US Dollar" -> the "Dollar" span matches (text "dollar"); @@ -41,28 +45,11 @@ class WordRecognition(Grammar[CurrencyNotation]): semantics = "word_recognition" single_value = True - def recognize(self, text: str) -> list[RecognitionMatch[CurrencyNotation]]: - """Extract standalone display-name word tokens from text. - - Args: - text: Raw input text. - - Returns: - List of span-bearing matches with shape "word" notations. - """ - if not text.strip(): - return [] - matches: list[RecognitionMatch[CurrencyNotation]] = [] - for match in _WORD_PATTERN.finditer(text): - matches.append( - RecognitionMatch( - notation=CurrencyNotation( - text=match.group(0).lower(), - shape="word", - ), - start=match.start(), - end=match.end(), - raw_text=match.group(0), - ) - ) - return matches + pre = StandardPre[CurrencyNotation](empty_guard=True) + lexicon = LexiconStage( + tokens=WORD_TOKENS, + boundary=BoundaryGuard.word_sign(), + longest_first=True, + notation_fn=_word_notation, + flags=re.IGNORECASE, + ) diff --git a/paxman/capabilities/Date/grammar/european_recognition.py b/paxman/capabilities/Date/grammar/european_recognition.py index 55caa4be..1cb6f37d 100644 --- a/paxman/capabilities/Date/grammar/european_recognition.py +++ b/paxman/capabilities/Date/grammar/european_recognition.py @@ -1,6 +1,19 @@ -"""European date grammar — recognizes DD/MM/YYYY and DD/MM/YY formats. - -Notation mapping: N1=day, N2=month, N3=year +"""European date recognition grammar (staged pipeline). + +Recognizes DD/MM/YYYY and DD/MM/YY formats. The 4-digit and 2-digit year +variants are merged into one year-length alternation; the digit lookarounds +(via BoundaryGuard.digit()) keep the pattern disjoint from surrounding +digits. Notation mapping: N1=day, N2=month, N3=year. + +Note: The legacy bespoke ``recognize()`` ran two separate ``finditer`` loops +(4-digit year first, then 2-digit), so grouped matches by year length rather +than document order (e.g. ``"01/02/26 foo 01/02/2026"`` yielded +``[01/02/2026, 01/02/26]``). The staged ``(\\d{4}|\\d{2})`` alternation uses a +single ``finditer`` in document order (``[01/02/26, 01/02/2026]``). The engine +(``paxman/engine/orchestrator.py``) sorts by ``start`` before dedup, so +end-to-end ``canonicalize()`` results are identical; direct ``recognize()`` +order is now document-order. See ADR-0008 D1 and harness note in +``tests/property/test_grammar_stage_parity.py``. """ from __future__ import annotations @@ -8,13 +21,20 @@ import re from paxman.capabilities.Date.notation import DateNotation -from paxman.core.domain import Grammar, RecognitionMatch +from paxman.core.grammar import BoundaryGuard, PipelineGrammar, RegexStage, StandardPre + +_GUARD = BoundaryGuard.digit() +_EUROPEAN_PATTERN = ( + _GUARD.lookbehind + r"(\d{1,2})/(\d{1,2})/(\d{4}|\d{2})" + _GUARD.lookahead +) + -_EUROPEAN_DATE_PATTERN_4DIGIT = re.compile(r"(? DateNotation: + """Map a European date match to its day/month/year notation.""" + return DateNotation(N1=match.group(1), N2=match.group(2), N3=match.group(3)) -class EuropeanDateGrammar(Grammar[DateNotation]): +class EuropeanDateGrammar(PipelineGrammar[DateNotation]): """European date recognition: DD/MM/YYYY and DD/MM/YY. Both year-length variants carry digit lookarounds, so a date glued to @@ -28,35 +48,7 @@ class EuropeanDateGrammar(Grammar[DateNotation]): semantics = "european_calendar_date" single_value = True - def recognize(self, text: str) -> list[RecognitionMatch[DateNotation]]: - """Extract European date patterns from text. - - Emits spans for every pattern match; the engine orders by document - position and resolves overlapping 2-digit/4-digit year spans via - span dedup. - """ - results: list[RecognitionMatch[DateNotation]] = [] - - for match in _EUROPEAN_DATE_PATTERN_4DIGIT.finditer(text): - day, month, year = match.group(1), match.group(2), match.group(3) - results.append( - RecognitionMatch( - notation=DateNotation(N1=day, N2=month, N3=year), - start=match.start(), - end=match.end(), - raw_text=match.group(0), - ) - ) - - for match in _EUROPEAN_DATE_PATTERN_2DIGIT.finditer(text): - day, month, year = match.group(1), match.group(2), match.group(3) - results.append( - RecognitionMatch( - notation=DateNotation(N1=day, N2=month, N3=year), - start=match.start(), - end=match.end(), - raw_text=match.group(0), - ) - ) - - return results + pre = StandardPre[DateNotation](empty_guard=True) + regex = RegexStage[DateNotation]( + pattern=_EUROPEAN_PATTERN, notation_fn=_european_notation + ) diff --git a/paxman/capabilities/Date/grammar/iso8601_recognition.py b/paxman/capabilities/Date/grammar/iso8601_recognition.py index 73d15603..d5094f82 100644 --- a/paxman/capabilities/Date/grammar/iso8601_recognition.py +++ b/paxman/capabilities/Date/grammar/iso8601_recognition.py @@ -1,6 +1,8 @@ -"""ISO 8601 date grammar — recognizes YYYY-MM-DD format. +"""ISO 8601 date recognition grammar (staged pipeline). -Notation mapping: N1=year, N2=month, N3=day +Recognizes YYYY-MM-DD format. The digit lookarounds are supplied by +BoundaryGuard.digit() (ADR-0008 D5) so no hard-coded lookaround literal +remains in this file. Notation mapping: N1=year, N2=month, N3=day. """ from __future__ import annotations @@ -8,12 +10,18 @@ import re from paxman.capabilities.Date.notation import DateNotation -from paxman.core.domain import Grammar, RecognitionMatch +from paxman.core.grammar import BoundaryGuard, PipelineGrammar, RegexStage, StandardPre -_ISO8601_PATTERN = re.compile(r"(? DateNotation: + """Map an ISO 8601 match to its year/month/day notation.""" + return DateNotation(N1=match.group(1), N2=match.group(2), N3=match.group(3)) + + +class ISO8601DateGrammar(PipelineGrammar[DateNotation]): """ISO 8601 date recognition: YYYY-MM-DD. Digit lookarounds keep the pattern disjoint from surrounding digits, so a @@ -27,15 +35,7 @@ class ISO8601DateGrammar(Grammar[DateNotation]): semantics = "iso8601_calendar_date" single_value = True - def recognize(self, text: str) -> list[RecognitionMatch[DateNotation]]: - """Extract ISO 8601 date patterns from text.""" - return [ - RecognitionMatch( - notation=DateNotation(N1=year, N2=month, N3=day), - start=match.start(), - end=match.end(), - raw_text=match.group(0), - ) - for match in _ISO8601_PATTERN.finditer(text) - for year, month, day in [match.groups()] - ] + pre = StandardPre[DateNotation](empty_guard=True) + regex = RegexStage[DateNotation]( + pattern=_ISO8601_PATTERN, notation_fn=_iso_notation + ) diff --git a/paxman/capabilities/Date/grammar/slash_iso_recognition.py b/paxman/capabilities/Date/grammar/slash_iso_recognition.py index 07af6838..252c3ba7 100644 --- a/paxman/capabilities/Date/grammar/slash_iso_recognition.py +++ b/paxman/capabilities/Date/grammar/slash_iso_recognition.py @@ -1,6 +1,8 @@ -"""Slash-ISO date grammar — recognizes YYYY/MM/DD format. +"""Slash-ISO date recognition grammar (staged pipeline). -Notation mapping: N1=year, N2=month, N3=day +Recognizes YYYY/MM/DD format. The digit lookarounds (via BoundaryGuard.digit()) +keep the pattern disjoint from surrounding digits. Notation mapping: +N1=year, N2=month, N3=day. """ from __future__ import annotations @@ -8,12 +10,20 @@ import re from paxman.capabilities.Date.notation import DateNotation -from paxman.core.domain import Grammar, RecognitionMatch +from paxman.core.grammar import BoundaryGuard, PipelineGrammar, RegexStage, StandardPre -_SLASH_ISO_PATTERN = re.compile(r"(? DateNotation: + """Map a slash-ISO match to its year/month/day notation.""" + return DateNotation(N1=match.group(1), N2=match.group(2), N3=match.group(3)) + + +class SlashISODateGrammar(PipelineGrammar[DateNotation]): """Slash-delimited ISO date recognition: YYYY/MM/DD. Shares the ISO 8601 position mapping (N1=year, N2=month, N3=day) with a @@ -32,15 +42,7 @@ class SlashISODateGrammar(Grammar[DateNotation]): semantics = "iso8601_calendar_date" single_value = True - def recognize(self, text: str) -> list[RecognitionMatch[DateNotation]]: - """Extract YYYY/MM/DD date patterns from text.""" - return [ - RecognitionMatch( - notation=DateNotation(N1=year, N2=month, N3=day), - start=match.start(), - end=match.end(), - raw_text=match.group(0), - ) - for match in _SLASH_ISO_PATTERN.finditer(text) - for year, month, day in [match.groups()] - ] + pre = StandardPre[DateNotation](empty_guard=True) + regex = RegexStage[DateNotation]( + pattern=_SLASH_ISO_PATTERN, notation_fn=_slash_iso_notation + ) diff --git a/paxman/capabilities/Date/grammar/us_recognition.py b/paxman/capabilities/Date/grammar/us_recognition.py index 487ef725..47f858aa 100644 --- a/paxman/capabilities/Date/grammar/us_recognition.py +++ b/paxman/capabilities/Date/grammar/us_recognition.py @@ -1,6 +1,19 @@ -"""US date grammar — recognizes MM/DD/YYYY and MM/DD/YY formats. - -Notation mapping: N1=month, N2=day, N3=year +"""US date recognition grammar (staged pipeline). + +Recognizes MM/DD/YYYY and MM/DD/YY formats. The 4-digit and 2-digit year +variants are merged into one year-length alternation; the digit lookarounds +(via BoundaryGuard.digit()) keep the pattern disjoint from surrounding +digits. Notation mapping: N1=month, N2=day, N3=year. + +Note: The legacy bespoke ``recognize()`` ran two separate ``finditer`` loops +(4-digit year first, then 2-digit), so grouped matches by year length rather +than document order (e.g. ``"01/02/26 foo 01/02/2026"`` yielded +``[01/02/2026, 01/02/26]``). The staged ``(\\d{4}|\\d{2})`` alternation uses a +single ``finditer`` in document order (``[01/02/26, 01/02/2026]``). The engine +(``paxman/engine/orchestrator.py``) sorts by ``start`` before dedup, so +end-to-end ``canonicalize()`` results are identical; direct ``recognize()`` +order is now document-order. See ADR-0008 D1 and harness note in +``tests/property/test_grammar_stage_parity.py``. """ from __future__ import annotations @@ -8,13 +21,20 @@ import re from paxman.capabilities.Date.notation import DateNotation -from paxman.core.domain import Grammar, RecognitionMatch +from paxman.core.grammar import BoundaryGuard, PipelineGrammar, RegexStage, StandardPre + +_GUARD = BoundaryGuard.digit() +_US_PATTERN = ( + _GUARD.lookbehind + r"(\d{1,2})/(\d{1,2})/(\d{4}|\d{2})" + _GUARD.lookahead +) + -_US_DATE_PATTERN_4DIGIT = re.compile(r"(? DateNotation: + """Map a US date match to its month/day/year notation.""" + return DateNotation(N1=match.group(1), N2=match.group(2), N3=match.group(3)) -class USDateGrammar(Grammar[DateNotation]): +class USDateGrammar(PipelineGrammar[DateNotation]): """US date recognition: MM/DD/YYYY and MM/DD/YY. Both year-length variants carry digit lookarounds, so a date glued to @@ -28,34 +48,5 @@ class USDateGrammar(Grammar[DateNotation]): semantics = "us_calendar_date" single_value = True - def recognize(self, text: str) -> list[RecognitionMatch[DateNotation]]: - """Extract US date patterns from text. - - Emits spans for every pattern match; the engine resolves - overlapping 2-digit/4-digit year spans via span dedup. - """ - results: list[RecognitionMatch[DateNotation]] = [] - - for match in _US_DATE_PATTERN_4DIGIT.finditer(text): - month, day, year = match.group(1), match.group(2), match.group(3) - results.append( - RecognitionMatch( - notation=DateNotation(N1=month, N2=day, N3=year), - start=match.start(), - end=match.end(), - raw_text=match.group(0), - ) - ) - - for match in _US_DATE_PATTERN_2DIGIT.finditer(text): - month, day, year = match.group(1), match.group(2), match.group(3) - results.append( - RecognitionMatch( - notation=DateNotation(N1=month, N2=day, N3=year), - start=match.start(), - end=match.end(), - raw_text=match.group(0), - ) - ) - - return results + pre = StandardPre[DateNotation](empty_guard=True) + regex = RegexStage[DateNotation](pattern=_US_PATTERN, notation_fn=_us_notation) diff --git a/paxman/capabilities/Email/grammar/localhost_recognition.py b/paxman/capabilities/Email/grammar/localhost_recognition.py index db3878a3..2efd997a 100644 --- a/paxman/capabilities/Email/grammar/localhost_recognition.py +++ b/paxman/capabilities/Email/grammar/localhost_recognition.py @@ -1,36 +1,35 @@ -"""Localhost email recognition grammar.""" +"""Localhost email recognition grammar (staged pipeline). + +Recognizes user@localhost. The word boundary and trailing lookahead are kept +verbatim (``\\b`` is a word boundary, not a hard-coded lookaround class — +ADR-0008 D5). Syntax only: the grammar never validates the address. +""" from __future__ import annotations import re from paxman.capabilities.Email.notation import EmailNotation -from paxman.core.domain import Grammar, RecognitionMatch +from paxman.core.grammar import PipelineGrammar, RegexStage, StandardPre -_LOCALHOST_PATTERN = re.compile( - r"\b([A-Za-z0-9._%+-]+)@localhost(?::\d+)?(?:(?=[\s,;()]|$)|\.(?=\s|$))", - re.IGNORECASE, +_LOCALHOST_PATTERN = ( + r"\b([A-Za-z0-9._%+-]+)@localhost(?::\d+)?(?:(?=[\s,;()]|$)|\.(?=\s|$))" ) -class LocalhostEmailGrammar(Grammar[EmailNotation]): +def _localhost_notation(match: re.Match[str]) -> EmailNotation: + """Map a localhost email match to its local/localhost notation.""" + return EmailNotation(local_part=match.group(1), domain_part="localhost") + + +class LocalhostEmailGrammar(PipelineGrammar[EmailNotation]): """Localhost email recognition: user@localhost.""" name = "localhost_recognition" semantics = "localhost_recognition" single_value = True - def recognize(self, text: str) -> list[RecognitionMatch[EmailNotation]]: - matches: list[RecognitionMatch[EmailNotation]] = [] - for match in _LOCALHOST_PATTERN.finditer(text): - matches.append( - RecognitionMatch( - notation=EmailNotation( - local_part=match.group(1), domain_part="localhost" - ), - start=match.start(), - end=match.end(), - raw_text=match.group(0), - ) - ) - return matches + pre = StandardPre[EmailNotation](empty_guard=True) + regex = RegexStage[EmailNotation]( + pattern=_LOCALHOST_PATTERN, notation_fn=_localhost_notation, flags=re.IGNORECASE + ) diff --git a/paxman/capabilities/Email/grammar/obfuscated_recognition.py b/paxman/capabilities/Email/grammar/obfuscated_recognition.py index 31745016..4f89cdc7 100644 --- a/paxman/capabilities/Email/grammar/obfuscated_recognition.py +++ b/paxman/capabilities/Email/grammar/obfuscated_recognition.py @@ -1,60 +1,53 @@ -"""Obfuscated email recognition grammar.""" +"""Obfuscated email recognition grammar (staged pipeline). + +Recognizes "user at domain dot tld" and "user at domain.tld". The two legacy +patterns (dot-form and at-only form) are merged into one alternation; the +notation factory branches on which alternative matched. Word boundaries are +kept verbatim (``\\b`` is a word boundary, not a hard-coded lookaround class — +ADR-0008 D5). Syntax only: the grammar never validates the address. +""" from __future__ import annotations import re from paxman.capabilities.Email.notation import EmailNotation -from paxman.core.domain import Grammar, RecognitionMatch - -# Matches: "user at domain dot tld" -_OBFUSCATED_PATTERN = re.compile( - r"\b([A-Za-z0-9._%+-]+)\s+at\s+([A-Za-z0-9.-]+)\s+dot\s+([A-Za-z]{2,})\b" -) -# Matches: "user at domain.tld" -_AT_ONLY_PATTERN = re.compile( - r"\b([A-Za-z0-9._%+-]+)\s+at\s+([A-Za-z0-9.-]+\.[A-Za-z]{2,})\b" +from paxman.core.grammar import PipelineGrammar, RegexStage, StandardPre + +# Dot-form: "user at example dot com" -> group(2).group(3). +# At-only form: "user at gmail.com" -> group(4). +_OBFUSCATED_PATTERN = ( + r"\b([A-Za-z0-9._%+-]+)\s+at\s+(?:" + r"([A-Za-z0-9.-]+)\s+dot\s+([A-Za-z]{2,})" + r"|" + r"([A-Za-z0-9.-]+\.[A-Za-z]{2,})" + r")\b" ) -class ObfuscatedEmailGrammar(Grammar[EmailNotation]): +def _obfuscated_notation(match: re.Match[str]) -> EmailNotation: + """Map an obfuscated email match to its local/domain notation. + + The dot-form alternative captures the domain before "dot" and the TLD as + separate groups; the at-only form captures the whole domain.tld as one + group. Mirrors the legacy two-pattern recognize() byte-for-byte. + """ + local = match.group(1) + if match.group(2) is not None: + domain = f"{match.group(2)}.{match.group(3)}" + else: + domain = match.group(4) + return EmailNotation(local_part=local, domain_part=domain) + + +class ObfuscatedEmailGrammar(PipelineGrammar[EmailNotation]): """Obfuscated email: 'user at domain dot tld' or 'user at domain.tld'.""" name = "obfuscated_recognition" semantics = "rfc5322_addr_spec" single_value = True - def recognize(self, text: str) -> list[RecognitionMatch[EmailNotation]]: - """Extract obfuscated email patterns from text. - - Both patterns emit span-bearing matches; the engine merges, orders - (document order), and dedups contained matches, and identical - candidate values collapse at the candidate stage. The grammar does - not de-duplicate. - """ - matches: list[RecognitionMatch[EmailNotation]] = [] - for match in _OBFUSCATED_PATTERN.finditer(text): - matches.append( - RecognitionMatch( - notation=EmailNotation( - local_part=match.group(1), - domain_part=f"{match.group(2)}.{match.group(3)}", - ), - start=match.start(), - end=match.end(), - raw_text=match.group(0), - ) - ) - for match in _AT_ONLY_PATTERN.finditer(text): - matches.append( - RecognitionMatch( - notation=EmailNotation( - local_part=match.group(1), - domain_part=match.group(2), - ), - start=match.start(), - end=match.end(), - raw_text=match.group(0), - ) - ) - return matches + pre = StandardPre[EmailNotation](empty_guard=True) + regex = RegexStage[EmailNotation]( + pattern=_OBFUSCATED_PATTERN, notation_fn=_obfuscated_notation + ) diff --git a/paxman/capabilities/Email/grammar/standard_recognition.py b/paxman/capabilities/Email/grammar/standard_recognition.py index c96807be..abfd1aea 100644 --- a/paxman/capabilities/Email/grammar/standard_recognition.py +++ b/paxman/capabilities/Email/grammar/standard_recognition.py @@ -1,36 +1,34 @@ -"""Standard email recognition grammar.""" +"""Standard email recognition grammar (staged pipeline). + +Recognizes user@domain.tld. The word boundaries are kept verbatim (``\\b`` is +a word boundary, not a hard-coded lookaround class — ADR-0008 D5). Syntax +only: the grammar never validates the address. +""" from __future__ import annotations import re from paxman.capabilities.Email.notation import EmailNotation -from paxman.core.domain import Grammar, RecognitionMatch +from paxman.core.grammar import PipelineGrammar, RegexStage, StandardPre + +_STANDARD_PATTERN = r"\b[A-Za-z0-9][A-Za-z0-9._%+-]*@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b" + -_STANDARD_PATTERN = re.compile( - r"\b[A-Za-z0-9][A-Za-z0-9._%+-]*@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b" -) +def _standard_notation(match: re.Match[str]) -> EmailNotation: + """Map a standard email match to its local/domain notation.""" + local, domain = match.group(0).split("@") + return EmailNotation(local_part=local, domain_part=domain) -class StandardEmailGrammar(Grammar[EmailNotation]): +class StandardEmailGrammar(PipelineGrammar[EmailNotation]): """Standard email recognition: user@domain.tld.""" name = "standard_recognition" semantics = "rfc5322_addr_spec" single_value = True - def recognize(self, text: str) -> list[RecognitionMatch[EmailNotation]]: - matches: list[RecognitionMatch[EmailNotation]] = [] - for match in _STANDARD_PATTERN.finditer(text): - matches.append( - RecognitionMatch( - notation=EmailNotation( - local_part=match.group(0).split("@")[0], - domain_part=match.group(0).split("@")[1], - ), - start=match.start(), - end=match.end(), - raw_text=match.group(0), - ) - ) - return matches + pre = StandardPre[EmailNotation](empty_guard=True) + regex = RegexStage[EmailNotation]( + pattern=_STANDARD_PATTERN, notation_fn=_standard_notation + ) diff --git a/paxman/capabilities/IP/grammar/ipv4_recognition.py b/paxman/capabilities/IP/grammar/ipv4_recognition.py index 249ad89a..e162dd64 100644 --- a/paxman/capabilities/IP/grammar/ipv4_recognition.py +++ b/paxman/capabilities/IP/grammar/ipv4_recognition.py @@ -1,32 +1,31 @@ -"""IPv4 recognition grammar — extracts dotted-decimal IPv4 addresses.""" +"""IPv4 recognition grammar (staged pipeline). + +Extracts dotted-decimal IPv4 addresses. The word boundaries are kept verbatim +(``\\b`` is a word boundary, not a hard-coded lookaround class — ADR-0008 +D5). Syntax only: the grammar never validates the address. +""" from __future__ import annotations import re from paxman.capabilities.IP.notation import IPNotation -from paxman.core.domain import Grammar, RecognitionMatch +from paxman.core.grammar import PipelineGrammar, RegexStage, StandardPre + +_IPV4_PATTERN = r"\b(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})\b" + -_IPV4_PATTERN = re.compile(r"\b(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})\b") +def _ipv4_notation(match: re.Match[str]) -> IPNotation: + """Map an IPv4 match to its address notation.""" + return IPNotation(address=match.group(0)) -class IPv4Grammar(Grammar[IPNotation]): +class IPv4Grammar(PipelineGrammar[IPNotation]): """IPv4 recognition: dotted-decimal format (e.g., 192.168.1.1).""" name = "ipv4_recognition" semantics = "ipv4_recognition" single_value = True - def recognize(self, text: str) -> list[RecognitionMatch[IPNotation]]: - """Extract IPv4 dotted-decimal patterns from text.""" - matches: list[RecognitionMatch[IPNotation]] = [] - for match in _IPV4_PATTERN.finditer(text): - matches.append( - RecognitionMatch( - notation=IPNotation(address=match.group(0)), - start=match.start(), - end=match.end(), - raw_text=match.group(0), - ) - ) - return matches + pre = StandardPre[IPNotation](empty_guard=True) + regex = RegexStage[IPNotation](pattern=_IPV4_PATTERN, notation_fn=_ipv4_notation) diff --git a/paxman/capabilities/IP/grammar/ipv6_recognition.py b/paxman/capabilities/IP/grammar/ipv6_recognition.py index 8d40ddf7..154f6c62 100644 --- a/paxman/capabilities/IP/grammar/ipv6_recognition.py +++ b/paxman/capabilities/IP/grammar/ipv6_recognition.py @@ -1,43 +1,57 @@ -"""IPv6 recognition grammar — extracts IPv6 addresses in various formats.""" +"""IPv6 recognition grammar (staged pipeline). + +Extracts IPv6 addresses in full and compressed formats. The full and +compressed legacy patterns are merged into one alternation wrapped by a +single outer capture group; the boundary lookarounds are supplied by +BoundaryGuard.ipv6_token() (ADR-0008 D5) so no hard-coded lookaround literal +remains in this file. Syntax only: the grammar never validates the address. + +Note: The legacy bespoke ``recognize()`` ran two separate ``finditer`` loops +(full before compressed), so grouped matches by address family rather than +document order (e.g. ``"2001:db8:85a3::8a2e:370:7334 ::1"`` yielded +``[full, ::1]`` regardless of positions). The staged +``(_IPV6_FULL_INNER|_IPV6_COMPRESSED_INNER)`` alternation uses a single +``finditer`` in document order. The engine sorts by ``start`` before dedup, +so end-to-end ``canonicalize()`` is identical; direct ``recognize()`` order +is now document-order. +""" from __future__ import annotations import re from paxman.capabilities.IP.notation import IPNotation -from paxman.core.domain import Grammar, RecognitionMatch +from paxman.core.grammar import BoundaryGuard, PipelineGrammar, RegexStage, StandardPre -# Boundary: start/end of string, whitespace, or common punctuation -_IPV6_BOUNDARY = r"(?:^|(?<=[\s,;([ ]))" -_IPV6_END = r"(?:$|(?=[\s,;().\]]))" - -# Full form: 8 groups of 1-4 hex digits separated by single colons -_IPV6_FULL = re.compile( - _IPV6_BOUNDARY + r"([0-9a-fA-F]{1,4}(?::[0-9a-fA-F]{1,4}){7})" + _IPV6_END +# Full form: 8 groups of 1-4 hex digits separated by single colons. +_IPV6_FULL_INNER = r"[0-9a-fA-F]{1,4}(?::[0-9a-fA-F]{1,4}){7}" +# Compressed forms: handles :: with groups on either side, loopback, link-local, +# and all-zeros. Each branch is a zero-width-boundary-free inner alternative. +_IPV6_COMPRESSED_INNER = ( + r"(?:[0-9a-fA-F]{1,4}:){0,6}[0-9a-fA-F]{0,4}::" + r"(?:[0-9a-fA-F]{0,4}:){0,6}[0-9a-fA-F]{1,4}" + r"|" + r"::(?:[0-9a-fA-F]{1,4}:){0,6}[0-9a-fA-F]{1,4}" + r"|" + r"(?:[0-9a-fA-F]{1,4}:){1,6}[0-9a-fA-F]{0,4}::" + r"|" + r"::" ) +# One outer capture group so the address is always group("addr"), mirroring the +# legacy recognize() (FULL used group(1), COMPRESSED used group(0) == address). +# Using a named group prevents future inner captures from shifting the index. +_IPV6_BODY = r"(?P" + _IPV6_FULL_INNER + r"|" + _IPV6_COMPRESSED_INNER + r")" -# Compressed form: handles :: with groups on either side -_IPV6_COMPRESSED = re.compile( - _IPV6_BOUNDARY - + r"((?:[0-9a-fA-F]{1,4}:){0,6}[0-9a-fA-F]{0,4}::" - + r"(?:[0-9a-fA-F]{0,4}:){0,6}[0-9a-fA-F]{1,4})" - + _IPV6_END - + "|" - + _IPV6_BOUNDARY - + r"(::(?:[0-9a-fA-F]{1,4}:){0,6}[0-9a-fA-F]{1,4})" - + _IPV6_END - + "|" - + _IPV6_BOUNDARY - + r"((?:[0-9a-fA-F]{1,4}:){1,6}[0-9a-fA-F]{0,4}::)" - + _IPV6_END - + "|" - + _IPV6_BOUNDARY - + r"(::)" - + _IPV6_END -) +_GUARD = BoundaryGuard.ipv6_token() +_IPV6_PATTERN = _GUARD.lookbehind + _IPV6_BODY + _GUARD.lookahead -class IPv6Grammar(Grammar[IPNotation]): +def _ipv6_notation(match: re.Match[str]) -> IPNotation: + """Map an IPv6 match to its address notation (the outer capture group).""" + return IPNotation(address=match.group("addr")) + + +class IPv6Grammar(PipelineGrammar[IPNotation]): """IPv6 recognition: full and compressed formats. Handles: @@ -52,33 +66,5 @@ class IPv6Grammar(Grammar[IPNotation]): semantics = "ipv6_recognition" single_value = True - def recognize(self, text: str) -> list[RecognitionMatch[IPNotation]]: - """Extract IPv6 address patterns from text. - - The full and compressed patterns are structurally disjoint, so every - match emits a span-bearing RecognitionMatch; the engine dedups - contained spans and identical candidate values collapse at the - candidate stage. - """ - matches: list[RecognitionMatch[IPNotation]] = [] - for match in _IPV6_FULL.finditer(text): - matches.append( - RecognitionMatch( - notation=IPNotation(address=match.group(1)), - start=match.start(), - end=match.end(), - raw_text=match.group(1), - ) - ) - for match in _IPV6_COMPRESSED.finditer(text): - # Boundary assertions are zero-width and each alternation branch - # has one capture group, so the full match text IS the address. - matches.append( - RecognitionMatch( - notation=IPNotation(address=match.group(0)), - start=match.start(), - end=match.end(), - raw_text=match.group(0), - ) - ) - return matches + pre = StandardPre[IPNotation](empty_guard=True) + regex = RegexStage[IPNotation](pattern=_IPV6_PATTERN, notation_fn=_ipv6_notation) diff --git a/paxman/capabilities/ISBN/grammar/isbn10_recognition.py b/paxman/capabilities/ISBN/grammar/isbn10_recognition.py index b10cc2ad..baec66fa 100644 --- a/paxman/capabilities/ISBN/grammar/isbn10_recognition.py +++ b/paxman/capabilities/ISBN/grammar/isbn10_recognition.py @@ -1,41 +1,42 @@ -"""ISBN-10 recognition grammar.""" +"""ISBN-10 recognition grammar (staged pipeline). + +Recognizes 10-digit ISBNs with optional label and separators. The leading +digit-glued guard is supplied by BoundaryGuard.isbn10_lead() (ADR-0008 D5) so +no hard-coded lookaround literal remains in this file. The trailing boundary is +handled by ``\\b`` (the previous ``isbn_trail`` lookbehind after the final digit +was inert). The hyphen/space tolerance is regex-native (the lookahead extracts +the digit run via a backreference). +""" from __future__ import annotations import re from paxman.capabilities.ISBN.notation import ISBNNotation -from paxman.core.domain import Grammar, RecognitionMatch +from paxman.core.grammar import BoundaryGuard, PipelineGrammar, RegexStage, StandardPre -_ISBN10_PATTERN = re.compile( - r"(? ISBNNotation: + """Map an ISBN-10 match to its digit-string notation (X uppercased).""" + digits = "".join(ch for ch in match.group(1) if ch in "0123456789Xx").upper() + return ISBNNotation(shape="isbn10", digits=digits) + + +class ISBN10RecognitionGrammar(PipelineGrammar[ISBNNotation]): """ISBN-10 recognition: 10-digit ISBN with optional label and separators.""" name = "isbn10_recognition" semantics = "isbn10_recognition" single_value = True - def recognize(self, text: str) -> list[RecognitionMatch[ISBNNotation]]: - matches: list[RecognitionMatch[ISBNNotation]] = [] - for m in _ISBN10_PATTERN.finditer(text): - digits = "".join(ch for ch in m.group(1) if ch in "0123456789Xx").upper() - if len(digits) != 10: - continue - matches.append( - RecognitionMatch( - notation=ISBNNotation( - shape="isbn10", - digits=digits, - ), - start=m.start(), - end=m.end(), - raw_text=m.group(0), - ) - ) - return matches + pre = StandardPre[ISBNNotation](empty_guard=True) + regex = RegexStage[ISBNNotation]( + pattern=_ISBN10_PATTERN, notation_fn=_isbn10_notation, flags=re.IGNORECASE + ) diff --git a/paxman/capabilities/ISBN/grammar/isbn13_recognition.py b/paxman/capabilities/ISBN/grammar/isbn13_recognition.py index 17a37b3e..d014f99f 100644 --- a/paxman/capabilities/ISBN/grammar/isbn13_recognition.py +++ b/paxman/capabilities/ISBN/grammar/isbn13_recognition.py @@ -1,40 +1,37 @@ -"""ISBN-13 recognition grammar.""" +"""ISBN-13 recognition grammar (staged pipeline). + +Recognizes 13-digit ISBNs with optional label and separators. The hyphen/space +tolerance is regex-native (the lookahead extracts the digit run via a +backreference). The trailing word boundary ``\\b`` handles the right edge; no +additional trailing guard is needed (the previous ``isbn_trail`` lookbehind was +inert after the final digit). +""" from __future__ import annotations import re from paxman.capabilities.ISBN.notation import ISBNNotation -from paxman.core.domain import Grammar, RecognitionMatch +from paxman.core.grammar import PipelineGrammar, RegexStage, StandardPre + +_ISBN13_BODY = r"\b(?:ISBN(?:-13)?[\s:-]+)?(?=((?:\d[ -]?){12}\d)(?![\d]))\1" +_ISBN13_PATTERN = _ISBN13_BODY + r"\b" + -_ISBN13_PATTERN = re.compile( - r"\b(?:ISBN(?:-13)?[\s:-]+)?(?=((?:\d[ -]?){12}\d)(?![\d]))\1(? ISBNNotation: + """Map an ISBN-13 match to its digit-string notation.""" + digits = "".join(ch for ch in match.group(1) if ch in "0123456789") + return ISBNNotation(shape="isbn13", digits=digits) -class ISBN13RecognitionGrammar(Grammar[ISBNNotation]): +class ISBN13RecognitionGrammar(PipelineGrammar[ISBNNotation]): """ISBN-13 recognition: 13-digit ISBN with optional label and separators.""" name = "isbn13_recognition" semantics = "isbn13_recognition" single_value = True - def recognize(self, text: str) -> list[RecognitionMatch[ISBNNotation]]: - matches: list[RecognitionMatch[ISBNNotation]] = [] - for m in _ISBN13_PATTERN.finditer(text): - digits = "".join(ch for ch in m.group(1) if ch in "0123456789") - if len(digits) != 13: - continue - matches.append( - RecognitionMatch( - notation=ISBNNotation( - shape="isbn13", - digits=digits, - ), - start=m.start(), - end=m.end(), - raw_text=m.group(0), - ) - ) - return matches + pre = StandardPre[ISBNNotation](empty_guard=True) + regex = RegexStage[ISBNNotation]( + pattern=_ISBN13_PATTERN, notation_fn=_isbn13_notation, flags=re.IGNORECASE + ) diff --git a/paxman/capabilities/Money/grammar/code_recognition.py b/paxman/capabilities/Money/grammar/code_recognition.py index 25f89fe9..d040fe5e 100644 --- a/paxman/capabilities/Money/grammar/code_recognition.py +++ b/paxman/capabilities/Money/grammar/code_recognition.py @@ -1,30 +1,36 @@ -"""ISO 4217 alpha-3 currency code recognition grammar. +"""ISO 4217 alpha-3 currency code recognition grammar (staged pipeline). Recognizes an ISO 4217 alpha-3 code shape adjacent to an amount, in -either order, as one span-bearing token. Syntax only: unknown codes are -still matched — deciding validity is the rules' job. +either order, as one span-bearing token. The fused either-order regex is +built by ``AmountComposer`` (S4): a 3-letter uppercase ASCII code +adjacent to an amount, prefix ("USD500", "USD 500") or suffix ("500 USD", +"100MYR"). Syntax only: unknown codes are still matched — deciding +validity is the rules' job. """ from __future__ import annotations -import re -from typing import cast - from paxman.capabilities.Money.grammar import AMOUNT_PATTERN, classify_amount_shape from paxman.capabilities.Money.notation import MoneyNotation -from paxman.core.domain import Grammar, RecognitionMatch - -# Sign characters ('-', U+2212, '+') are outside the amount grammar; the -# boundary guards reject sign-adjacent tokens so the sign is never dropped. -_CODE_PATTERN = re.compile( - rf"(?[A-Z]{{3}}) ?(?P{AMOUNT_PATTERN})" - rf"|(?P{AMOUNT_PATTERN}) ?(?P[A-Z]{{3}}))" - rf"(?![\w\-+\u2212])" +from paxman.core.grammar import ( + AmountComposer, + BoundaryGuard, + PipelineGrammar, + StandardPre, ) -class CodeRecognition(Grammar[MoneyNotation]): +def _code_notation(lex: str, amount: str, amount_shape: str) -> MoneyNotation: + """Map a matched code+amount token to its notation.""" + return MoneyNotation( + currency_part=lex, + amount_part=amount, + currency_shape="code", + amount_shape=amount_shape, + ) + + +class CodeRecognition(PipelineGrammar[MoneyNotation]): """Recognizes ISO 4217 alpha-3 code + amount tokens. Matches a 3-letter uppercase ASCII code adjacent to an amount in @@ -41,36 +47,12 @@ class CodeRecognition(Grammar[MoneyNotation]): semantics = "code_recognition" single_value = True - def recognize(self, text: str) -> list[RecognitionMatch[MoneyNotation]]: - """Extract code+amount tokens from text. - - Args: - text: Raw input text. - - Returns: - List of span-bearing matches with shape "code" notations. - """ - if not text.strip(): - return [] - matches: list[RecognitionMatch[MoneyNotation]] = [] - for match in _CODE_PATTERN.finditer(text): - currency = cast( - str, match.group("prefix_code") or match.group("suffix_code") - ) - amount = cast( - str, match.group("prefix_amount") or match.group("suffix_amount") - ) - matches.append( - RecognitionMatch( - notation=MoneyNotation( - currency_part=currency, - amount_part=amount, - currency_shape="code", - amount_shape=classify_amount_shape(amount), - ), - start=match.start(), - end=match.end(), - raw_text=match.group(0), - ) - ) - return matches + pre = StandardPre[MoneyNotation](empty_guard=True) + composer = AmountComposer[MoneyNotation]( + pattern=AMOUNT_PATTERN, + lexicon_tokens=None, + notation_fn=_code_notation, + classify=classify_amount_shape, + boundary=BoundaryGuard.word_sign(), + flags=0, + ) diff --git a/paxman/capabilities/Money/grammar/symbol_recognition.py b/paxman/capabilities/Money/grammar/symbol_recognition.py index f1e96d85..d127bde7 100644 --- a/paxman/capabilities/Money/grammar/symbol_recognition.py +++ b/paxman/capabilities/Money/grammar/symbol_recognition.py @@ -1,33 +1,23 @@ -"""CLDR currency symbol recognition grammar. +"""CLDR currency symbol recognition grammar (staged pipeline). Recognizes a currency symbol token adjacent to an amount, in either order, as one span-bearing token. The symbol alternation is built from SYMBOL_TOKENS (qualified forms first, longest-first within each class, -so "US$" alternates before "$"). Syntax only: resolving the symbol to a -code is the rules' job. +so "US$" alternates before "$") and fused with the amount by +``AmountComposer`` (S4). Syntax only: resolving the symbol to a code is +the rules' job. """ from __future__ import annotations -import re -from typing import cast - from paxman.capabilities.Money.grammar import AMOUNT_PATTERN, classify_amount_shape from paxman.capabilities.Money.grammar.data.currency_symbols import SYMBOL_TOKENS from paxman.capabilities.Money.notation import MoneyNotation -from paxman.core.domain import Grammar, RecognitionMatch - -_SYMBOL_ALTERNATION = "|".join(re.escape(token) for token in SYMBOL_TOKENS) -# Lookarounds, not \b: pure-symbol tokens ("$", "€") are non-word -# characters that \b would reject at string start, and the lookarounds -# still block matches inside a longer token. Sign characters ('-', -# U+2212, '+') are also rejected at the boundary so a sign-adjacent -# token never matches (the sign would otherwise be silently dropped). -_SYMBOL_PATTERN = re.compile( - rf"(?{_SYMBOL_ALTERNATION})" - rf" ?(?P{AMOUNT_PATTERN})" - rf"|(?P{AMOUNT_PATTERN}) ?(?P{_SYMBOL_ALTERNATION}))" - rf"(?![\w\-+\u2212])" +from paxman.core.grammar import ( + AmountComposer, + BoundaryGuard, + PipelineGrammar, + StandardPre, ) @@ -36,7 +26,17 @@ def _is_qualified(token: str) -> bool: return any(ch.isascii() and ch.isalpha() for ch in token) -class SymbolRecognition(Grammar[MoneyNotation]): +def _symbol_notation(lex: str, amount: str, amount_shape: str) -> MoneyNotation: + """Map a matched symbol+amount token to its qualified/bare notation.""" + return MoneyNotation( + currency_part=lex, + amount_part=amount, + currency_shape="qualified_symbol" if _is_qualified(lex) else "symbol", + amount_shape=amount_shape, + ) + + +class SymbolRecognition(PipelineGrammar[MoneyNotation]): """Recognizes currency symbol + amount tokens. Matches a CLDR symbol adjacent to an amount in either order: @@ -56,39 +56,12 @@ class SymbolRecognition(Grammar[MoneyNotation]): semantics = "symbol_recognition" single_value = True - def recognize(self, text: str) -> list[RecognitionMatch[MoneyNotation]]: - """Extract symbol+amount tokens from text. - - Args: - text: Raw input text. - - Returns: - List of span-bearing matches with "symbol"/"qualified_symbol" - notations. - """ - if not text.strip(): - return [] - matches: list[RecognitionMatch[MoneyNotation]] = [] - for match in _SYMBOL_PATTERN.finditer(text): - symbol = cast( - str, match.group("prefix_symbol") or match.group("suffix_symbol") - ) - amount = cast( - str, match.group("prefix_amount") or match.group("suffix_amount") - ) - matches.append( - RecognitionMatch( - notation=MoneyNotation( - currency_part=symbol, - amount_part=amount, - currency_shape=( - "qualified_symbol" if _is_qualified(symbol) else "symbol" - ), - amount_shape=classify_amount_shape(amount), - ), - start=match.start(), - end=match.end(), - raw_text=match.group(0), - ) - ) - return matches + pre = StandardPre[MoneyNotation](empty_guard=True) + composer = AmountComposer[MoneyNotation]( + pattern=AMOUNT_PATTERN, + lexicon_tokens=SYMBOL_TOKENS, + notation_fn=_symbol_notation, + classify=classify_amount_shape, + boundary=BoundaryGuard.word_sign(), + flags=0, + ) diff --git a/paxman/capabilities/Money/grammar/word_recognition.py b/paxman/capabilities/Money/grammar/word_recognition.py index b7246ed3..0d4eb395 100644 --- a/paxman/capabilities/Money/grammar/word_recognition.py +++ b/paxman/capabilities/Money/grammar/word_recognition.py @@ -1,35 +1,38 @@ -"""CLDR currency display-name word recognition grammar. +"""CLDR currency display-name word recognition grammar (staged pipeline). -Recognizes a currency display-name word adjacent to an amount, in -either order, as one span-bearing token. The word alternation is built -from WORD_TOKENS (longest-first), word-boundary anchored and -case-insensitive. Syntax only: resolving the word to a code is the -rules' job. +Recognizes a currency display-name word adjacent to an amount, in either +order, as one span-bearing token. The word alternation is built from +WORD_TOKENS (longest-first), word-boundary anchored and case-insensitive, +and fused with the amount by ``AmountComposer`` (S4). Syntax only: +resolving the word to a code is the rules' job. """ from __future__ import annotations import re -from typing import cast from paxman.capabilities.Money.grammar import AMOUNT_PATTERN, classify_amount_shape from paxman.capabilities.Money.grammar.data.currency_words import WORD_TOKENS from paxman.capabilities.Money.notation import MoneyNotation -from paxman.core.domain import Grammar, RecognitionMatch - -_WORD_ALTERNATION = "|".join(re.escape(token) for token in WORD_TOKENS) -# Sign characters ('-', U+2212, '+') are outside the amount grammar; the -# boundary guards reject sign-adjacent tokens so the sign is never dropped. -_WORD_PATTERN = re.compile( - rf"(?{_WORD_ALTERNATION})" - rf" ?(?P{AMOUNT_PATTERN})" - rf"|(?P{AMOUNT_PATTERN}) ?(?P{_WORD_ALTERNATION}))" - rf"(?![\w\-+\u2212])", - re.IGNORECASE, +from paxman.core.grammar import ( + AmountComposer, + BoundaryGuard, + PipelineGrammar, + StandardPre, ) -class WordRecognition(Grammar[MoneyNotation]): +def _word_notation(lex: str, amount: str, amount_shape: str) -> MoneyNotation: + """Map a matched word+amount token to its notation.""" + return MoneyNotation( + currency_part=lex, + amount_part=amount, + currency_shape="word", + amount_shape=amount_shape, + ) + + +class WordRecognition(PipelineGrammar[MoneyNotation]): """Recognizes currency display-name word + amount tokens. Matches a CLDR display-name word adjacent to an amount in either @@ -48,34 +51,12 @@ class WordRecognition(Grammar[MoneyNotation]): semantics = "word_recognition" single_value = True - def recognize(self, text: str) -> list[RecognitionMatch[MoneyNotation]]: - """Extract word+amount tokens from text. - - Args: - text: Raw input text. - - Returns: - List of span-bearing matches with shape "word" notations. - """ - if not text.strip(): - return [] - matches: list[RecognitionMatch[MoneyNotation]] = [] - for match in _WORD_PATTERN.finditer(text): - word = cast(str, match.group("prefix_word") or match.group("suffix_word")) - amount = cast( - str, match.group("prefix_amount") or match.group("suffix_amount") - ) - matches.append( - RecognitionMatch( - notation=MoneyNotation( - currency_part=word, - amount_part=amount, - currency_shape="word", - amount_shape=classify_amount_shape(amount), - ), - start=match.start(), - end=match.end(), - raw_text=match.group(0), - ) - ) - return matches + pre = StandardPre[MoneyNotation](empty_guard=True) + composer = AmountComposer[MoneyNotation]( + pattern=AMOUNT_PATTERN, + lexicon_tokens=WORD_TOKENS, + notation_fn=_word_notation, + classify=classify_amount_shape, + boundary=BoundaryGuard.word_sign(), + flags=re.IGNORECASE, + ) diff --git a/paxman/capabilities/Phone/grammar/common.py b/paxman/capabilities/Phone/grammar/_common.py similarity index 53% rename from paxman/capabilities/Phone/grammar/common.py rename to paxman/capabilities/Phone/grammar/_common.py index d83a1287..d1f78abc 100644 --- a/paxman/capabilities/Phone/grammar/common.py +++ b/paxman/capabilities/Phone/grammar/_common.py @@ -1,17 +1,15 @@ -"""Shared helpers for Phone recognition grammars. +"""Shared Phone grammar helpers — separator stripping. -Space, dash, dot, and parentheses are the separators every Phone grammar -tolerates inside a number. ``strip_separators`` normalizes a raw match to -digit-only text. Grammar-level value dedup was removed in the recognition- -homogeneity migration: the engine dedups contained matches by span and -identical candidates by value. +Capability-local helper for Phone grammars (ADR-0008 D5 self-containment). +``paxman/core/grammar`` stays capability-agnostic (D2); Phone grammars share +this helper internally to avoid quadruplicated ``strip_separators`` and +translation-table construction. """ from __future__ import annotations -# Digits are preserved; space, dash, dot, and parentheses are removed. +# Translation tables — built once, reused. _SEPARATORS = str.maketrans("", "", " ().-") -# E.164 / tel-URI matches also carry a leading "+" to strip. _SEPARATORS_WITH_PLUS = str.maketrans("", "", "+ ().-") diff --git a/paxman/capabilities/Phone/grammar/e164_recognition.py b/paxman/capabilities/Phone/grammar/e164_recognition.py index 1897c6c7..d5a40b0f 100644 --- a/paxman/capabilities/Phone/grammar/e164_recognition.py +++ b/paxman/capabilities/Phone/grammar/e164_recognition.py @@ -1,22 +1,31 @@ -"""E.164 international number recognition grammar.""" +"""E.164 international number recognition grammar (staged pipeline). + +Recognizes a leading "+" followed by digits with optional separators +(space, dash, dot, parens). The grammar is intentionally loose — validation +happens in rules. The negative lookbehind (via BoundaryGuard.e164()) excludes +word characters, ":", and "." so email plus-tags, algebra, decimals, and +"tel:+..." are NOT double-matched. The trailing digit-ending lookbehind +forces the match to end on a digit so trailing separators/whitespace/ +punctuation are not swallowed. A PostStage trims runaway matches at the +15-digit E.164 window (ADR-0008 S5) so a following number is not merged into +the span. +""" from __future__ import annotations +import functools import re -from paxman.capabilities.Phone.grammar.common import strip_separators +from paxman.capabilities.Phone.grammar._common import strip_separators from paxman.capabilities.Phone.notation import PhoneNotation -from paxman.core.domain import Grammar, RecognitionMatch - -# A "+" followed by digits with optional separators (space, dash, dot, parens). -# The grammar is intentionally loose — validation happens in rules. The -# negative lookbehind excludes word characters (letters, digits, underscore), -# ":" and "." — so email plus-tags ("user+123@"), algebra ("x+11"), decimals -# (".+1.5"), and "tel:+..." (which the tel-URI grammar handles) are NOT -# double-matched. The trailing (?<=\d) lookbehind forces the match to end on -# a digit, so the trailing character class cannot swallow separators, -# whitespace, or sentence punctuation after the number. -_E164_PATTERN = re.compile(r"(? str: """Trim a runaway raw match at the last digit-run group within the limit. @@ -37,6 +47,10 @@ def _trim_to_e164_boundary(raw: str) -> str: swallowed into the match. If the first run alone exceeds the limit, the raw match is kept whole: validation then rejects the oversized value instead of silently recognizing a truncated 15-digit prefix. + + Cached so the paired ``_e164_notation`` / ``_e164_trim`` calls for the + same match reuse a single trimmed result instead of scanning digit runs + twice. """ runs = list(re.finditer(r"\d+", raw)) total = 0 @@ -49,7 +63,34 @@ def _trim_to_e164_boundary(raw: str) -> str: return raw -class E164Grammar(Grammar[PhoneNotation]): +# Body: "+" then digits with optional separators, ending on a digit. The +# leading lookbehind is supplied by BoundaryGuard.e164() (ADR-0008 D5) so no +# hard-coded lookaround literal remains in this file. +_E164_BODY = r"\+\d[\d\s().\-]*(?<=\d)" +_GUARD = BoundaryGuard.e164() +_E164_PATTERN = _GUARD.lookbehind + _E164_BODY + + +def _e164_notation(match: re.Match[str]) -> PhoneNotation: + """Map a raw E.164 match to its digit-only notation (trimmed to 15 digits).""" + raw = _trim_to_e164_boundary(match.group(0)) + return PhoneNotation(shape="e164", value=strip_separators(raw, plus=True)) + + +def _e164_trim( + match: RecognitionMatch[PhoneNotation], +) -> RecognitionMatch[PhoneNotation]: + """Adjust the span to the trimmed 15-digit window (end = start + len).""" + raw = _trim_to_e164_boundary(match.raw_text) + return RecognitionMatch( + notation=match.notation, + start=match.start, + end=match.start + len(raw), + raw_text=raw, + ) + + +class E164Grammar(PipelineGrammar[PhoneNotation]): """Recognizes E.164-style international numbers (leading +). Examples: "+15551234567", "+1 555 123 4567", "+44-20-7946-0958" @@ -60,25 +101,6 @@ class E164Grammar(Grammar[PhoneNotation]): semantics = "e164_international" single_value = True - def recognize(self, text: str) -> list[RecognitionMatch[PhoneNotation]]: - """Extract e164 patterns from text. - - Returns: - List of RecognitionMatches; notation.value is the digit-only - number (leading "+" and separators removed). - """ - matches: list[RecognitionMatch[PhoneNotation]] = [] - for match in _E164_PATTERN.finditer(text): - raw_text = _trim_to_e164_boundary(match.group(0)) - matches.append( - RecognitionMatch( - notation=PhoneNotation( - shape="e164", - value=strip_separators(raw_text, plus=True), - ), - start=match.start(), - end=match.start() + len(raw_text), - raw_text=raw_text, - ) - ) - return matches + pre = StandardPre[PhoneNotation](empty_guard=True) + regex = RegexStage[PhoneNotation](pattern=_E164_PATTERN, notation_fn=_e164_notation) + post = PostStage[PhoneNotation](transform=_e164_trim) diff --git a/paxman/capabilities/Phone/grammar/international_00_recognition.py b/paxman/capabilities/Phone/grammar/international_00_recognition.py index 843ce239..d2e39ead 100644 --- a/paxman/capabilities/Phone/grammar/international_00_recognition.py +++ b/paxman/capabilities/Phone/grammar/international_00_recognition.py @@ -1,35 +1,45 @@ -"""International 00-prefix recognition grammar. +"""International 00-prefix recognition grammar (staged pipeline). The international prefix "00" is the ITU-T E.164 recommended prefix used when dialing from within most countries. The digits AFTER the prefix form the E.164 number, so this grammar produces shape="e164" with the prefix stripped. + +The leading lookbehind (via BoundaryGuard.e164_00()) excludes word +characters, ":", ".", and "+" so "10044...", "x0044...", "0.0044...", and +"+0044..." are not treated as prefixes. The trailing digit-ending lookbehind +forces the match to end on a digit so trailing separators/whitespace/ +sentence punctuation are not swallowed (mirrors the E.164 grammar). """ from __future__ import annotations import re -from paxman.capabilities.Phone.grammar.common import strip_separators +from paxman.capabilities.Phone.grammar._common import strip_separators from paxman.capabilities.Phone.notation import PhoneNotation -from paxman.core.domain import Grammar, RecognitionMatch - -# "00" followed by the international number digits (optional separators). -# The leading digit of the number must be 1-9 (country codes never start -# with 0), and a single "0" alone is not the international prefix. -# Separators between "00" and the first digit are allowed ("00 44 ..."). -# The (? PhoneNotation: + """Map a 00-prefixed match to its digit-only E.164 notation (prefix stripped).""" + return PhoneNotation(shape="e164", value=strip_separators(match.group(0)[2:])) + -class International00Grammar(Grammar[PhoneNotation]): +class International00Grammar(PipelineGrammar[PhoneNotation]): """Recognizes international numbers written with the 00 prefix. Examples: "00 44 20 7946 0958", "00442079460958" @@ -40,23 +50,7 @@ class International00Grammar(Grammar[PhoneNotation]): semantics = "e164_international" single_value = True - def recognize(self, text: str) -> list[RecognitionMatch[PhoneNotation]]: - """Extract 00-prefixed international patterns from text. - - Returns: - List of RecognitionMatches; notation.value is the digit-only - number with the "00" prefix stripped (the E.164 number itself). - """ - return [ - RecognitionMatch( - notation=PhoneNotation( - shape="e164", - # Strip the leading "00" before removing separators. - value=strip_separators(match.group(0)[2:]), - ), - start=match.start(), - end=match.end(), - raw_text=match.group(0), - ) - for match in _INTERNATIONAL_00_PATTERN.finditer(text) - ] + pre = StandardPre[PhoneNotation](empty_guard=True) + regex = RegexStage[PhoneNotation]( + pattern=_INTERNATIONAL_00_PATTERN, notation_fn=_international_00_notation + ) diff --git a/paxman/capabilities/Phone/grammar/national_recognition.py b/paxman/capabilities/Phone/grammar/national_recognition.py index 2126a390..6b65c804 100644 --- a/paxman/capabilities/Phone/grammar/national_recognition.py +++ b/paxman/capabilities/Phone/grammar/national_recognition.py @@ -1,4 +1,4 @@ -"""NANP national number recognition grammar. +"""NANP national number recognition grammar (staged pipeline). Recognizes domestic (NANP-style) dialing formats: optional trunk "1", optional parenthesized NPA, then 3-3-4 digit groups with any of space, @@ -10,36 +10,39 @@ import re -from paxman.capabilities.Phone.grammar.common import strip_separators +from paxman.capabilities.Phone.grammar._common import strip_separators from paxman.capabilities.Phone.notation import PhoneNotation -from paxman.core.domain import Grammar, RecognitionMatch +from paxman.core.grammar import ( + BoundaryGuard, + PipelineGrammar, + RegexStage, + StandardPre, +) # Optional trunk 1, optional (NPA), NXX, XXXX. NPA first digit 2-9 is a # recognition heuristic — strict validation (including NXX first digit 2-9) # happens in the rules. NXX is deliberately loose here so the grammar -# recognizes the NANP *shape* even for unassignable exchanges (e.g., +# recognizes the NANP *shape* even for unassignable exchanges (e.g. # "555-123-4567"), which the NANP rule then rejects as INVALID. # -# Four fixed-width negative lookbehinds ensure this grammar does NOT match -# inside E.164 numbers or tel: URIs (those belong to the e164 / tel-URI -# grammars). They reject a match when the characters immediately before it -# belong to an international number: -# 1. digit or "+" -> "+15551234567" (compact) -# 2. separator after d/+ -> "+1-555-123-4567", "+1 555 123 4567", "+1.555..." -# 3. "( " after sep after d/+ -> "+1 (555) 123-4567" (parens w/ separator) -# 4. "(" directly after d/+ -> "+1(555)123-4567" (parens, no separator) -# -# No-plus local tel: URIs ("tel:212-555-6789") are NOT global numbers -# (RFC 3966 §3.1) and are not recognized by the tel-URI grammar; their -# NANP-shaped number content may be recognized here as a national number. -_NATIONAL_PATTERN = re.compile( - r"(? PhoneNotation: + """Map a national match to its digit-only notation (trunk 1 preserved).""" + return PhoneNotation(shape="national", value=strip_separators(match.group(0))) + + +class NationalGrammar(PipelineGrammar[PhoneNotation]): """Recognizes NANP national dialing formats. Examples: "(555) 123-4567", "555-123-4567", "1-555-123-4567" @@ -50,21 +53,7 @@ class NationalGrammar(Grammar[PhoneNotation]): semantics = "national_recognition" single_value = True - def recognize(self, text: str) -> list[RecognitionMatch[PhoneNotation]]: - """Extract national patterns from text. - - Returns: - List of RecognitionMatches; notation.value is the digit-only - number; a leading trunk "1" is preserved when present. - """ - return [ - RecognitionMatch( - notation=PhoneNotation( - shape="national", value=strip_separators(match.group(0)) - ), - start=match.start(), - end=match.end(), - raw_text=match.group(0), - ) - for match in _NATIONAL_PATTERN.finditer(text) - ] + pre = StandardPre[PhoneNotation](empty_guard=True) + regex = RegexStage[PhoneNotation]( + pattern=_NATIONAL_PATTERN, notation_fn=_national_notation + ) diff --git a/paxman/capabilities/Phone/grammar/tel_uri_recognition.py b/paxman/capabilities/Phone/grammar/tel_uri_recognition.py index 0d829e65..4707cd5a 100644 --- a/paxman/capabilities/Phone/grammar/tel_uri_recognition.py +++ b/paxman/capabilities/Phone/grammar/tel_uri_recognition.py @@ -1,24 +1,43 @@ -"""RFC 3966 tel-URI recognition grammar.""" +"""RFC 3966 tel-URI recognition grammar (staged pipeline). + +Recognizes a tel: URI with a GLOBAL number (optional separators) and optional +";ext=" parameter. Per RFC 3966 §3.1 global numbers REQUIRE a leading "+" — +no-plus URIs are local numbers (out of scope), so this grammar does not match +them. The scheme is matched case-insensitively; the leading lookbehind (via +BoundaryGuard.word_only()) keeps "xtel:"/"hotel:" from matching the scheme. +""" from __future__ import annotations import re -from paxman.capabilities.Phone.grammar.common import strip_separators +from paxman.capabilities.Phone.grammar._common import strip_separators from paxman.capabilities.Phone.notation import PhoneNotation -from paxman.core.domain import Grammar, RecognitionMatch - -# tel: URI with a GLOBAL number (optional separators) and optional ";ext=" -# parameter. Per RFC 3966 §3.1 global numbers REQUIRE a leading "+" — -# no-plus URIs are local numbers (out of scope), so this grammar does not -# match them. The scheme is matched case-insensitively; the (? PhoneNotation: + """Map a tel: URI match to its digit-only notation + extension.""" + return PhoneNotation( + shape="rfc3966", + value=strip_separators(match.group(1), plus=True), + extension=match.group(2) or "", + ) + -class TelUriGrammar(Grammar[PhoneNotation]): +class TelUriGrammar(PipelineGrammar[PhoneNotation]): """Recognizes RFC 3966 tel: URIs. Examples: "tel:+15551234567", "tel:+1-201-555-0123;ext=890" @@ -29,24 +48,7 @@ class TelUriGrammar(Grammar[PhoneNotation]): semantics = "tel_uri_recognition" single_value = True - def recognize(self, text: str) -> list[RecognitionMatch[PhoneNotation]]: - """Extract tel: URI patterns from text. - - Returns: - List of RecognitionMatches; notation.value is the digit-only - number (leading "+" and separators removed); notation.extension - is the ";ext=" parameter value if present. - """ - return [ - RecognitionMatch( - notation=PhoneNotation( - shape="rfc3966", - value=strip_separators(match.group(1), plus=True), - extension=match.group(2) or "", - ), - start=match.start(), - end=match.end(), - raw_text=match.group(0), - ) - for match in _TEL_URI_PATTERN.finditer(text) - ] + pre = StandardPre[PhoneNotation](empty_guard=True) + regex = RegexStage[PhoneNotation]( + pattern=_TEL_URI_PATTERN, notation_fn=_tel_notation, flags=re.IGNORECASE + ) diff --git a/paxman/capabilities/SIUnit/grammar/compound_recognition.py b/paxman/capabilities/SIUnit/grammar/compound_recognition.py index cba8d7f5..dd2c5c55 100644 --- a/paxman/capabilities/SIUnit/grammar/compound_recognition.py +++ b/paxman/capabilities/SIUnit/grammar/compound_recognition.py @@ -1,4 +1,4 @@ -"""Compound recognition grammar for SI Unit. +"""Compound recognition grammar for SI Unit (staged pipeline). Recognizes product/quotient compound shapes over unit symbols: UNIT (separator UNIT){1,3} where each UNIT is a symbol character run with an @@ -6,6 +6,11 @@ grammar is shape-only: it does not validate that the units are known (the ISO 80000-1 rule does that). "m/s²", "N·m", "kg·m/s²", "g/cm³" are recognized as single spans; "m s" (space) is not a compound. + +The single bespoke regex from the legacy grammar is reproduced exactly +as a ``RegexStage`` body guarded by ``BoundaryGuard.word_sign()`` (the +compound boundary has no degree prefix, matching the legacy +lookarounds). The notation factory emits the compound body verbatim. """ from __future__ import annotations @@ -17,7 +22,7 @@ EXPONENT_CHARACTERS, ) from paxman.capabilities.SIUnit.notation import SIUnitNotation -from paxman.core.domain import Grammar, RecognitionMatch +from paxman.core.grammar import BoundaryGuard, PipelineGrammar, RegexStage, StandardPre # Shape constants come from the Task 4 generated module (grammars may import # from grammar/data/ — only rules are barred by the grammar↔rules purity @@ -30,25 +35,35 @@ # disambiguation for a solidus followed by another separator, so a # parenthesized denominator is a single compound factor (e.g. "(m·s²)"). _FACTOR = rf"(?:{_UNIT}|\({_UNIT}(?:{_SEP}{_UNIT}){{0,3}}\))" -_COMPOUND_RE = re.compile( - rf"(?{_FACTOR}(?:{_SEP}{_FACTOR}){{1,3}})(?![\w\-+\u2212])" +# The word_sign guard reproduces the legacy compound lookarounds exactly +# (no degree prefix in the compound boundary) — no hard-coded lookaround in +# this file (ADR-0008 D5). +_GUARD = BoundaryGuard.word_sign() +_COMPOUND_PATTERN = ( + _GUARD.lookbehind + + r"(?P" + + _FACTOR + + r"(?:" + + _SEP + + _FACTOR + + r"){1,3})" + + _GUARD.lookahead ) -class CompoundRecognition(Grammar[SIUnitNotation]): +def _compound_notation(match: re.Match[str]) -> SIUnitNotation: + """Emit the compound body verbatim with shape "compound".""" + return SIUnitNotation(text=match.group("body"), shape="compound") + + +class CompoundRecognition(PipelineGrammar[SIUnitNotation]): """Grammar: compound_recognition — product/quotient unit shapes.""" name = "compound_recognition" semantics = "compound_recognition" # SEAM (ADR-0003): identity id - def recognize(self, text: str) -> list[RecognitionMatch[SIUnitNotation]]: - """Emit one RecognitionMatch per compound shape found in text.""" - return [ - RecognitionMatch( - raw_text=text[m.start() : m.end()], - start=m.start(), - end=m.end(), - notation=SIUnitNotation(text=m.group("body"), shape="compound"), - ) - for m in _COMPOUND_RE.finditer(text) - ] + pre = StandardPre[SIUnitNotation](empty_guard=True) + regex = RegexStage[SIUnitNotation]( + pattern=_COMPOUND_PATTERN, + notation_fn=_compound_notation, + ) diff --git a/paxman/capabilities/SIUnit/grammar/name_recognition.py b/paxman/capabilities/SIUnit/grammar/name_recognition.py index d95ac52d..8c4d903e 100644 --- a/paxman/capabilities/SIUnit/grammar/name_recognition.py +++ b/paxman/capabilities/SIUnit/grammar/name_recognition.py @@ -1,9 +1,15 @@ -"""Name recognition grammar for SI Unit. +"""Name recognition grammar for SI Unit (staged pipeline). Recognizes unit names case-insensitively: the grammar folds the input to lowercase and matches against the longest-first name token table (D4). "Kilogram", "KILOGRAM", "kilogram" all emit a RecognitionMatch over the span of the name text. Recognition only: no validation. + +The single bespoke regex from the legacy grammar is reproduced exactly +as a ``RegexStage`` body (with ``re.IGNORECASE``) guarded by +``BoundaryGuard.degree_word_sign()``. The split-prefix shape +("kilo gram" -> ``split_word_prefix``) is computed inline in the +notation factory, byte-identically to the legacy ``recognize()``. """ from __future__ import annotations @@ -13,7 +19,7 @@ from paxman.capabilities.SIUnit.grammar.data.prefix_tokens import PREFIX_WORD_TOKENS from paxman.capabilities.SIUnit.grammar.data.unit_name_tokens import NAME_TOKENS from paxman.capabilities.SIUnit.notation import SIUnitNotation -from paxman.core.domain import Grammar, RecognitionMatch +from paxman.core.grammar import BoundaryGuard, PipelineGrammar, RegexStage, StandardPre # Prefix words for O(1) split detection (see recognize). PREFIX_WORDS = frozenset(PREFIX_WORD_TOKENS) @@ -23,8 +29,6 @@ # adjacent word/separator/sign. Digits and underscore are blocked too (via # \w), keeping quantity-adjacent names (e.g. "5kilogram") from being # recognized — consistent with the symbol grammar's digit boundary. -_LOOKBEHIND = r"(?" + _NAME_BODY + r")" + _LOOKAHEAD, re.IGNORECASE -) +# The degree_word_sign guard preserves the ° in the lookbehind exactly as the +# legacy grammar's _LOOKBEHIND/_LOOKAHEAD literals did — no hard-coded +# lookaround in this file (ADR-0008 D5). re.IGNORECASE reproduces the legacy +# case-insensitive name match. +_GUARD = BoundaryGuard.degree_word_sign() +_NAME_PATTERN = _GUARD.lookbehind + r"(?P" + _NAME_BODY + r")" + _GUARD.lookahead + + +def _name_notation(match: re.Match[str]) -> SIUnitNotation: + """Map a matched name token to its split/attached notation. + + Mirrors the legacy recognize(): the matched text is folded to lowercase; + a known prefix word followed by a space and a unit name (e.g. "kilo gram") + is a rejectable split, while a multi-word unit name like "degree celsius" + is a plain name (its first word is not a prefix). + """ + token = match.group("tok").lower() + parts = token.split() + if len(parts) >= 2 and parts[0] in PREFIX_WORDS: + shape = "split_word_prefix" + else: + shape = "name" + return SIUnitNotation(text=token, shape=shape) -class NameRecognition(Grammar[SIUnitNotation]): +class NameRecognition(PipelineGrammar[SIUnitNotation]): """Grammar: name_recognition — case-folded unit names.""" name = "name_recognition" semantics = "name_recognition" # SEAM (ADR-0003): identity id - def recognize(self, text: str) -> list[RecognitionMatch[SIUnitNotation]]: - """Emit one RecognitionMatch per unit name (or split prefix) found.""" - matches: list[RecognitionMatch[SIUnitNotation]] = [] - for m in _NAME_RE.finditer(text): - token = m.group("tok").lower() - parts = token.split() - # A split prefix is a known prefix word followed by a space and a - # unit name (e.g. "kilo gram"); a multi-word unit name like - # "degree celsius" is NOT a split (its first word is not a prefix). - if len(parts) >= 2 and parts[0] in PREFIX_WORDS: - shape = "split_word_prefix" - else: - shape = "name" - matches.append( - RecognitionMatch( - raw_text=text[m.start() : m.end()], - start=m.start(), - end=m.end(), - notation=SIUnitNotation(text=token, shape=shape), - ) - ) - return matches + pre = StandardPre[SIUnitNotation](empty_guard=True) + regex = RegexStage[SIUnitNotation]( + pattern=_NAME_PATTERN, + notation_fn=_name_notation, + flags=re.IGNORECASE, + ) diff --git a/paxman/capabilities/SIUnit/grammar/symbol_recognition.py b/paxman/capabilities/SIUnit/grammar/symbol_recognition.py index ec902ef3..2f9b3f07 100644 --- a/paxman/capabilities/SIUnit/grammar/symbol_recognition.py +++ b/paxman/capabilities/SIUnit/grammar/symbol_recognition.py @@ -1,10 +1,16 @@ -"""Symbol recognition grammar for SI Unit. +"""Symbol recognition grammar for SI Unit (staged pipeline). Recognizes unit symbols exactly as written (case-exact): base symbols ("m", "kg"), derived special-name symbols ("Pa", "°C"), non-SI symbols ("min", "L"), prefix symbols ("k", "M") and prefixed units ("km", "MHz"). Each recognition emits a span-bearing RecognitionMatch over the symbol text. Recognition only: no validation, no canonicalization (D1/D2/D6). + +The single bespoke regex from the legacy grammar is reproduced exactly +as a ``RegexStage`` body guarded by ``BoundaryGuard.degree_word_sign()`` +(preserving the ``°`` degree prefix in the lookbehind). The split-prefix +shape ("k g" -> ``split_symbol_prefix``) is computed inline in the +notation factory, byte-identically to the legacy ``recognize()``. """ from __future__ import annotations @@ -14,7 +20,7 @@ from paxman.capabilities.SIUnit.grammar.data.prefix_tokens import PREFIX_SYMBOL_TOKENS from paxman.capabilities.SIUnit.grammar.data.unit_symbol_tokens import SYMBOL_TOKENS from paxman.capabilities.SIUnit.notation import SIUnitNotation -from paxman.core.domain import Grammar, RecognitionMatch +from paxman.core.grammar import BoundaryGuard, PipelineGrammar, RegexStage, StandardPre # Prefix symbols that coincide with a standalone unit symbol (a fixed SI # fact): "m" (milli + metre), "h" (hecto + hour), "a" (atto + annum/are), @@ -35,8 +41,6 @@ # lookbehind and the right a lookahead — a lookahead placed before the # token would see the token's own first character (a word char) and # reject the match before the token is even consumed. -_LOOKBEHIND = r"(?" + _SYMBOL_BODY + r")" + _LOOKAHEAD) +# The degree_word_sign guard preserves the ° in the lookbehind exactly as the +# legacy grammar's _LOOKBEHIND/_LOOKAHEAD literals did — no hard-coded +# lookaround in this file (ADR-0008 D5). +_GUARD = BoundaryGuard.degree_word_sign() +_SYMBOL_PATTERN = ( + _GUARD.lookbehind + r"(?P" + _SYMBOL_BODY + r")" + _GUARD.lookahead +) + + +def _symbol_notation(match: re.Match[str]) -> SIUnitNotation: + """Map a matched symbol token to its split/attached notation. + + Mirrors the legacy recognize(): a prefix-only symbol followed by a space + and a unit symbol (e.g. "k g") is a rejectable split; otherwise the token + is an attached symbol. Case is preserved (symbols are case-exact). + """ + token = match.group("tok") + parts = token.split() + if len(parts) >= 2 and parts[0] in PREFIX_ONLY_SYMBOLS: + shape = "split_symbol_prefix" + else: + shape = "symbol" + return SIUnitNotation(text=token, shape=shape) -class SymbolRecognition(Grammar[SIUnitNotation]): +class SymbolRecognition(PipelineGrammar[SIUnitNotation]): """Grammar: symbol_recognition — case-exact unit symbol tokens.""" name = "symbol_recognition" semantics = "symbol_recognition" # SEAM (ADR-0003): identity id - def recognize(self, text: str) -> list[RecognitionMatch[SIUnitNotation]]: - """Emit one RecognitionMatch per unit symbol (or split prefix) found.""" - matches: list[RecognitionMatch[SIUnitNotation]] = [] - for m in _SYMBOL_RE.finditer(text): - token = m.group("tok") - parts = token.split() - # A split prefix is a prefix-only symbol followed by a space and a - # unit symbol (e.g. "k g"); dual-role symbols (m, h, a, d) are not - # prefix-only, so a spaced "m s" stays two units (metre second). - # A symbol token never contains a space otherwise. - if len(parts) >= 2 and parts[0] in PREFIX_ONLY_SYMBOLS: - shape = "split_symbol_prefix" - else: - shape = "symbol" - matches.append( - RecognitionMatch( - raw_text=text[m.start() : m.end()], - start=m.start(), - end=m.end(), - notation=SIUnitNotation(text=token, shape=shape), - ) - ) - return matches + pre = StandardPre[SIUnitNotation](empty_guard=True) + regex = RegexStage[SIUnitNotation]( + pattern=_SYMBOL_PATTERN, + notation_fn=_symbol_notation, + ) diff --git a/paxman/capabilities/URL/grammar/absolute_uri_recognition.py b/paxman/capabilities/URL/grammar/absolute_uri_recognition.py index 59024a60..40924d82 100644 --- a/paxman/capabilities/URL/grammar/absolute_uri_recognition.py +++ b/paxman/capabilities/URL/grammar/absolute_uri_recognition.py @@ -1,9 +1,14 @@ -"""Absolute-URI recognition grammar for the URL capability. +"""Absolute-URI recognition grammar for the URL capability (staged pipeline). -Recognizes absolute-URI/IRI spans (RFC 3986 §4.2, RFC 3987 §2.2) as -scheme-anchored shape matches. Shape-only per D7/D8: validity is the -rule layer's job — the grammar never validates the scheme, host, or -port, and carries no scheme table. +Recognizes absolute-URI/IRI spans (RFC 3986 section 4.2, RFC 3987 section +2.2) as scheme-anchored shape matches. Shape-only per D7/D8: validity is the +rule layer's job — the grammar never validates the scheme, host, or port, +and carries no scheme table. + +The leading lookbehind (via BoundaryGuard.scheme_char()) rejects a scheme +preceded by a scheme-legal character. A PostStage applies the Appendix C +paren-balance trim and the D16 bare-scheme drop (ADR-0008 S5) so the emitted +span is byte-identical to the legacy recognize(). """ from __future__ import annotations @@ -11,66 +16,69 @@ import re from paxman.capabilities.URL.notation import URLNotation -from paxman.core.domain import Grammar, RecognitionMatch +from paxman.core.domain import RecognitionMatch +from paxman.core.grammar import ( + BoundaryGuard, + PipelineGrammar, + PostStage, + RegexStage, + StandardPre, +) -# Scheme anchor (RFC 3986 §3.1 / RFC 3987 §2.1): -# ALPHA *( ALPHA / DIGIT / "+" / "-" / "." ) ":" -# Left boundary: not preceded by a scheme-legal character (word rejection). -# Body: URI/IRI code points (RFC 3986 §2 + RFC 3987 §2.2 ucschar) plus -# tab/newline (Appendix C multi-line URIs); at least ONE body character -# after the colon (D16). -# Right boundary: whitespace, control characters (except tab/newline), -# "<", ">", '"' (Appendix C delimiters). Trailing "." kept. -_ABSOLUTE_URI_PATTERN = re.compile( - r"(?"\x00-\x08\x0B\x0C\x0E-\x1F\x7F]*[^ <>"\x00-\x08\x0B\x0C\x0E-\x1F\x7F]' ) +_GUARD = BoundaryGuard.scheme_char() +_URL_PATTERN = _GUARD.lookbehind + _URL_BODY + + +def _url_notation(match: re.Match[str]) -> URLNotation: + """Map a raw absolute-URI match to its verbatim-text notation.""" + return URLNotation(text=match.group(0)) + +def _url_trim( + match: RecognitionMatch[URLNotation], +) -> RecognitionMatch[URLNotation] | None: + """Appendix C paren-balance trim + D16 bare-scheme drop. -class AbsoluteUriRecognition(Grammar[URLNotation]): + Drops trailing ")" only while it outnumbers "(" (counting once then + trimming the run equals the legacy loop in one pass). After trimming, + a span reduced to the bare scheme (no body past the colon) is dropped + entirely (D16) — it is not a valid absolute-URI match. + """ + raw_span = match.raw_text + excess = raw_span.count(")") - raw_span.count("(") + trim = 0 + while trim < excess and raw_span[-(trim + 1)] == ")": + trim += 1 + if trim: + raw_span = raw_span[:-trim] + scheme_end = raw_span.find(":") + if len(raw_span) <= scheme_end + 1: + return None + return RecognitionMatch( + notation=URLNotation(text=raw_span), + start=match.start, + end=match.start + len(raw_span), + raw_text=raw_span, + ) + + +class AbsoluteUriRecognition(PipelineGrammar[URLNotation]): """Absolute-URI recognition: extracts scheme-anchored URI spans.""" name = "absolute_uri_recognition" semantics = "absolute_uri_recognition" single_value = True - def recognize(self, text: str) -> list[RecognitionMatch[URLNotation]]: - """Extract absolute-URI spans from text. - - Shape-only recognition (D7/D8): any syntactically scheme-anchored - absolute reference is emitted as a span; validity is decided by - the rule layer, never here. - - Returns: - Span-bearing RecognitionMatch per absolute-URI occurrence. - """ - results: list[RecognitionMatch[URLNotation]] = [] - for match in _ABSOLUTE_URI_PATTERN.finditer(text): - raw_span = match.group(0) - # Appendix C: drop trailing ")" only while it outnumbers "("; - # counting once then trimming the run equals the loop, in one pass. - excess = raw_span.count(")") - raw_span.count("(") - trim = 0 - while trim < excess and raw_span[-(trim + 1)] == ")": - trim += 1 - if trim: - raw_span = raw_span[:-trim] - # D16: the pattern guarantees at least one body character after - # the colon, and stripping only removes trailing ")" — so a span - # reduced to the bare scheme has lost its body and must not be - # emitted as an absolute-URI match. - scheme_end = raw_span.find(":") - if len(raw_span) <= scheme_end + 1: - continue - start = match.start() - end = start + len(raw_span) - results.append( - RecognitionMatch( - notation=URLNotation(text=raw_span), - start=start, - end=end, - raw_text=raw_span, - ) - ) - return results + pre = StandardPre[URLNotation](empty_guard=True) + regex = RegexStage[URLNotation](pattern=_URL_PATTERN, notation_fn=_url_notation) + post = PostStage[URLNotation](transform=_url_trim) diff --git a/paxman/core/grammar/__init__.py b/paxman/core/grammar/__init__.py new file mode 100644 index 00000000..133d895d --- /dev/null +++ b/paxman/core/grammar/__init__.py @@ -0,0 +1,31 @@ +"""Recognition-layer pipeline internals (capability-agnostic).""" + +from __future__ import annotations + +from paxman.core.grammar.boundary import BoundaryGuard +from paxman.core.grammar.composer import AmountComposer +from paxman.core.grammar.lexicon import LexiconAlternation +from paxman.core.grammar.pipeline import PipelineGrammar +from paxman.core.grammar.stages import ( + LexiconStage, + PipelineState, + PostStage, + RegexStage, + Stage, + StandardPre, + WholeInputLookup, +) + +__all__ = [ + "AmountComposer", + "BoundaryGuard", + "LexiconAlternation", + "LexiconStage", + "PipelineGrammar", + "PipelineState", + "PostStage", + "RegexStage", + "Stage", + "StandardPre", + "WholeInputLookup", +] diff --git a/paxman/core/grammar/boundary.py b/paxman/core/grammar/boundary.py new file mode 100644 index 00000000..cf9f10e0 --- /dev/null +++ b/paxman/core/grammar/boundary.py @@ -0,0 +1,110 @@ +"""BoundaryGuard family — parameterized lookarounds replacing 10 distinct literals. + +Each guard produces a compiled alternation-ready regex via `wrap(alternation)`, +or exposes its `(lookbehind, lookahead)` pair for `LexiconStage` injection. +No grammar file hard-codes a lookaround literal after migration — each grammar +references a `BoundaryGuard` instance (ADR-0008 D5). +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass + + +@dataclass(frozen=True, slots=True) +class BoundaryGuard: + """A parameterized boundary guard producing lookaround-wrapped patterns.""" + + lookbehind: str + lookahead: str + + def wrap(self, alternation: str, flags: int = 0) -> re.Pattern[str]: + """Wrap an alternation with this guard's lookarounds and compile. + + Args: + alternation: The escaped ``|``-joined token alternation. + flags: Optional ``re`` flags (e.g. ``re.IGNORECASE``) passed to + ``re.compile`` so case-insensitive lexicon grammars (Currency + word, Money word) preserve the old ``re.IGNORECASE`` behavior. + """ + return re.compile(rf"{self.lookbehind}(?:{alternation}){self.lookahead}", flags) + + # Factory constructors — one per distinct semantic variant. + @classmethod + def word_sign(cls) -> BoundaryGuard: + return cls(lookbehind=r"(? BoundaryGuard: + # SIUnit degree prefix: ° must be preserved in the lookbehind. + return cls( + lookbehind=r"(? BoundaryGuard: + return cls(lookbehind=r"(? BoundaryGuard: + return cls(lookbehind=r"(? BoundaryGuard: + return cls(lookbehind=r"(? BoundaryGuard: + """International 00-prefix: like e164() but also excludes a leading "+". + + Contradictory "+00..." input is left to the e164 grammar; the 00 + grammar must not treat it as a 00-prefixed number. + """ + return cls(lookbehind=r"(? BoundaryGuard: + return cls(lookbehind=r"(? BoundaryGuard: + # 4-lookbehind chain: blocks a national number that is itself preceded + # by a digit/+, a separator, an opening paren, or a digit/paren pair. + return cls( + lookbehind=( + r"(? BoundaryGuard: + # Token boundary for IPv6: start/end of string or a delimiter class. + # NOTE: Unlike the other guards, this is a *consuming* anchor pair + # (``(?:^|(?<=...))`` / ``(?:$|(?=...))``) rather than a zero-width + # lookaround. It cannot be used interchangeably with + # LexiconStage.wrap's lookaround mental model; it is a token + # delimiter for RegexStage only. Parity-proven against the legacy + # ``(?:^|(?<=[\\s,;([ ]))`` / ``(?:$|(?=[\\s,;().\\]]))`` idiom. + return cls( + lookbehind=r"(?:^|(?<=[\s,;([ ]))", + lookahead=r"(?:$|(?=[\s,;().\]]))", + ) + + @classmethod + def isbn_trail(cls) -> BoundaryGuard: + # Trailing guard for ISBN-13/ISBN-10: the address must not be + # immediately preceded by a separator (whitespace, colon, hyphen), + # which would mean it is glued to surrounding label text. + return cls(lookbehind=r"(? BoundaryGuard: + # Leading guard for ISBN-10: the address must not be immediately + # preceded by a digit or a digit followed by a separator, which would + # mean it is glued to surrounding digits (e.g. an ID run). + return cls(lookbehind=r"(?{ALT}) ?(?P{pattern})| + (?P{pattern}) ?(?P{ALT})){lookahead}`` and, for + each match, emits a ``RecognitionMatch`` via ``notation_fn``. + + Attributes: + pattern: The amount sub-pattern (e.g. ``AMOUNT_PATTERN``), supplied + by the caller. Interpolated **twice** (prefix and suffix + branches), so it **must not contain named capturing groups** or + rely on numbered-group indices — violations can fail during regex + compilation (duplicate group names) or silently shift indices. + Keep it group-free or use non-capturing ``(?:...)`` only. + boundary: Boundary guard supplying the lookbehind/lookahead pair. + Required — a missing guard would silently emit zero matches. + lexicon_tokens: The currency lexicon (SYMBOL_TOKENS, WORD_TOKENS) or + ``None`` for the code case, where the alternation is the fixed + ``[A-Z]{3}``. + notation_fn: ``(lex, amount, amount_shape) -> Notation`` builder. + classify: Amount-shape classifier (e.g. ``classify_amount_shape``). + flags: ``re`` flags passed to ``re.compile`` (e.g. ``re.IGNORECASE`` + for the word case). + """ + + pattern: str + boundary: BoundaryGuard + lexicon_tokens: list[str] | set[str] | frozenset[str] | tuple[str, ...] | None = ( + None + ) + notation_fn: Callable[[str, str, str], NotationT] | None = None + classify: Callable[[str], str] | None = None + flags: int = 0 + + _compiled: re.Pattern[str] = field(init=False, repr=False) + + def __post_init__(self) -> None: + if self.lexicon_tokens is None: + alt = r"[A-Z]{3}" + else: + alt = LexiconAlternation( + tokens=self.lexicon_tokens, longest_first=True + ).alternation + lookbehind = self.boundary.lookbehind + lookahead = self.boundary.lookahead + fused = ( + rf"{lookbehind}" + rf"(?:(?P{alt}) ?(?P{self.pattern})" + rf"|(?P{self.pattern}) ?(?P{alt}))" + rf"{lookahead}" + ) + object.__setattr__(self, "_compiled", re.compile(fused, self.flags)) + + def run(self, state: PipelineState[NotationT]) -> PipelineState[NotationT]: + if self.notation_fn is None or self.classify is None: + return state + new_matches: list[RecognitionMatch[NotationT]] = list(state.matches) + for m in self._compiled.finditer(state.text): + groups = m.groupdict() + lex = groups["prefix_lex"] or groups["suffix_lex"] + amt = groups["prefix_amt"] or groups["suffix_amt"] + if lex is None or amt is None: + continue + notation = self.notation_fn(lex, amt, self.classify(amt)) + new_matches.append( + RecognitionMatch( + notation=notation, + start=m.start(), + end=m.end(), + raw_text=m.group(0), + ) + ) + return PipelineState( + text=state.text, matches=new_matches, scratch=dict(state.scratch) + ) diff --git a/paxman/core/grammar/lexicon.py b/paxman/core/grammar/lexicon.py new file mode 100644 index 00000000..586ad817 --- /dev/null +++ b/paxman/core/grammar/lexicon.py @@ -0,0 +1,42 @@ +"""LexiconAlternation builder — longest-first, qualified-first, escaped.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field + + +def _is_qualified(token: str) -> bool: + """Return whether a token carries at least one ASCII alphabetic character. + + Mirrors ``Currency/SymbolRecognition._is_qualified``: a token that contains + an ASCII letter (e.g. ``"US$"``) is "qualified" and sorts ahead of a bare + symbol (e.g. ``"$"``) of the same length, so the longer/qualified form is + tried first by the regex engine and wins the match. + """ + return any(c.isascii() and c.isalpha() for c in token) + + +@dataclass(frozen=True, slots=True) +class LexiconAlternation: + """Builds a longest-first, qualified-first escaped alternation. + + Tokens are sorted by ``(-len(token), -is_qualified, token)`` and joined with + ``|`` after ``re.escape``. This reproduces the ``SYMBOL_TOKENS`` duplication + between Currency and Money without per-file copy-paste (ADR-0008 D4). + """ + + tokens: frozenset[str] | set[str] | list[str] | tuple[str, ...] + longest_first: bool = True + + ordered_tokens: list[str] = field(init=False) + alternation: str = field(init=False) + + def __post_init__(self) -> None: + toks = list(self.tokens) + if self.longest_first: + toks.sort(key=lambda t: (-len(t), -int(_is_qualified(t)), t)) + else: + toks.sort() + object.__setattr__(self, "ordered_tokens", toks) + object.__setattr__(self, "alternation", "|".join(re.escape(t) for t in toks)) diff --git a/paxman/core/grammar/pipeline.py b/paxman/core/grammar/pipeline.py new file mode 100644 index 00000000..1137b360 --- /dev/null +++ b/paxman/core/grammar/pipeline.py @@ -0,0 +1,45 @@ +"""PipelineGrammar base — fixed-order pipeline with optional stages.""" + +from __future__ import annotations + +from typing import ClassVar, TypeVar + +from paxman.core.domain import Grammar, RecognitionMatch +from paxman.core.grammar.stages import PipelineState, Stage + +NotationT = TypeVar("NotationT") + + +class PipelineGrammar(Grammar[NotationT]): + """Grammar that declares optional stages; recognize() walks them in fixed order.""" + + # Placeholder semantics for the abstract base; concrete grammars override it + # (Grammar.__init_subclass__ requires a non-empty semantics at class-def time). + semantics: ClassVar[str] = "pipeline_grammar" + + # Stages — each is Optional[Stage]; None means "skip". + pre: Stage[NotationT] | None = None + regex: Stage[NotationT] | None = None + lexicon: Stage[NotationT] | None = None + composer: Stage[NotationT] | None = None + post: Stage[NotationT] | None = None + + def recognize(self, text: str) -> list[RecognitionMatch[NotationT]]: + state: PipelineState[NotationT] = PipelineState( + text=text, matches=[], scratch={} + ) + # Pre short-circuit: if StandardPre emptied matches on whitespace-only + # input, skip remaining stages — they would find nothing anyway. + if self.pre is not None: + state = self.pre.run(state) + if not state.text.strip() and not state.matches: + return list(state.matches) + for stage in ( + self.regex, + self.lexicon, + self.composer, + self.post, + ): + if stage is not None: + state = stage.run(state) + return list(state.matches) diff --git a/paxman/core/grammar/stages.py b/paxman/core/grammar/stages.py new file mode 100644 index 00000000..009981ff --- /dev/null +++ b/paxman/core/grammar/stages.py @@ -0,0 +1,195 @@ +"""Stage Protocol and concrete stage types for the recognition pipeline.""" + +from __future__ import annotations + +import re +from collections.abc import Callable +from dataclasses import dataclass, field +from typing import Generic, Protocol, TypeVar + +from paxman.core.domain import RecognitionMatch +from paxman.core.grammar.boundary import BoundaryGuard +from paxman.core.grammar.lexicon import LexiconAlternation + +NotationT = TypeVar("NotationT") + + +@dataclass(frozen=True, slots=True) +class PipelineState(Generic[NotationT]): + """Mutable-through-replacement state threaded through stages. + + ``text`` is the original ``recognize()`` input and **must remain unchanged** + through all stages — stages must place normalized or transformed views in + ``scratch`` instead, preserving ``RecognitionMatch`` offsets relative to the + original input. ``matches`` accumulates span-bearing recognitions; + ``scratch`` is stage-local auxiliary storage. + """ + + text: str + matches: list[RecognitionMatch[NotationT]] = field( + default_factory=lambda: list[RecognitionMatch[NotationT]]() + ) + scratch: dict[str, object] = field(default_factory=lambda: dict[str, object]()) + + +class Stage(Protocol[NotationT]): + """Inter-stage contract — each stage consumes and returns a PipelineState. + + Every stage **must return a PipelineState with ``state.text`` unchanged** + from its input. Normalized or transformed text views belong in + ``state.scratch``, not by mutating ``text``, so ``RecognitionMatch`` spans + remain relative to the original ``recognize()`` input. Stages may append + to ``state.matches`` and update ``scratch``. + """ + + def run(self, state: PipelineState[NotationT]) -> PipelineState[NotationT]: ... + + +@dataclass(frozen=True, slots=True) +class StandardPre(Generic[NotationT]): + """Pre-processing stage: empty/whitespace early-exit, optional normalizer.""" + + empty_guard: bool = True + + def run(self, state: PipelineState[NotationT]) -> PipelineState[NotationT]: + if self.empty_guard and not state.text.strip(): + return PipelineState( + text=state.text, matches=[], scratch=dict(state.scratch) + ) + return state + + +@dataclass(frozen=True, slots=True) +class RegexStage(Generic[NotationT]): + """Regex parser stage: pure shape scan via finditer.""" + + pattern: str + notation_fn: Callable[[re.Match[str]], NotationT] | None = None + flags: int = 0 + _compiled: re.Pattern[str] = field(init=False, repr=False) + + def __post_init__(self) -> None: + object.__setattr__(self, "_compiled", re.compile(self.pattern, self.flags)) + + def run(self, state: PipelineState[NotationT]) -> PipelineState[NotationT]: + if self.notation_fn is None: + return state + new_matches: list[RecognitionMatch[NotationT]] = list(state.matches) + for m in self._compiled.finditer(state.text): + notation = self.notation_fn(m) + new_matches.append( + RecognitionMatch( + notation=notation, + start=m.start(), + end=m.end(), + raw_text=m.group(0), + ) + ) + return PipelineState( + text=state.text, matches=new_matches, scratch=dict(state.scratch) + ) + + +@dataclass(frozen=True, slots=True) +class LexiconStage(Generic[NotationT]): + """Lexicon parser stage: alternation scan guarded by a BoundaryGuard.""" + + tokens: frozenset[str] | set[str] | list[str] | tuple[str, ...] + boundary: BoundaryGuard + longest_first: bool = True + notation_fn: Callable[[str], NotationT] | None = None + flags: int = 0 + + _compiled: re.Pattern[str] = field(init=False, repr=False) + + def __post_init__(self) -> None: + alt = LexiconAlternation( + tokens=self.tokens, longest_first=self.longest_first + ).alternation + object.__setattr__(self, "_compiled", self.boundary.wrap(alt, self.flags)) + + def run(self, state: PipelineState[NotationT]) -> PipelineState[NotationT]: + if self.notation_fn is None: + return state + new_matches: list[RecognitionMatch[NotationT]] = list(state.matches) + for m in self._compiled.finditer(state.text): + token = m.group(0) + new_matches.append( + RecognitionMatch( + notation=self.notation_fn(token), + start=m.start(), + end=m.end(), + raw_text=token, + ) + ) + return PipelineState( + text=state.text, matches=new_matches, scratch=dict(state.scratch) + ) + + +@dataclass(frozen=True, slots=True) +class PostStage(Generic[NotationT]): + """Post-processing stage: applies a transform to each emitted match. + + Used for span-trimming behaviors that the regex span alone cannot + express — e.g. the E.164 15-digit window trim (end = start + + len(trimmed_raw)) and the URL paren-balance trim with bare-scheme drop + (D16). The transform maps a ``RecognitionMatch`` to a (possibly + different) ``RecognitionMatch``, or to ``None`` to drop the match + entirely. Capability-specific trim logic lives in the grammar file's + transform closure; this stage stays capability-agnostic. + """ + + transform: Callable[ + [RecognitionMatch[NotationT]], RecognitionMatch[NotationT] | None + ] + + def run(self, state: PipelineState[NotationT]) -> PipelineState[NotationT]: + new_matches: list[RecognitionMatch[NotationT]] = [] + for m in state.matches: + result = self.transform(m) + if result is not None: + new_matches.append(result) + return PipelineState( + text=state.text, matches=new_matches, scratch=dict(state.scratch) + ) + + +@dataclass(frozen=True, slots=True) +class WholeInputLookup(Generic[NotationT]): + """S2 whole-input membership — a LexiconStage variant for Country/name_recognition. + + The entire (trimmed) input is looked up against a set of normalized keys. + The emitted match carries the *original* trimmed text and span (D7), not the + normalized key. ``normalizer`` is required: Country must pass its + ``normalize_name`` so that the lookup key is derived deterministically rather + than by a hard-coded ``lower()`` that would break other capabilities. + """ + + keys: frozenset[str] | set[str] + normalizer: Callable[[str], str] + notation_fn: Callable[[str], NotationT] | None = None + + def run(self, state: PipelineState[NotationT]) -> PipelineState[NotationT]: + if self.notation_fn is None: + return state + trimmed = state.text.strip() + if not trimmed: + return state + normalized = self.normalizer(trimmed) + if normalized in self.keys: + start = len(state.text) - len(state.text.lstrip()) + end = start + len(trimmed) + new_matches: list[RecognitionMatch[NotationT]] = list(state.matches) + new_matches.append( + RecognitionMatch( + notation=self.notation_fn(trimmed), + start=start, + end=end, + raw_text=trimmed, + ) + ) + return PipelineState( + text=state.text, matches=new_matches, scratch=dict(state.scratch) + ) + return state diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/capabilities/country/test_data_consistency.py b/tests/capabilities/country/test_data_consistency.py index 2e31c7b4..883e25c8 100644 --- a/tests/capabilities/country/test_data_consistency.py +++ b/tests/capabilities/country/test_data_consistency.py @@ -32,7 +32,7 @@ from paxman.capabilities.Country.grammar.data.localized_names import ( LOCALIZED_NAME_KEYS, ) -from paxman.capabilities.Country.name_normalization import normalize_name +from paxman.capabilities.Country.notation import normalize_name from paxman.capabilities.Country.rules.data.cldr_ed2025 import ( LOCALIZED_TO_ALPHA2, ) diff --git a/tests/capabilities/country/test_grammar.py b/tests/capabilities/country/test_grammar.py index ce22d1ff..bf455e7c 100644 --- a/tests/capabilities/country/test_grammar.py +++ b/tests/capabilities/country/test_grammar.py @@ -4,8 +4,7 @@ from paxman.capabilities.Country.grammar.alpha3_recognition import Alpha3Grammar from paxman.capabilities.Country.grammar.name_recognition import NameGrammar from paxman.capabilities.Country.grammar.numeric_recognition import NumericGrammar -from paxman.capabilities.Country.name_normalization import normalize_name -from paxman.capabilities.Country.notation import CountryNotation +from paxman.capabilities.Country.notation import CountryNotation, normalize_name class TestAlpha2Grammar: diff --git a/tests/property/_legacy_currency_grammars.py b/tests/property/_legacy_currency_grammars.py new file mode 100644 index 00000000..6fb6e19a --- /dev/null +++ b/tests/property/_legacy_currency_grammars.py @@ -0,0 +1,117 @@ +"""Verbatim legacy Currency grammars (pre-PipelineGrammar migration). + +Snapshot of the bespoke ``recognize()`` logic from commit 1a7c4b2, used by +the Migration Proof Harness (ADR-0008 §4.1) to assert byte-identical +``RecognitionMatch`` output after the staged-pipeline migration. Classes are +renamed ``Legacy*`` to avoid colliding with the migrated grammar classes. + +Do NOT edit by hand — this is a frozen reference. The live grammar files are +the source of truth post-migration; this module exists only so the parity +test can compare old vs new behavior. +""" + +from __future__ import annotations + +import re + +from paxman.capabilities.Currency.grammar.data.currency_symbols import SYMBOL_TOKENS +from paxman.capabilities.Currency.grammar.data.currency_words import WORD_TOKENS +from paxman.capabilities.Currency.notation import CurrencyNotation +from paxman.core.domain import Grammar, RecognitionMatch + +_SYMBOL_ALTERNATION = "|".join(re.escape(token) for token in SYMBOL_TOKENS) +_SYMBOL_PATTERN = re.compile( + rf"(?[A-Za-z]{3})(?![\w\-+\u2212])") + + +def _is_qualified(token: str) -> bool: + """Whether a symbol token carries an ASCII letter (e.g. "US$").""" + return any(char.isascii() and char.isalpha() for char in token) + + +class LegacySymbolRecognition(Grammar[CurrencyNotation]): + """Legacy standalone CLDR currency symbol recognition (verbatim).""" + + name = "symbol_recognition" + semantics = "symbol_recognition" + single_value = True + + def recognize(self, text: str) -> list[RecognitionMatch[CurrencyNotation]]: + if not text.strip(): + return [] + matches: list[RecognitionMatch[CurrencyNotation]] = [] + for match in _SYMBOL_PATTERN.finditer(text): + token = match.group(0) + matches.append( + RecognitionMatch( + notation=CurrencyNotation( + text=token, + shape="qualified_symbol" if _is_qualified(token) else "symbol", + ), + start=match.start(), + end=match.end(), + raw_text=token, + ) + ) + return matches + + +class LegacyCodeRecognition(Grammar[CurrencyNotation]): + """Legacy standalone ISO 4217 alpha-3 code recognition (verbatim).""" + + name = "code_recognition" + semantics = "code_recognition" + single_value = True + + def recognize(self, text: str) -> list[RecognitionMatch[CurrencyNotation]]: + if not text.strip(): + return [] + matches: list[RecognitionMatch[CurrencyNotation]] = [] + for match in _CODE_PATTERN.finditer(text): + matches.append( + RecognitionMatch( + notation=CurrencyNotation( + text=match.group("code").upper(), + shape="code", + ), + start=match.start(), + end=match.end(), + raw_text=match.group(0), + ) + ) + return matches + + +class LegacyWordRecognition(Grammar[CurrencyNotation]): + """Legacy standalone CLDR currency display-name word recognition (verbatim).""" + + name = "word_recognition" + semantics = "word_recognition" + single_value = True + + def recognize(self, text: str) -> list[RecognitionMatch[CurrencyNotation]]: + if not text.strip(): + return [] + matches: list[RecognitionMatch[CurrencyNotation]] = [] + for match in _WORD_PATTERN.finditer(text): + matches.append( + RecognitionMatch( + notation=CurrencyNotation( + text=match.group(0).lower(), + shape="word", + ), + start=match.start(), + end=match.end(), + raw_text=match.group(0), + ) + ) + return matches diff --git a/tests/property/_legacy_money_grammars.py b/tests/property/_legacy_money_grammars.py new file mode 100644 index 00000000..86573110 --- /dev/null +++ b/tests/property/_legacy_money_grammars.py @@ -0,0 +1,153 @@ +"""Verbatim legacy Money grammars (pre-PipelineGrammar migration). + +Snapshot of the bespoke ``recognize()`` logic from commit 1a7c4b2, used by +the Migration Proof Harness (ADR-0008 §4.1) to assert byte-identical +``RecognitionMatch`` output after the staged-pipeline migration. Classes are +renamed ``Legacy*`` to avoid colliding with the migrated grammar classes. + +Do NOT edit by hand — this is a frozen reference. The live grammar files are +the source of truth post-migration; this module exists only so the parity +test can compare old vs new behavior. +""" + +from __future__ import annotations + +import re +from typing import cast + +from paxman.capabilities.Money.grammar import AMOUNT_PATTERN, classify_amount_shape +from paxman.capabilities.Money.grammar.data.currency_symbols import SYMBOL_TOKENS +from paxman.capabilities.Money.grammar.data.currency_words import WORD_TOKENS +from paxman.capabilities.Money.notation import MoneyNotation +from paxman.core.domain import Grammar, RecognitionMatch + +_SYMBOL_ALTERNATION = "|".join(re.escape(token) for token in SYMBOL_TOKENS) +_SYMBOL_PATTERN = re.compile( + rf"(?{_SYMBOL_ALTERNATION})" + rf" ?(?P{AMOUNT_PATTERN})" + rf"|(?P{AMOUNT_PATTERN}) ?(?P{_SYMBOL_ALTERNATION}))" + rf"(?![\w\-+\u2212])" +) + +_WORD_ALTERNATION = "|".join(re.escape(token) for token in WORD_TOKENS) +_WORD_PATTERN = re.compile( + rf"(?{_WORD_ALTERNATION})" + rf" ?(?P{AMOUNT_PATTERN})" + rf"|(?P{AMOUNT_PATTERN}) ?(?P{_WORD_ALTERNATION}))" + rf"(?![\w\-+\u2212])", + re.IGNORECASE, +) + +_CODE_PATTERN = re.compile( + rf"(?[A-Z]{{3}}) ?(?P{AMOUNT_PATTERN})" + rf"|(?P{AMOUNT_PATTERN}) ?(?P[A-Z]{{3}}))" + rf"(?![\w\-+\u2212])" +) + + +def _is_qualified(token: str) -> bool: + """A symbol is qualified when it contains an ASCII letter (US$, RM).""" + return any(ch.isascii() and ch.isalpha() for ch in token) + + +class LegacyCodeRecognition(Grammar[MoneyNotation]): + """Legacy ISO 4217 alpha-3 code + amount recognition (verbatim).""" + + name = "code_recognition" + semantics = "code_recognition" + single_value = True + + def recognize(self, text: str) -> list[RecognitionMatch[MoneyNotation]]: + if not text.strip(): + return [] + matches: list[RecognitionMatch[MoneyNotation]] = [] + for match in _CODE_PATTERN.finditer(text): + currency = cast( + str, match.group("prefix_code") or match.group("suffix_code") + ) + amount = cast( + str, match.group("prefix_amount") or match.group("suffix_amount") + ) + matches.append( + RecognitionMatch( + notation=MoneyNotation( + currency_part=currency, + amount_part=amount, + currency_shape="code", + amount_shape=classify_amount_shape(amount), + ), + start=match.start(), + end=match.end(), + raw_text=match.group(0), + ) + ) + return matches + + +class LegacySymbolRecognition(Grammar[MoneyNotation]): + """Legacy CLDR currency symbol + amount recognition (verbatim).""" + + name = "symbol_recognition" + semantics = "symbol_recognition" + single_value = True + + def recognize(self, text: str) -> list[RecognitionMatch[MoneyNotation]]: + if not text.strip(): + return [] + matches: list[RecognitionMatch[MoneyNotation]] = [] + for match in _SYMBOL_PATTERN.finditer(text): + symbol = cast( + str, match.group("prefix_symbol") or match.group("suffix_symbol") + ) + amount = cast( + str, match.group("prefix_amount") or match.group("suffix_amount") + ) + matches.append( + RecognitionMatch( + notation=MoneyNotation( + currency_part=symbol, + amount_part=amount, + currency_shape=( + "qualified_symbol" if _is_qualified(symbol) else "symbol" + ), + amount_shape=classify_amount_shape(amount), + ), + start=match.start(), + end=match.end(), + raw_text=match.group(0), + ) + ) + return matches + + +class LegacyWordRecognition(Grammar[MoneyNotation]): + """Legacy CLDR currency display-name word + amount recognition (verbatim).""" + + name = "word_recognition" + semantics = "word_recognition" + single_value = True + + def recognize(self, text: str) -> list[RecognitionMatch[MoneyNotation]]: + if not text.strip(): + return [] + matches: list[RecognitionMatch[MoneyNotation]] = [] + for match in _WORD_PATTERN.finditer(text): + word = cast(str, match.group("prefix_word") or match.group("suffix_word")) + amount = cast( + str, match.group("prefix_amount") or match.group("suffix_amount") + ) + matches.append( + RecognitionMatch( + notation=MoneyNotation( + currency_part=word, + amount_part=amount, + currency_shape="word", + amount_shape=classify_amount_shape(amount), + ), + start=match.start(), + end=match.end(), + raw_text=match.group(0), + ) + ) + return matches diff --git a/tests/property/_legacy_phone_url_grammars.py b/tests/property/_legacy_phone_url_grammars.py new file mode 100644 index 00000000..53d665dc --- /dev/null +++ b/tests/property/_legacy_phone_url_grammars.py @@ -0,0 +1,218 @@ +"""Verbatim legacy Phone + URL grammars (pre-PipelineGrammar migration). + +Snapshot of the bespoke ``recognize()`` logic from the branch +refactor/staged-recognition-pipeline (commit eeb9529 SIUnit), used by the +Migration Proof Harness (ADR-0008 §4.1) to assert byte-identical +``RecognitionMatch`` output after the staged-pipeline migration. Classes are +renamed ``Legacy*`` to avoid colliding with the migrated grammar classes. + +Do NOT edit by hand — this is a frozen reference. The live grammar files are +the source of truth post-migration; this module exists only so the parity +test can compare old vs new behavior. +""" + +from __future__ import annotations + +import re + +from paxman.capabilities.Phone.notation import PhoneNotation +from paxman.capabilities.URL.notation import URLNotation +from paxman.core.domain import Grammar, RecognitionMatch + +# Verbatim copy of Phone/grammar/common.py:strip_separators (the legacy helper +# retired in ADR-0008 Task 10) so this frozen reference stays self-contained. +_SEPARATORS = str.maketrans("", "", " ().-") +_SEPARATORS_WITH_PLUS = str.maketrans("", "", "+ ().-") + + +def strip_separators(value: str, *, plus: bool = False) -> str: + """Remove phone separators from a raw match (legacy verbatim).""" + if plus: + return value.translate(_SEPARATORS_WITH_PLUS) + return value.translate(_SEPARATORS) + + +# --------------------------------------------------------------------------- +# Phone / E.164 (verbatim) +# --------------------------------------------------------------------------- + +_E164_PATTERN = re.compile(r"(? str: + """Trim a runaway raw match at the last digit-run group within the limit.""" + runs = list(re.finditer(r"\d+", raw)) + total = 0 + for index, run in enumerate(runs): + total += len(run.group(0)) + if total > _MAX_E164_DIGITS: + if index == 0: + return raw + return raw[: runs[index - 1].end()] + return raw + + +class LegacyE164Grammar(Grammar[PhoneNotation]): + """Recognizes E.164-style international numbers (leading +).""" + + name = "e164_recognition" + semantics = "e164_international" + single_value = True + + def recognize(self, text: str) -> list[RecognitionMatch[PhoneNotation]]: + matches: list[RecognitionMatch[PhoneNotation]] = [] + for match in _E164_PATTERN.finditer(text): + raw_text = _trim_to_e164_boundary(match.group(0)) + matches.append( + RecognitionMatch( + notation=PhoneNotation( + shape="e164", + value=strip_separators(raw_text, plus=True), + ), + start=match.start(), + end=match.start() + len(raw_text), + raw_text=raw_text, + ) + ) + return matches + + +# --------------------------------------------------------------------------- +# Phone / tel-URI (verbatim) +# --------------------------------------------------------------------------- + +_TEL_URI_PATTERN = re.compile( + r"(? list[RecognitionMatch[PhoneNotation]]: + return [ + RecognitionMatch( + notation=PhoneNotation( + shape="rfc3966", + value=strip_separators(match.group(1), plus=True), + extension=match.group(2) or "", + ), + start=match.start(), + end=match.end(), + raw_text=match.group(0), + ) + for match in _TEL_URI_PATTERN.finditer(text) + ] + + +# --------------------------------------------------------------------------- +# Phone / international 00-prefix (verbatim) +# --------------------------------------------------------------------------- + +_INTERNATIONAL_00_PATTERN = re.compile( + r"(? list[RecognitionMatch[PhoneNotation]]: + return [ + RecognitionMatch( + notation=PhoneNotation( + shape="e164", + value=strip_separators(match.group(0)[2:]), + ), + start=match.start(), + end=match.end(), + raw_text=match.group(0), + ) + for match in _INTERNATIONAL_00_PATTERN.finditer(text) + ] + + +# --------------------------------------------------------------------------- +# Phone / national (NANP) (verbatim) +# --------------------------------------------------------------------------- + +_NATIONAL_PATTERN = re.compile( + r"(? list[RecognitionMatch[PhoneNotation]]: + return [ + RecognitionMatch( + notation=PhoneNotation( + shape="national", value=strip_separators(match.group(0)) + ), + start=match.start(), + end=match.end(), + raw_text=match.group(0), + ) + for match in _NATIONAL_PATTERN.finditer(text) + ] + + +# --------------------------------------------------------------------------- +# URL / absolute-URI (verbatim) +# --------------------------------------------------------------------------- + +_ABSOLUTE_URI_PATTERN = re.compile( + r"(?"\x00-\x08\x0B\x0C\x0E-\x1F\x7F]*[^ <>"\x00-\x08\x0B\x0C\x0E-\x1F\x7F]' +) + + +class LegacyAbsoluteUriRecognition(Grammar[URLNotation]): + """Absolute-URI recognition: extracts scheme-anchored URI spans.""" + + name = "absolute_uri_recognition" + semantics = "absolute_uri_recognition" + single_value = True + + def recognize(self, text: str) -> list[RecognitionMatch[URLNotation]]: + results: list[RecognitionMatch[URLNotation]] = [] + for match in _ABSOLUTE_URI_PATTERN.finditer(text): + raw_span = match.group(0) + excess = raw_span.count(")") - raw_span.count("(") + trim = 0 + while trim < excess and raw_span[-(trim + 1)] == ")": + trim += 1 + if trim: + raw_span = raw_span[:-trim] + scheme_end = raw_span.find(":") + if len(raw_span) <= scheme_end + 1: + continue + start = match.start() + end = start + len(raw_span) + results.append( + RecognitionMatch( + notation=URLNotation(text=raw_span), + start=start, + end=end, + raw_text=raw_span, + ) + ) + return results diff --git a/tests/property/_legacy_remaining_grammars.py b/tests/property/_legacy_remaining_grammars.py new file mode 100644 index 00000000..4f33c54f --- /dev/null +++ b/tests/property/_legacy_remaining_grammars.py @@ -0,0 +1,523 @@ +"""Verbatim legacy Date/Email/IP/ISBN/Country grammars (pre-PipelineGrammar migration). + +Snapshot of the bespoke ``recognize()`` logic at the start of Task 9, used by +the Migration Proof Harness (ADR-0008 §4.1) to assert byte-identical +``RecognitionMatch`` output after the staged-pipeline migration. Classes are +renamed ``Legacy*`` to avoid colliding with the migrated grammar classes. + +Do NOT edit by hand — this is a frozen reference. The live grammar files are +the source of truth post-migration; this module exists only so the parity +test can compare old vs new behavior. +""" + +from __future__ import annotations + +import re + +from paxman.capabilities.Country.grammar.data.chinese_names import ( + CHINESE_NAME_KEYS, +) +from paxman.capabilities.Country.grammar.data.english_names import ( + ENGLISH_NAME_KEYS, +) +from paxman.capabilities.Country.grammar.data.historical_names import ( + HISTORICAL_NAME_KEYS, +) +from paxman.capabilities.Country.grammar.data.localized_names import ( + LOCALIZED_NAME_KEYS, +) +from paxman.capabilities.Country.notation import CountryNotation, normalize_name +from paxman.capabilities.Date.notation import DateNotation +from paxman.capabilities.Email.notation import EmailNotation +from paxman.capabilities.IP.notation import IPNotation +from paxman.capabilities.ISBN.notation import ISBNNotation +from paxman.core.domain import Grammar, RecognitionMatch + +# --------------------------------------------------------------------------- +# Date +# --------------------------------------------------------------------------- + +_ISO8601_PATTERN = re.compile(r"(? list[RecognitionMatch[DateNotation]]: + return [ + RecognitionMatch( + notation=DateNotation(N1=year, N2=month, N3=day), + start=match.start(), + end=match.end(), + raw_text=match.group(0), + ) + for match in _ISO8601_PATTERN.finditer(text) + for year, month, day in [match.groups()] + ] + + +class LegacyUSDateGrammar(Grammar[DateNotation]): + """Legacy US date recognition (verbatim).""" + + name = "us_recognition" + semantics = "us_calendar_date" + single_value = True + + def recognize(self, text: str) -> list[RecognitionMatch[DateNotation]]: + results: list[RecognitionMatch[DateNotation]] = [] + for match in _US_DATE_PATTERN_4DIGIT.finditer(text): + month, day, year = match.group(1), match.group(2), match.group(3) + results.append( + RecognitionMatch( + notation=DateNotation(N1=month, N2=day, N3=year), + start=match.start(), + end=match.end(), + raw_text=match.group(0), + ) + ) + for match in _US_DATE_PATTERN_2DIGIT.finditer(text): + month, day, year = match.group(1), match.group(2), match.group(3) + results.append( + RecognitionMatch( + notation=DateNotation(N1=month, N2=day, N3=year), + start=match.start(), + end=match.end(), + raw_text=match.group(0), + ) + ) + return results + + +class LegacyEuropeanDateGrammar(Grammar[DateNotation]): + """Legacy European date recognition (verbatim).""" + + name = "european_recognition" + semantics = "european_calendar_date" + single_value = True + + def recognize(self, text: str) -> list[RecognitionMatch[DateNotation]]: + results: list[RecognitionMatch[DateNotation]] = [] + for match in _EUROPEAN_DATE_PATTERN_4DIGIT.finditer(text): + day, month, year = match.group(1), match.group(2), match.group(3) + results.append( + RecognitionMatch( + notation=DateNotation(N1=day, N2=month, N3=year), + start=match.start(), + end=match.end(), + raw_text=match.group(0), + ) + ) + for match in _EUROPEAN_DATE_PATTERN_2DIGIT.finditer(text): + day, month, year = match.group(1), match.group(2), match.group(3) + results.append( + RecognitionMatch( + notation=DateNotation(N1=day, N2=month, N3=year), + start=match.start(), + end=match.end(), + raw_text=match.group(0), + ) + ) + return results + + +class LegacySlashISODateGrammar(Grammar[DateNotation]): + """Legacy slash-ISO date recognition (verbatim).""" + + name = "slash_iso_recognition" + semantics = "iso8601_calendar_date" + single_value = True + + def recognize(self, text: str) -> list[RecognitionMatch[DateNotation]]: + return [ + RecognitionMatch( + notation=DateNotation(N1=year, N2=month, N3=day), + start=match.start(), + end=match.end(), + raw_text=match.group(0), + ) + for match in _SLASH_ISO_PATTERN.finditer(text) + for year, month, day in [match.groups()] + ] + + +# --------------------------------------------------------------------------- +# Email +# --------------------------------------------------------------------------- + +_STANDARD_PATTERN = re.compile( + r"\b[A-Za-z0-9][A-Za-z0-9._%+-]*@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b" +) + +_OBFUSCATED_PATTERN = re.compile( + r"\b([A-Za-z0-9._%+-]+)\s+at\s+([A-Za-z0-9.-]+)\s+dot\s+([A-Za-z]{2,})\b" +) +_AT_ONLY_PATTERN = re.compile( + r"\b([A-Za-z0-9._%+-]+)\s+at\s+([A-Za-z0-9.-]+\.[A-Za-z]{2,})\b" +) + +_LOCALHOST_PATTERN = re.compile( + r"\b([A-Za-z0-9._%+-]+)@localhost(?::\d+)?(?:(?=[\s,;()]|$)|\.(?=\s|$))", + re.IGNORECASE, +) + + +class LegacyStandardEmailGrammar(Grammar[EmailNotation]): + """Legacy standard email recognition (verbatim).""" + + name = "standard_recognition" + semantics = "rfc5322_addr_spec" + single_value = True + + def recognize(self, text: str) -> list[RecognitionMatch[EmailNotation]]: + matches: list[RecognitionMatch[EmailNotation]] = [] + for match in _STANDARD_PATTERN.finditer(text): + matches.append( + RecognitionMatch( + notation=EmailNotation( + local_part=match.group(0).split("@")[0], + domain_part=match.group(0).split("@")[1], + ), + start=match.start(), + end=match.end(), + raw_text=match.group(0), + ) + ) + return matches + + +class LegacyObfuscatedEmailGrammar(Grammar[EmailNotation]): + """Legacy obfuscated email recognition (verbatim).""" + + name = "obfuscated_recognition" + semantics = "rfc5322_addr_spec" + single_value = True + + def recognize(self, text: str) -> list[RecognitionMatch[EmailNotation]]: + matches: list[RecognitionMatch[EmailNotation]] = [] + for match in _OBFUSCATED_PATTERN.finditer(text): + matches.append( + RecognitionMatch( + notation=EmailNotation( + local_part=match.group(1), + domain_part=f"{match.group(2)}.{match.group(3)}", + ), + start=match.start(), + end=match.end(), + raw_text=match.group(0), + ) + ) + for match in _AT_ONLY_PATTERN.finditer(text): + matches.append( + RecognitionMatch( + notation=EmailNotation( + local_part=match.group(1), + domain_part=match.group(2), + ), + start=match.start(), + end=match.end(), + raw_text=match.group(0), + ) + ) + return matches + + +class LegacyLocalhostEmailGrammar(Grammar[EmailNotation]): + """Legacy localhost email recognition (verbatim).""" + + name = "localhost_recognition" + semantics = "localhost_recognition" + single_value = True + + def recognize(self, text: str) -> list[RecognitionMatch[EmailNotation]]: + matches: list[RecognitionMatch[EmailNotation]] = [] + for match in _LOCALHOST_PATTERN.finditer(text): + matches.append( + RecognitionMatch( + notation=EmailNotation( + local_part=match.group(1), domain_part="localhost" + ), + start=match.start(), + end=match.end(), + raw_text=match.group(0), + ) + ) + return matches + + +# --------------------------------------------------------------------------- +# IP +# --------------------------------------------------------------------------- + +_IPV4_PATTERN = re.compile(r"\b(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})\b") + +_IPV6_BOUNDARY = r"(?:^|(?<=[\s,;([ ]))" +_IPV6_END = r"(?:$|(?=[\s,;().\]]))" + +_IPV6_FULL = re.compile( + _IPV6_BOUNDARY + r"([0-9a-fA-F]{1,4}(?::[0-9a-fA-F]{1,4}){7})" + _IPV6_END +) +_IPV6_COMPRESSED = re.compile( + _IPV6_BOUNDARY + + r"((?:[0-9a-fA-F]{1,4}:){0,6}[0-9a-fA-F]{0,4}::" + + r"(?:[0-9a-fA-F]{0,4}:){0,6}[0-9a-fA-F]{1,4})" + + _IPV6_END + + "|" + + _IPV6_BOUNDARY + + r"(::(?:[0-9a-fA-F]{1,4}:){0,6}[0-9a-fA-F]{1,4})" + + _IPV6_END + + "|" + + _IPV6_BOUNDARY + + r"((?:[0-9a-fA-F]{1,4}:){1,6}[0-9a-fA-F]{0,4}::)" + + _IPV6_END + + "|" + + _IPV6_BOUNDARY + + r"(::)" + + _IPV6_END +) + + +class LegacyIPv4Grammar(Grammar[IPNotation]): + """Legacy IPv4 recognition (verbatim).""" + + name = "ipv4_recognition" + semantics = "ipv4_recognition" + single_value = True + + def recognize(self, text: str) -> list[RecognitionMatch[IPNotation]]: + matches: list[RecognitionMatch[IPNotation]] = [] + for match in _IPV4_PATTERN.finditer(text): + matches.append( + RecognitionMatch( + notation=IPNotation(address=match.group(0)), + start=match.start(), + end=match.end(), + raw_text=match.group(0), + ) + ) + return matches + + +class LegacyIPv6Grammar(Grammar[IPNotation]): + """Legacy IPv6 recognition (verbatim).""" + + name = "ipv6_recognition" + semantics = "ipv6_recognition" + single_value = True + + def recognize(self, text: str) -> list[RecognitionMatch[IPNotation]]: + matches: list[RecognitionMatch[IPNotation]] = [] + for match in _IPV6_FULL.finditer(text): + matches.append( + RecognitionMatch( + notation=IPNotation(address=match.group(1)), + start=match.start(), + end=match.end(), + raw_text=match.group(1), + ) + ) + for match in _IPV6_COMPRESSED.finditer(text): + matches.append( + RecognitionMatch( + notation=IPNotation(address=match.group(0)), + start=match.start(), + end=match.end(), + raw_text=match.group(0), + ) + ) + return matches + + +# --------------------------------------------------------------------------- +# ISBN +# --------------------------------------------------------------------------- + +_ISBN13_PATTERN = re.compile( + r"\b(?:ISBN(?:-13)?[\s:-]+)?(?=((?:\d[ -]?){12}\d)(?![\d]))\1(? list[RecognitionMatch[ISBNNotation]]: + matches: list[RecognitionMatch[ISBNNotation]] = [] + for m in _ISBN13_PATTERN.finditer(text): + digits = "".join(ch for ch in m.group(1) if ch in "0123456789") + if len(digits) != 13: + continue + matches.append( + RecognitionMatch( + notation=ISBNNotation( + shape="isbn13", + digits=digits, + ), + start=m.start(), + end=m.end(), + raw_text=m.group(0), + ) + ) + return matches + + +class LegacyISBN10RecognitionGrammar(Grammar[ISBNNotation]): + """Legacy ISBN-10 recognition (verbatim).""" + + name = "isbn10_recognition" + semantics = "isbn10_recognition" + single_value = True + + def recognize(self, text: str) -> list[RecognitionMatch[ISBNNotation]]: + matches: list[RecognitionMatch[ISBNNotation]] = [] + for m in _ISBN10_PATTERN.finditer(text): + digits = "".join(ch for ch in m.group(1) if ch in "0123456789Xx").upper() + if len(digits) != 10: + continue + matches.append( + RecognitionMatch( + notation=ISBNNotation( + shape="isbn10", + digits=digits, + ), + start=m.start(), + end=m.end(), + raw_text=m.group(0), + ) + ) + return matches + + +# --------------------------------------------------------------------------- +# Country +# --------------------------------------------------------------------------- + +_KNOWN_NAME_KEYS = ( + ENGLISH_NAME_KEYS | HISTORICAL_NAME_KEYS | CHINESE_NAME_KEYS | LOCALIZED_NAME_KEYS +) + +_ALPHA2_PATTERN = re.compile(r"\b[A-Za-z]{2}\b") +_ALPHA3_PATTERN = re.compile(r"\b[A-Za-z]{3}\b") +_NUMERIC_PATTERN = re.compile(r"\b\d{1,3}\b") + + +class LegacyAlpha2Grammar(Grammar[CountryNotation]): + """Legacy alpha-2 country code recognition (verbatim).""" + + name = "alpha2_recognition" + semantics = "alpha2_recognition" + single_value = True + + def recognize(self, text: str) -> list[RecognitionMatch[CountryNotation]]: + if not text.strip(): + return [] + matches: list[RecognitionMatch[CountryNotation]] = [] + for match in _ALPHA2_PATTERN.finditer(text): + matches.append( + RecognitionMatch( + notation=CountryNotation( + shape="alpha2", value=match.group(0).upper() + ), + start=match.start(), + end=match.end(), + raw_text=match.group(0), + ) + ) + return matches + + +class LegacyAlpha3Grammar(Grammar[CountryNotation]): + """Legacy alpha-3 country code recognition (verbatim).""" + + name = "alpha3_recognition" + semantics = "alpha3_recognition" + single_value = True + + def recognize(self, text: str) -> list[RecognitionMatch[CountryNotation]]: + if not text.strip(): + return [] + matches: list[RecognitionMatch[CountryNotation]] = [] + for match in _ALPHA3_PATTERN.finditer(text): + matches.append( + RecognitionMatch( + notation=CountryNotation( + shape="alpha3", value=match.group(0).upper() + ), + start=match.start(), + end=match.end(), + raw_text=match.group(0), + ) + ) + return matches + + +class LegacyNumericGrammar(Grammar[CountryNotation]): + """Legacy numeric country code recognition (verbatim).""" + + name = "numeric_recognition" + semantics = "numeric_recognition" + single_value = True + + def recognize(self, text: str) -> list[RecognitionMatch[CountryNotation]]: + if not text.strip(): + return [] + matches: list[RecognitionMatch[CountryNotation]] = [] + for match in _NUMERIC_PATTERN.finditer(text): + matches.append( + RecognitionMatch( + notation=CountryNotation(shape="numeric", value=match.group(0)), + start=match.start(), + end=match.end(), + raw_text=match.group(0), + ) + ) + return matches + + +class LegacyNameGrammar(Grammar[CountryNotation]): + """Legacy country name recognition (verbatim).""" + + name = "name_recognition" + semantics = "name_recognition" + single_value = True + + def recognize(self, text: str) -> list[RecognitionMatch[CountryNotation]]: + trimmed = text.strip() + if not trimmed: + return [] + + normalized = normalize_name(trimmed) + + if normalized in _KNOWN_NAME_KEYS: + start = len(text) - len(text.lstrip()) + return [ + RecognitionMatch( + notation=CountryNotation(shape="name", value=trimmed), + start=start, + end=start + len(trimmed), + raw_text=trimmed, + ) + ] + + return [] diff --git a/tests/property/_legacy_siunit_grammars.py b/tests/property/_legacy_siunit_grammars.py new file mode 100644 index 00000000..c935a399 --- /dev/null +++ b/tests/property/_legacy_siunit_grammars.py @@ -0,0 +1,145 @@ +"""Verbatim legacy SIUnit grammars (pre-PipelineGrammar migration). + +Snapshot of the bespoke ``recognize()`` logic from commit 1a7c4b2, used by +the Migration Proof Harness (ADR-0008 §4.1) to assert byte-identical +``RecognitionMatch`` output after the staged-pipeline migration. Classes are +renamed ``Legacy*`` to avoid colliding with the migrated grammar classes. + +Do NOT edit by hand — this is a frozen reference. The live grammar files are +the source of truth post-migration; this module exists only so the parity +test can compare old vs new behavior. +""" + +from __future__ import annotations + +import re + +from paxman.capabilities.SIUnit.grammar.data.compound_tokens import ( + COMPOUND_SEPARATORS, + EXPONENT_CHARACTERS, +) +from paxman.capabilities.SIUnit.grammar.data.prefix_tokens import ( + PREFIX_SYMBOL_TOKENS, + PREFIX_WORD_TOKENS, +) +from paxman.capabilities.SIUnit.grammar.data.unit_name_tokens import NAME_TOKENS +from paxman.capabilities.SIUnit.grammar.data.unit_symbol_tokens import SYMBOL_TOKENS +from paxman.capabilities.SIUnit.notation import SIUnitNotation +from paxman.core.domain import Grammar, RecognitionMatch + +# --- symbol_recognition (verbatim) ----------------------------------------- + +DUAL_ROLE_PREFIX_SYMBOLS = frozenset({"a", "d", "h", "m"}) +PREFIX_ONLY_SYMBOLS = frozenset(PREFIX_SYMBOL_TOKENS) - DUAL_ROLE_PREFIX_SYMBOLS + +_SYMBOL_LOOKBEHIND = r"(?" + _SYMBOL_BODY + r")" + _SYMBOL_LOOKAHEAD +) + + +class LegacySymbolRecognition(Grammar[SIUnitNotation]): + """Legacy case-exact unit symbol recognition (verbatim).""" + + name = "symbol_recognition" + semantics = "symbol_recognition" + + def recognize(self, text: str) -> list[RecognitionMatch[SIUnitNotation]]: + matches: list[RecognitionMatch[SIUnitNotation]] = [] + for m in _SYMBOL_RE.finditer(text): + token = m.group("tok") + parts = token.split() + if len(parts) >= 2 and parts[0] in PREFIX_ONLY_SYMBOLS: + shape = "split_symbol_prefix" + else: + shape = "symbol" + matches.append( + RecognitionMatch( + raw_text=text[m.start() : m.end()], + start=m.start(), + end=m.end(), + notation=SIUnitNotation(text=token, shape=shape), + ) + ) + return matches + + +# --- name_recognition (verbatim) ------------------------------------------- + +PREFIX_WORDS = frozenset(PREFIX_WORD_TOKENS) + +_NAME_LOOKBEHIND = r"(?" + _NAME_BODY + r")" + _NAME_LOOKAHEAD, re.IGNORECASE +) + + +class LegacyNameRecognition(Grammar[SIUnitNotation]): + """Legacy case-folded unit name recognition (verbatim).""" + + name = "name_recognition" + semantics = "name_recognition" + + def recognize(self, text: str) -> list[RecognitionMatch[SIUnitNotation]]: + matches: list[RecognitionMatch[SIUnitNotation]] = [] + for m in _NAME_RE.finditer(text): + token = m.group("tok").lower() + parts = token.split() + if len(parts) >= 2 and parts[0] in PREFIX_WORDS: + shape = "split_word_prefix" + else: + shape = "name" + matches.append( + RecognitionMatch( + raw_text=text[m.start() : m.end()], + start=m.start(), + end=m.end(), + notation=SIUnitNotation(text=token, shape=shape), + ) + ) + return matches + + +# --- compound_recognition (verbatim) --------------------------------------- + +_EXPONENT = rf"[{EXPONENT_CHARACTERS}]*" +_UNIT = rf"(?:°?[A-Za-zµΩÅ][A-Za-zµΩÅ0-9]*{_EXPONENT})" +_SEP = f"[{COMPOUND_SEPARATORS}]" +_FACTOR = rf"(?:{_UNIT}|\({_UNIT}(?:{_SEP}{_UNIT}){{0,3}}\))" +_COMPOUND_RE = re.compile( + rf"(?{_FACTOR}(?:{_SEP}{_FACTOR}){{1,3}})(?![\w\-+\u2212])" +) + + +class LegacyCompoundRecognition(Grammar[SIUnitNotation]): + """Legacy product/quotient compound shape recognition (verbatim).""" + + name = "compound_recognition" + semantics = "compound_recognition" + + def recognize(self, text: str) -> list[RecognitionMatch[SIUnitNotation]]: + return [ + RecognitionMatch( + raw_text=text[m.start() : m.end()], + start=m.start(), + end=m.end(), + notation=SIUnitNotation(text=m.group("body"), shape="compound"), + ) + for m in _COMPOUND_RE.finditer(text) + ] diff --git a/tests/property/grammar_stage_parity.py b/tests/property/grammar_stage_parity.py new file mode 100644 index 00000000..9edb88d8 --- /dev/null +++ b/tests/property/grammar_stage_parity.py @@ -0,0 +1,31 @@ +"""Helper for Migration Proof Harness.""" + +from __future__ import annotations + +from typing import Any + +from paxman.core.domain import Grammar + + +def assert_grammar_parity(old: Grammar[Any], new: Grammar[Any], text: str) -> None: + """Assert byte-identical RecognitionMatch lists. + + The migration gate (ADR-0008 §4.1) requires that a new + ``PipelineGrammar`` declaration produces the same ``list[RecognitionMatch]`` + as the old bespoke ``recognize()`` for a given input: identical length and, + element-wise, identical ``start``, ``end``, ``raw_text``, and ``notation``. + """ + old_matches = old.recognize(text) + new_matches = new.recognize(text) + assert len(old_matches) == len(new_matches), ( + f"len mismatch for {text!r}: {old_matches} vs {new_matches}" + ) + for o, n in zip(old_matches, new_matches, strict=True): + assert o.start == n.start, f"start mismatch for {text!r}: {o} vs {n}" + assert o.end == n.end, f"end mismatch for {text!r}: {o} vs {n}" + assert o.raw_text == n.raw_text, ( + f"raw_text mismatch for {text!r}: {o.raw_text!r} vs {n.raw_text!r}" + ) + assert o.notation == n.notation, ( + f"notation mismatch for {text!r}: {o.notation!r} vs {n.notation!r}" + ) diff --git a/tests/property/test_grammar_stage_parity.py b/tests/property/test_grammar_stage_parity.py new file mode 100644 index 00000000..97757725 --- /dev/null +++ b/tests/property/test_grammar_stage_parity.py @@ -0,0 +1,952 @@ +"""Migration Proof Harness — byte-identical RecognitionMatch parity. + +Every migration PR must prove the new PipelineGrammar declaration produces +the same list[RecognitionMatch] as the old bespoke recognize() for a +curated corpus plus property-generated inputs. + +Corpus must cover: +- Country/name_recognition: normalized-key vs original-case value +- Phone/e164: E164Trim (end = start + len(trimmed)) vs match.end() +- URL: paren-balance (end = start + len(trimmed)) +- Money: either-order SYMBOL ? AMOUNT span-merge +- SIUnit: split-prefix classifier +- ISBN: hyphen/space tolerance +""" + +from __future__ import annotations + +from typing import Any + +import pytest + +from paxman.capabilities.Country.grammar.alpha2_recognition import Alpha2Grammar +from paxman.capabilities.Country.grammar.alpha3_recognition import Alpha3Grammar +from paxman.capabilities.Country.grammar.name_recognition import NameGrammar +from paxman.capabilities.Country.grammar.numeric_recognition import NumericGrammar +from paxman.capabilities.Currency.grammar.code_recognition import CodeRecognition +from paxman.capabilities.Currency.grammar.symbol_recognition import SymbolRecognition +from paxman.capabilities.Currency.grammar.word_recognition import WordRecognition +from paxman.capabilities.Date.grammar.european_recognition import EuropeanDateGrammar +from paxman.capabilities.Date.grammar.iso8601_recognition import ISO8601DateGrammar +from paxman.capabilities.Date.grammar.slash_iso_recognition import ( + SlashISODateGrammar, +) +from paxman.capabilities.Date.grammar.us_recognition import USDateGrammar +from paxman.capabilities.Email.grammar.localhost_recognition import ( + LocalhostEmailGrammar, +) +from paxman.capabilities.Email.grammar.obfuscated_recognition import ( + ObfuscatedEmailGrammar, +) +from paxman.capabilities.Email.grammar.standard_recognition import ( + StandardEmailGrammar, +) +from paxman.capabilities.IP.grammar.ipv4_recognition import IPv4Grammar +from paxman.capabilities.IP.grammar.ipv6_recognition import IPv6Grammar +from paxman.capabilities.ISBN.grammar.isbn10_recognition import ( + ISBN10RecognitionGrammar, +) +from paxman.capabilities.ISBN.grammar.isbn13_recognition import ( + ISBN13RecognitionGrammar, +) +from paxman.capabilities.Money.grammar.code_recognition import ( + CodeRecognition as MoneyCodeRecognition, +) +from paxman.capabilities.Money.grammar.symbol_recognition import ( + SymbolRecognition as MoneySymbolRecognition, +) +from paxman.capabilities.Money.grammar.word_recognition import ( + WordRecognition as MoneyWordRecognition, +) +from paxman.capabilities.Phone.grammar.e164_recognition import E164Grammar +from paxman.capabilities.Phone.grammar.international_00_recognition import ( + International00Grammar, +) +from paxman.capabilities.Phone.grammar.national_recognition import NationalGrammar +from paxman.capabilities.Phone.grammar.tel_uri_recognition import TelUriGrammar +from paxman.capabilities.SIUnit.grammar.compound_recognition import ( + CompoundRecognition as SiCompoundRecognition, +) +from paxman.capabilities.SIUnit.grammar.name_recognition import ( + NameRecognition as SiNameRecognition, +) +from paxman.capabilities.SIUnit.grammar.symbol_recognition import ( + SymbolRecognition as SiSymbolRecognition, +) +from paxman.capabilities.URL.grammar.absolute_uri_recognition import ( + AbsoluteUriRecognition, +) +from paxman.core.domain import Grammar +from tests.property._legacy_currency_grammars import ( + LegacyCodeRecognition, + LegacySymbolRecognition, + LegacyWordRecognition, +) +from tests.property._legacy_money_grammars import ( + LegacyCodeRecognition as LegacyMoneyCodeRecognition, +) +from tests.property._legacy_money_grammars import ( + LegacySymbolRecognition as LegacyMoneySymbolRecognition, +) +from tests.property._legacy_money_grammars import ( + LegacyWordRecognition as LegacyMoneyWordRecognition, +) +from tests.property._legacy_phone_url_grammars import ( + LegacyAbsoluteUriRecognition, + LegacyE164Grammar, + LegacyInternational00Grammar, + LegacyNationalGrammar, + LegacyTelUriGrammar, +) +from tests.property._legacy_remaining_grammars import ( + LegacyAlpha2Grammar, + LegacyAlpha3Grammar, + LegacyEuropeanDateGrammar, + LegacyIPv4Grammar, + LegacyIPv6Grammar, + LegacyISBN10RecognitionGrammar, + LegacyISBN13RecognitionGrammar, + LegacyISO8601DateGrammar, + LegacyLocalhostEmailGrammar, + LegacyNameGrammar, + LegacyNumericGrammar, + LegacyObfuscatedEmailGrammar, + LegacySlashISODateGrammar, + LegacyStandardEmailGrammar, + LegacyUSDateGrammar, +) +from tests.property._legacy_siunit_grammars import ( + LegacyCompoundRecognition as LegacySiCompound, +) +from tests.property._legacy_siunit_grammars import ( + LegacyNameRecognition as LegacySiName, +) +from tests.property._legacy_siunit_grammars import ( + LegacySymbolRecognition as LegacySiSymbol, +) + +# Import harness helper — implemented in this task. +from tests.property.grammar_stage_parity import assert_grammar_parity + +# Currency migration (Task 5): old bespoke recognize() vs new PipelineGrammar +# declaration. Each tuple is (legacy_grammar, new_grammar, text). The corpus +# covers qualified/bare symbols, case-folded codes, case-insensitive words, +# amount/sign-glued rejections, and inside-token rejections — every branch the +# migration must preserve byte-identically. +CURRENCY_PARITY_CASES: list[tuple[Grammar[Any], Grammar[Any], str]] = [ + # Symbol — qualified vs bare, longest/qualified-first precedence. + (LegacySymbolRecognition(), SymbolRecognition(), "US$"), + (LegacySymbolRecognition(), SymbolRecognition(), "€"), + (LegacySymbolRecognition(), SymbolRecognition(), "A$"), + (LegacySymbolRecognition(), SymbolRecognition(), "CA$"), + (LegacySymbolRecognition(), SymbolRecognition(), "US$ is the dollar"), + (LegacySymbolRecognition(), SymbolRecognition(), "A$ is the Australian dollar"), + (LegacySymbolRecognition(), SymbolRecognition(), "US$5"), # amount-glued reject + (LegacySymbolRecognition(), SymbolRecognition(), "$500"), # amount-glued reject + (LegacySymbolRecognition(), SymbolRecognition(), "x€"), # inside-token reject + (LegacySymbolRecognition(), SymbolRecognition(), ""), # empty + # Code — case folding, whitespace, multi-match, glued rejections. + (LegacyCodeRecognition(), CodeRecognition(), "USD"), + (LegacyCodeRecognition(), CodeRecognition(), " usd "), + (LegacyCodeRecognition(), CodeRecognition(), "GBP, EUR"), + (LegacyCodeRecognition(), CodeRecognition(), "US$"), # not 3 letters + (LegacyCodeRecognition(), CodeRecognition(), "xUSD"), # inside-token reject + (LegacyCodeRecognition(), CodeRecognition(), "USD-500"), # sign-glued reject + (LegacyCodeRecognition(), CodeRecognition(), "USD500"), # amount-glued reject + (LegacyCodeRecognition(), CodeRecognition(), "123"), # digits + (LegacyCodeRecognition(), CodeRecognition(), ""), # empty + # Word — case-insensitive fold, multi-word, plural/amount rejections. + (LegacyWordRecognition(), WordRecognition(), "euro"), + (LegacyWordRecognition(), WordRecognition(), "Euro"), + (LegacyWordRecognition(), WordRecognition(), "EURO"), + (LegacyWordRecognition(), WordRecognition(), "US Dollar"), + (LegacyWordRecognition(), WordRecognition(), "Euro and Dollar"), + (LegacyWordRecognition(), WordRecognition(), "Dollars"), # plural reject + (LegacyWordRecognition(), WordRecognition(), "euro500"), # amount-glued reject + (LegacyWordRecognition(), WordRecognition(), "the"), # non-token reject + (LegacyWordRecognition(), WordRecognition(), ""), # empty +] + + +@pytest.mark.property +@pytest.mark.parametrize(("legacy", "new", "text"), CURRENCY_PARITY_CASES) +def test_currency_grammar_parity( + legacy: Grammar[Any], new: Grammar[Any], text: str +) -> None: + """Byte-identical RecognitionMatch parity for migrated Currency grammars. + + Proves the PipelineGrammar declarations emit the same spans and notations + as the old bespoke recognize() bodies (ADR-0008 §4.1 migration gate). + """ + assert_grammar_parity(legacy, new, text) + + +# Money migration (Task 6): old bespoke recognize() vs new PipelineGrammar +# declaration using AmountComposer (S4 fused either-order span-merge). Each +# tuple is (legacy_grammar, new_grammar, text). The corpus covers both +# either-order arrangements, qualified/bare symbols, case-insensitive words, +# amount/sign-glued rejections, and inside-token rejections — every branch +# the migration must preserve byte-identically. +MONEY_PARITY_CASES: list[tuple[Grammar[Any], Grammar[Any], str]] = [ + # Code — either order, glued/inside-token rejections. + (LegacyMoneyCodeRecognition(), MoneyCodeRecognition(), "USD500"), + (LegacyMoneyCodeRecognition(), MoneyCodeRecognition(), "USD 500"), + (LegacyMoneyCodeRecognition(), MoneyCodeRecognition(), "500 USD"), + (LegacyMoneyCodeRecognition(), MoneyCodeRecognition(), "100MYR"), + (LegacyMoneyCodeRecognition(), MoneyCodeRecognition(), "USD"), # no amount + (LegacyMoneyCodeRecognition(), MoneyCodeRecognition(), "usd 500"), # lowercase + (LegacyMoneyCodeRecognition(), MoneyCodeRecognition(), "xUSD500"), # inside + (LegacyMoneyCodeRecognition(), MoneyCodeRecognition(), ""), # empty + # Symbol — qualified vs bare, either order, glued/inside rejections. + (LegacyMoneySymbolRecognition(), MoneySymbolRecognition(), "US$50.79"), + (LegacyMoneySymbolRecognition(), MoneySymbolRecognition(), "$500"), + (LegacyMoneySymbolRecognition(), MoneySymbolRecognition(), "RM100"), + (LegacyMoneySymbolRecognition(), MoneySymbolRecognition(), "€5"), + (LegacyMoneySymbolRecognition(), MoneySymbolRecognition(), "500 €"), + (LegacyMoneySymbolRecognition(), MoneySymbolRecognition(), "1.000,00 €"), + (LegacyMoneySymbolRecognition(), MoneySymbolRecognition(), "US$"), # no amount + (LegacyMoneySymbolRecognition(), MoneySymbolRecognition(), "$"), # no amount + (LegacyMoneySymbolRecognition(), MoneySymbolRecognition(), "x€"), # inside + (LegacyMoneySymbolRecognition(), MoneySymbolRecognition(), ""), # empty + # Word — case-insensitive, either order, plural/glued rejections. + (LegacyMoneyWordRecognition(), MoneyWordRecognition(), "18 Dollar"), + (LegacyMoneyWordRecognition(), MoneyWordRecognition(), "500 euro"), + (LegacyMoneyWordRecognition(), MoneyWordRecognition(), "500 Ringgit"), + (LegacyMoneyWordRecognition(), MoneyWordRecognition(), "500 Euro"), + (LegacyMoneyWordRecognition(), MoneyWordRecognition(), "Dollars"), # plural + (LegacyMoneyWordRecognition(), MoneyWordRecognition(), "euro500"), # glued + (LegacyMoneyWordRecognition(), MoneyWordRecognition(), ""), # empty +] + + +@pytest.mark.property +@pytest.mark.parametrize(("legacy", "new", "text"), MONEY_PARITY_CASES) +def test_money_grammar_parity( + legacy: Grammar[Any], new: Grammar[Any], text: str +) -> None: + """Byte-identical RecognitionMatch parity for migrated Money grammars. + + Proves the AmountComposer (S4) PipelineGrammar declarations emit the same + spans and notations as the old bespoke recognize() bodies (ADR-0008 §4.1 + migration gate). + """ + assert_grammar_parity(legacy, new, text) + + +# SIUnit migration (Task 7): old bespoke recognize() vs new PipelineGrammar +# declarations (S3 lexicon + S5 split-prefix classifier + S4 compound). Each +# tuple is (legacy_grammar, new_grammar, text). The corpus covers attached +# symbols, prefix-only split symbols (rejectable spans), dual-role spaced +# units (two units, not a split), the ° degree guard, case-folded names, +# word-prefix splits, multi-word names, and compound shapes — every branch +# the migration must preserve byte-identically. +SIUNIT_SYMBOL_PARITY_CASES: list[tuple[Grammar[Any], Grammar[Any], str]] = [ + (LegacySiSymbol(), SiSymbolRecognition(), "kg"), + (LegacySiSymbol(), SiSymbolRecognition(), "MHz"), + (LegacySiSymbol(), SiSymbolRecognition(), "k g"), # split_symbol_prefix + (LegacySiSymbol(), SiSymbolRecognition(), "da m"), # split_symbol_prefix + (LegacySiSymbol(), SiSymbolRecognition(), "µ g"), # split_symbol_prefix + (LegacySiSymbol(), SiSymbolRecognition(), "m s"), # two units, not split + (LegacySiSymbol(), SiSymbolRecognition(), "N m"), # two units, not split + (LegacySiSymbol(), SiSymbolRecognition(), "m/s"), # symbol "m" only + (LegacySiSymbol(), SiSymbolRecognition(), "m/s²"), # symbol "m" only + (LegacySiSymbol(), SiSymbolRecognition(), "kg/m/s"), # symbol "kg" only + (LegacySiSymbol(), SiSymbolRecognition(), "m·kg"), # symbol "m" only + (LegacySiSymbol(), SiSymbolRecognition(), "kPa"), # prefixed symbol + (LegacySiSymbol(), SiSymbolRecognition(), "°C"), # degree special name + (LegacySiSymbol(), SiSymbolRecognition(), "25°C"), # degree guard reject + (LegacySiSymbol(), SiSymbolRecognition(), "xkg"), # inside-token reject + (LegacySiSymbol(), SiSymbolRecognition(), "kg5"), # digit-glued reject + (LegacySiSymbol(), SiSymbolRecognition(), "2m"), # digit-glued reject + (LegacySiSymbol(), SiSymbolRecognition(), " kg "), # whitespace span + (LegacySiSymbol(), SiSymbolRecognition(), "kilo gram"), # not a symbol + (LegacySiSymbol(), SiSymbolRecognition(), "degree celsius"), # not a symbol + (LegacySiSymbol(), SiSymbolRecognition(), "Pa"), + (LegacySiSymbol(), SiSymbolRecognition(), "min"), + (LegacySiSymbol(), SiSymbolRecognition(), "da"), # bare prefix + (LegacySiSymbol(), SiSymbolRecognition(), "k"), # bare prefix + (LegacySiSymbol(), SiSymbolRecognition(), ""), # empty +] + +SIUNIT_NAME_PARITY_CASES: list[tuple[Grammar[Any], Grammar[Any], str]] = [ + (LegacySiName(), SiNameRecognition(), "kilogram"), + (LegacySiName(), SiNameRecognition(), "Kilogram"), # case fold + (LegacySiName(), SiNameRecognition(), "KILOGRAM"), # case fold + (LegacySiName(), SiNameRecognition(), "kelvin"), + (LegacySiName(), SiNameRecognition(), "degree celsius"), # name, not split + (LegacySiName(), SiNameRecognition(), "Degree Celsius"), # case fold + (LegacySiName(), SiNameRecognition(), "megahertz"), + (LegacySiName(), SiNameRecognition(), "kilometre"), + (LegacySiName(), SiNameRecognition(), "kilo gram"), # split_word_prefix + (LegacySiName(), SiNameRecognition(), "kelvin pascal"), # two names + (LegacySiName(), SiNameRecognition(), "5kilogram"), # digit-glued reject + (LegacySiName(), SiNameRecognition(), "kilogram5"), # digit-glued reject + (LegacySiName(), SiNameRecognition(), "xkelvin"), # inside-token reject + (LegacySiName(), SiNameRecognition(), "kg"), # not a name + (LegacySiName(), SiNameRecognition(), "kilograms"), # plural reject + (LegacySiName(), SiNameRecognition(), ""), # empty +] + +SIUNIT_COMPOUND_PARITY_CASES: list[tuple[Grammar[Any], Grammar[Any], str]] = [ + (LegacySiCompound(), SiCompoundRecognition(), "m/s²"), + (LegacySiCompound(), SiCompoundRecognition(), "m/s2"), + (LegacySiCompound(), SiCompoundRecognition(), "km/h"), + (LegacySiCompound(), SiCompoundRecognition(), "N·m"), + (LegacySiCompound(), SiCompoundRecognition(), "N⋅m"), + (LegacySiCompound(), SiCompoundRecognition(), "kg·m/s²"), + (LegacySiCompound(), SiCompoundRecognition(), "g/cm³"), + (LegacySiCompound(), SiCompoundRecognition(), "m·s⁻²"), + (LegacySiCompound(), SiCompoundRecognition(), "m/°C"), + ( + LegacySiCompound(), + SiCompoundRecognition(), + "5°C/W", + ), # degree-excluded: matches "C/W" at 2 + (LegacySiCompound(), SiCompoundRecognition(), "µg/mL"), + (LegacySiCompound(), SiCompoundRecognition(), "QQQ/zzz"), # shape-only + (LegacySiCompound(), SiCompoundRecognition(), "m/sx"), # shape-only + (LegacySiCompound(), SiCompoundRecognition(), "xN·m"), # shape-only + (LegacySiCompound(), SiCompoundRecognition(), "m"), # single unit reject + (LegacySiCompound(), SiCompoundRecognition(), "m s"), # space reject + (LegacySiCompound(), SiCompoundRecognition(), "5m/s"), # digit-glued reject + (LegacySiCompound(), SiCompoundRecognition(), ""), # empty +] + + +@pytest.mark.property +@pytest.mark.parametrize(("legacy", "new", "text"), SIUNIT_SYMBOL_PARITY_CASES) +def test_siunit_symbol_parity( + legacy: Grammar[Any], new: Grammar[Any], text: str +) -> None: + """Byte-identical RecognitionMatch parity for migrated SIUnit symbol grammar.""" + assert_grammar_parity(legacy, new, text) + + +@pytest.mark.property +@pytest.mark.parametrize(("legacy", "new", "text"), SIUNIT_NAME_PARITY_CASES) +def test_siunit_name_parity(legacy: Grammar[Any], new: Grammar[Any], text: str) -> None: + """Byte-identical RecognitionMatch parity for migrated SIUnit name grammar.""" + assert_grammar_parity(legacy, new, text) + + +@pytest.mark.property +@pytest.mark.parametrize(("legacy", "new", "text"), SIUNIT_COMPOUND_PARITY_CASES) +def test_siunit_compound_parity( + legacy: Grammar[Any], new: Grammar[Any], text: str +) -> None: + """Byte-identical RecognitionMatch parity for migrated SIUnit compound grammar.""" + assert_grammar_parity(legacy, new, text) + + +# Phone migration (Task 8): old bespoke recognize() vs new PipelineGrammar +# declarations (S5 RegexStage + PostStage trim). Each tuple is +# (legacy_grammar, new_grammar, text). The corpus covers the E.164 15-digit +# window trim, the 00-prefix lookbehind (incl. "+00" / digit / dot rejections), +# the tel-URI scheme + extension, the NANP national 4-chain lookbehind, and +# every branch the migration must preserve byte-identically. +PHONE_PARITY_CASES: list[tuple[Grammar[Any], Grammar[Any], str]] = [ + # E.164 — normal forms + separators. + (LegacyE164Grammar(), E164Grammar(), "+15551234567"), + (LegacyE164Grammar(), E164Grammar(), "+1 555 123 4567"), + (LegacyE164Grammar(), E164Grammar(), "+44-20-7946-0958"), + (LegacyE164Grammar(), E164Grammar(), "+1.555.123.4567"), + (LegacyE164Grammar(), E164Grammar(), "+1 (555) 123-4567"), + # E.164 — runaway trim at the 15-digit window (end = start + len(trimmed)). + (LegacyE164Grammar(), E164Grammar(), "+15551234567 5551234567"), + # E.164 — oversized first run is NOT truncated (validation rejects later). + (LegacyE164Grammar(), E164Grammar(), "+12345678901234567890"), + # E.164 — rejections (no plus, national, word-char-before-+). + (LegacyE164Grammar(), E164Grammar(), "15551234567"), + (LegacyE164Grammar(), E164Grammar(), "(555) 123-4567"), + (LegacyE164Grammar(), E164Grammar(), "user+123@example.com"), + (LegacyE164Grammar(), E164Grammar(), "a+123"), + (LegacyE164Grammar(), E164Grammar(), "x+11=y"), + (LegacyE164Grammar(), E164Grammar(), "1+11=12"), + # E.164 — in-text span + multiple + trailing period. + (LegacyE164Grammar(), E164Grammar(), "Call +1 555 123 4567 now"), + (LegacyE164Grammar(), E164Grammar(), "+15551234567 or +442079460958"), + (LegacyE164Grammar(), E164Grammar(), "End of +15551234567."), + (LegacyE164Grammar(), E164Grammar(), ""), # empty + # tel-URI — normal, dashes, extension, in-text, uppercase scheme. + (LegacyTelUriGrammar(), TelUriGrammar(), "tel:+15551234567"), + (LegacyTelUriGrammar(), TelUriGrammar(), "tel:+1-201-555-0123"), + (LegacyTelUriGrammar(), TelUriGrammar(), "tel:+15551234567;ext=890"), + (LegacyTelUriGrammar(), TelUriGrammar(), "Reach me at tel:+15551234567 now"), + (LegacyTelUriGrammar(), TelUriGrammar(), "TEL:+15551234567"), + # tel-URI — rejections (no scheme, no-plus local, scheme inside word). + (LegacyTelUriGrammar(), TelUriGrammar(), "+15551234567"), + (LegacyTelUriGrammar(), TelUriGrammar(), "tel:2125550123"), + (LegacyTelUriGrammar(), TelUriGrammar(), "tel:15551234567"), + (LegacyTelUriGrammar(), TelUriGrammar(), "tel:44 20 7946 0958"), + (LegacyTelUriGrammar(), TelUriGrammar(), "hotel:+15551234567"), + (LegacyTelUriGrammar(), TelUriGrammar(), "xtel:+15551234567"), + (LegacyTelUriGrammar(), TelUriGrammar(), ""), # empty + # International 00 — normal, compact, in-text, trailing period. + (LegacyInternational00Grammar(), International00Grammar(), "00 44 20 7946 0958"), + (LegacyInternational00Grammar(), International00Grammar(), "00442079460958"), + ( + LegacyInternational00Grammar(), + International00Grammar(), + "Dial 00 44 20 7946 0958 from abroad", + ), + (LegacyInternational00Grammar(), International00Grammar(), "00 44 20 7946 0958."), + # International 00 — rejections (plus, single zero, digit/dot/word before). + (LegacyInternational00Grammar(), International00Grammar(), "+442079460958"), + (LegacyInternational00Grammar(), International00Grammar(), "0 44 20 7946 0958"), + (LegacyInternational00Grammar(), International00Grammar(), "100442079460958"), + (LegacyInternational00Grammar(), International00Grammar(), "+00442079460958"), + (LegacyInternational00Grammar(), International00Grammar(), "0.00442079460958"), + (LegacyInternational00Grammar(), International00Grammar(), "user00123@example.com"), + (LegacyInternational00Grammar(), International00Grammar(), "x0044 20 7946 0958"), + (LegacyInternational00Grammar(), International00Grammar(), ""), # empty + # National — parens, dashes, dots, spaces, trunk, in-text. + (LegacyNationalGrammar(), NationalGrammar(), "(555) 123-4567"), + (LegacyNationalGrammar(), NationalGrammar(), "555-123-4567"), + (LegacyNationalGrammar(), NationalGrammar(), "555.123.4567"), + (LegacyNationalGrammar(), NationalGrammar(), "555 123 4567"), + (LegacyNationalGrammar(), NationalGrammar(), "1-555-123-4567"), + (LegacyNationalGrammar(), NationalGrammar(), "1 (555) 123-4567"), + (LegacyNationalGrammar(), NationalGrammar(), "Call (555) 123-4567 today"), + # National — rejections (international in all separator shapes, tel-URI, short). + (LegacyNationalGrammar(), NationalGrammar(), "+15551234567"), + (LegacyNationalGrammar(), NationalGrammar(), "+1-555-123-4567"), + (LegacyNationalGrammar(), NationalGrammar(), "+1 555 123 4567"), + (LegacyNationalGrammar(), NationalGrammar(), "+1.555.123.4567"), + (LegacyNationalGrammar(), NationalGrammar(), "+1 (555) 123-4567"), + (LegacyNationalGrammar(), NationalGrammar(), "tel:+1-201-555-0123"), + (LegacyNationalGrammar(), NationalGrammar(), "tel:+15551234567"), + (LegacyNationalGrammar(), NationalGrammar(), "tel:+1 (555) 123-4567"), + (LegacyNationalGrammar(), NationalGrammar(), "555-1234"), + (LegacyNationalGrammar(), NationalGrammar(), ""), # empty +] + + +# URL migration (Task 8): old bespoke recognize() vs new PipelineGrammar +# declaration (S5 RegexStage + PostStage paren-balance/D16 drop). Each tuple +# is (legacy_grammar, new_grammar, text). The corpus covers the Appendix C +# paren-balance trim, the D16 bare-scheme drop, the scheme-char left +# boundary, multi-line spans, and shape-only recognition — every branch the +# migration must preserve byte-identically. +URL_PARITY_CASES: list[tuple[Grammar[Any], Grammar[Any], str]] = [ + # Paren-balance trim + bare-scheme drop. + ( + LegacyAbsoluteUriRecognition(), + AbsoluteUriRecognition(), + "https://example.com/path_(with_parens)", + ), + (LegacyAbsoluteUriRecognition(), AbsoluteUriRecognition(), "Note:"), # bare scheme + (LegacyAbsoluteUriRecognition(), AbsoluteUriRecognition(), "(https://example.com)"), + (LegacyAbsoluteUriRecognition(), AbsoluteUriRecognition(), "http://exa\nmple.com/"), + (LegacyAbsoluteUriRecognition(), AbsoluteUriRecognition(), "http://example.com."), + # Left boundary: word rejection keeps "ahttps" span, digit start rejected. + (LegacyAbsoluteUriRecognition(), AbsoluteUriRecognition(), "ahttps://example.com"), + (LegacyAbsoluteUriRecognition(), AbsoluteUriRecognition(), "1https://example.com"), + (LegacyAbsoluteUriRecognition(), AbsoluteUriRecognition(), "(https://example.com"), + # Non-ASCII body + shape-only recognition. + ( + LegacyAbsoluteUriRecognition(), + AbsoluteUriRecognition(), + "mailto:user@münchen.de", + ), + (LegacyAbsoluteUriRecognition(), AbsoluteUriRecognition(), "https://"), + (LegacyAbsoluteUriRecognition(), AbsoluteUriRecognition(), "http://99999/"), + # All-paren body collapses to bare scheme -> dropped. + (LegacyAbsoluteUriRecognition(), AbsoluteUriRecognition(), "https:))))"), + (LegacyAbsoluteUriRecognition(), AbsoluteUriRecognition(), "https://example.com))"), + # Double-quote right boundary. + ( + LegacyAbsoluteUriRecognition(), + AbsoluteUriRecognition(), + '"https://example.com/"', + ), + (LegacyAbsoluteUriRecognition(), AbsoluteUriRecognition(), ""), # empty +] + + +@pytest.mark.property +@pytest.mark.parametrize(("legacy", "new", "text"), PHONE_PARITY_CASES) +def test_phone_grammar_parity( + legacy: Grammar[Any], new: Grammar[Any], text: str +) -> None: + """Byte-identical RecognitionMatch parity for migrated Phone grammars.""" + assert_grammar_parity(legacy, new, text) + + +@pytest.mark.property +@pytest.mark.parametrize(("legacy", "new", "text"), URL_PARITY_CASES) +def test_url_grammar_parity(legacy: Grammar[Any], new: Grammar[Any], text: str) -> None: + """Byte-identical RecognitionMatch parity for migrated URL grammar.""" + assert_grammar_parity(legacy, new, text) + + +# --------------------------------------------------------------------------- +# Remaining S1+S2 grammars (Task 9): Date, Email, IP, ISBN, Country. +# Old bespoke recognize() (legacy snapshot) vs new PipelineGrammar +# declaration. Each tuple is (legacy_grammar, new_grammar, text). The corpus +# covers valid matches, rejections (inside-token, digit-glued, word-glued), +# empty, whitespace, and representative capability-test inputs — every branch +# the migration must preserve byte-identically. +# --------------------------------------------------------------------------- + +DATE_ISO_PARITY_CASES: list[tuple[Grammar[Any], Grammar[Any], str]] = [ + (LegacyISO8601DateGrammar(), ISO8601DateGrammar(), "2026-01-15"), + (LegacyISO8601DateGrammar(), ISO8601DateGrammar(), "2026/01/15"), # no match + ( + LegacyISO8601DateGrammar(), + ISO8601DateGrammar(), + "12026-01-15", + ), # digit-glued reject + ( + LegacyISO8601DateGrammar(), + ISO8601DateGrammar(), + "2026-01-261", + ), # digit-glued reject + ( + LegacyISO8601DateGrammar(), + ISO8601DateGrammar(), + "Dates: 2026-07-26 and 2025-12-31", + ), + (LegacyISO8601DateGrammar(), ISO8601DateGrammar(), "x 2026-07-26 y"), + ( + LegacyISO8601DateGrammar(), + ISO8601DateGrammar(), + " 2026-01-15 ", + ), # whitespace span + (LegacyISO8601DateGrammar(), ISO8601DateGrammar(), ""), # empty + (LegacyISO8601DateGrammar(), ISO8601DateGrammar(), "No dates here"), # no match +] + +DATE_US_PARITY_CASES: list[tuple[Grammar[Any], Grammar[Any], str]] = [ + (LegacyUSDateGrammar(), USDateGrammar(), "07/26/2026"), # 4-digit year + (LegacyUSDateGrammar(), USDateGrammar(), "07/26/26"), # 2-digit year + (LegacyUSDateGrammar(), USDateGrammar(), "7/26/2026"), # single-digit month + (LegacyUSDateGrammar(), USDateGrammar(), "1207/26/2026"), # digit-glued reject + (LegacyUSDateGrammar(), USDateGrammar(), "07/26/20261"), # digit-glued reject + ( + LegacyUSDateGrammar(), + USDateGrammar(), + "Dates: 07/26/2026 and 12/31/2025", + ), # 4-digit x2 + ( + LegacyUSDateGrammar(), + USDateGrammar(), + "Dates: 07/26/26 and 12/31/25", + ), # 2-digit x2 + (LegacyUSDateGrammar(), USDateGrammar(), "x 07/26/2026 y"), + (LegacyUSDateGrammar(), USDateGrammar(), ""), # empty + (LegacyUSDateGrammar(), USDateGrammar(), "No dates here"), # no match +] + +DATE_EUROPEAN_PARITY_CASES: list[tuple[Grammar[Any], Grammar[Any], str]] = [ + (LegacyEuropeanDateGrammar(), EuropeanDateGrammar(), "26/07/2026"), # 4-digit year + (LegacyEuropeanDateGrammar(), EuropeanDateGrammar(), "26/07/26"), # 2-digit year + (LegacyEuropeanDateGrammar(), EuropeanDateGrammar(), "6/7/2026"), # single-digit + (LegacyEuropeanDateGrammar(), EuropeanDateGrammar(), "1226/07/2026"), # reject + (LegacyEuropeanDateGrammar(), EuropeanDateGrammar(), "26/07/20261"), # reject + ( + LegacyEuropeanDateGrammar(), + EuropeanDateGrammar(), + "Dates: 26/07/2026 and 31/12/2025", + ), + (LegacyEuropeanDateGrammar(), EuropeanDateGrammar(), "x 26/07/2026 y"), + (LegacyEuropeanDateGrammar(), EuropeanDateGrammar(), ""), # empty + (LegacyEuropeanDateGrammar(), EuropeanDateGrammar(), "No dates here"), # no match +] + +DATE_SLASH_PARITY_CASES: list[tuple[Grammar[Any], Grammar[Any], str]] = [ + (LegacySlashISODateGrammar(), SlashISODateGrammar(), "2026/07/26"), + (LegacySlashISODateGrammar(), SlashISODateGrammar(), "2026/7/6"), # single-digit + ( + LegacySlashISODateGrammar(), + SlashISODateGrammar(), + "07/26/2026", + ), # no match (US order) + (LegacySlashISODateGrammar(), SlashISODateGrammar(), "12026/07/26"), # reject + (LegacySlashISODateGrammar(), SlashISODateGrammar(), "2026/07/261"), # reject + ( + LegacySlashISODateGrammar(), + SlashISODateGrammar(), + "Dates: 2026/07/26 and 2025/12/31", + ), + (LegacySlashISODateGrammar(), SlashISODateGrammar(), "x 2026/07/26 y"), + (LegacySlashISODateGrammar(), SlashISODateGrammar(), ""), # empty +] + +EMAIL_STANDARD_PARITY_CASES: list[tuple[Grammar[Any], Grammar[Any], str]] = [ + ( + LegacyStandardEmailGrammar(), + StandardEmailGrammar(), + "Contact us at user@example.com", + ), + ( + LegacyStandardEmailGrammar(), + StandardEmailGrammar(), + "Send to first.last@domain.co.uk", + ), + (LegacyStandardEmailGrammar(), StandardEmailGrammar(), "user+tag@gmail.com"), + ( + LegacyStandardEmailGrammar(), + StandardEmailGrammar(), + "Email a@b.com or c@d.org", + ), # two + (LegacyStandardEmailGrammar(), StandardEmailGrammar(), "not an email"), # no match + ( + LegacyStandardEmailGrammar(), + StandardEmailGrammar(), + "user at example dot com", + ), # no match + (LegacyStandardEmailGrammar(), StandardEmailGrammar(), ""), # empty +] + +EMAIL_OBFUSCATED_PARITY_CASES: list[tuple[Grammar[Any], Grammar[Any], str]] = [ + ( + LegacyObfuscatedEmailGrammar(), + ObfuscatedEmailGrammar(), + "Contact user at example dot com", + ), + ( + LegacyObfuscatedEmailGrammar(), + ObfuscatedEmailGrammar(), + "Email user at gmail.com", + ), + ( + LegacyObfuscatedEmailGrammar(), + ObfuscatedEmailGrammar(), + "user@example.com", + ), # no match + ( + LegacyObfuscatedEmailGrammar(), + ObfuscatedEmailGrammar(), + "no email here", + ), # no match + (LegacyObfuscatedEmailGrammar(), ObfuscatedEmailGrammar(), ""), # empty +] + +EMAIL_LOCALHOST_PARITY_CASES: list[tuple[Grammar[Any], Grammar[Any], str]] = [ + (LegacyLocalhostEmailGrammar(), LocalhostEmailGrammar(), "Send to admin@localhost"), + (LegacyLocalhostEmailGrammar(), LocalhostEmailGrammar(), "user@localhost:8080"), + ( + LegacyLocalhostEmailGrammar(), + LocalhostEmailGrammar(), + "user@example.com", + ), # no match + ( + LegacyLocalhostEmailGrammar(), + LocalhostEmailGrammar(), + "no email here", + ), # no match + (LegacyLocalhostEmailGrammar(), LocalhostEmailGrammar(), ""), # empty +] + +IPV4_PARITY_CASES: list[tuple[Grammar[Any], Grammar[Any], str]] = [ + (LegacyIPv4Grammar(), IPv4Grammar(), "192.168.1.1"), + (LegacyIPv4Grammar(), IPv4Grammar(), "10.0.0.1"), + (LegacyIPv4Grammar(), IPv4Grammar(), "foo 192.168.1.1 bar"), + (LegacyIPv4Grammar(), IPv4Grammar(), "192.168.1.1 and 10.0.0.1"), # two + (LegacyIPv4Grammar(), IPv4Grammar(), "256.1.1.1"), # regex-valid (no range check) + (LegacyIPv4Grammar(), IPv4Grammar(), "not an ip"), # no match + (LegacyIPv4Grammar(), IPv4Grammar(), ""), # empty +] + +IPV6_PARITY_CASES: list[tuple[Grammar[Any], Grammar[Any], str]] = [ + ( + LegacyIPv6Grammar(), + IPv6Grammar(), + "2001:0db8:85a3:0000:0000:8a2e:0370:7334", + ), # full + (LegacyIPv6Grammar(), IPv6Grammar(), "2001:db8:85a3::8a2e:370:7334"), # compressed + (LegacyIPv6Grammar(), IPv6Grammar(), "::1"), # loopback + (LegacyIPv6Grammar(), IPv6Grammar(), "fe80::1"), # link-local + (LegacyIPv6Grammar(), IPv6Grammar(), "::"), # all-zeros + (LegacyIPv6Grammar(), IPv6Grammar(), "See 2001:db8::1 here"), # in-text + (LegacyIPv6Grammar(), IPv6Grammar(), "not ipv6"), # no match + (LegacyIPv6Grammar(), IPv6Grammar(), ""), # empty +] + +ISBN13_PARITY_CASES: list[tuple[Grammar[Any], Grammar[Any], str]] = [ + (LegacyISBN13RecognitionGrammar(), ISBN13RecognitionGrammar(), "9780306406157"), + (LegacyISBN13RecognitionGrammar(), ISBN13RecognitionGrammar(), "978-0-11-000222-4"), + ( + LegacyISBN13RecognitionGrammar(), + ISBN13RecognitionGrammar(), + "ISBN 9780306406157", + ), + ( + LegacyISBN13RecognitionGrammar(), + ISBN13RecognitionGrammar(), + "ISBN: 978-0-11-000222-4", + ), + ( + LegacyISBN13RecognitionGrammar(), + ISBN13RecognitionGrammar(), + "foo 9780306406157 bar", + ), + ( + LegacyISBN13RecognitionGrammar(), + ISBN13RecognitionGrammar(), + "978030640615", + ), # 12 digits + ( + LegacyISBN13RecognitionGrammar(), + ISBN13RecognitionGrammar(), + "x9780306406157", + ), # glued + (LegacyISBN13RecognitionGrammar(), ISBN13RecognitionGrammar(), ""), # empty +] + +ISBN10_PARITY_CASES: list[tuple[Grammar[Any], Grammar[Any], str]] = [ + (LegacyISBN10RecognitionGrammar(), ISBN10RecognitionGrammar(), "0306406152"), + (LegacyISBN10RecognitionGrammar(), ISBN10RecognitionGrammar(), "0-306-40615-2"), + (LegacyISBN10RecognitionGrammar(), ISBN10RecognitionGrammar(), "ISBN 0306406152"), + ( + LegacyISBN10RecognitionGrammar(), + ISBN10RecognitionGrammar(), + "ISBN: 0-306-40615-2", + ), + ( + LegacyISBN10RecognitionGrammar(), + ISBN10RecognitionGrammar(), + "foo 0306406152 bar", + ), + ( + LegacyISBN10RecognitionGrammar(), + ISBN10RecognitionGrammar(), + "030640615", + ), # 9 digits + (LegacyISBN10RecognitionGrammar(), ISBN10RecognitionGrammar(), ""), # empty +] + +COUNTRY_ALPHA2_PARITY_CASES: list[tuple[Grammar[Any], Grammar[Any], str]] = [ + (LegacyAlpha2Grammar(), Alpha2Grammar(), "US"), + (LegacyAlpha2Grammar(), Alpha2Grammar(), "us"), # case fold + (LegacyAlpha2Grammar(), Alpha2Grammar(), "GB"), + (LegacyAlpha2Grammar(), Alpha2Grammar(), "US and GB"), # two + (LegacyAlpha2Grammar(), Alpha2Grammar(), "USA"), # 3 letters, no match + (LegacyAlpha2Grammar(), Alpha2Grammar(), "xUS"), # inside-token reject + (LegacyAlpha2Grammar(), Alpha2Grammar(), " US "), # whitespace span + (LegacyAlpha2Grammar(), Alpha2Grammar(), ""), # empty +] + +COUNTRY_ALPHA3_PARITY_CASES: list[tuple[Grammar[Any], Grammar[Any], str]] = [ + (LegacyAlpha3Grammar(), Alpha3Grammar(), "USA"), + (LegacyAlpha3Grammar(), Alpha3Grammar(), "usa"), # case fold + (LegacyAlpha3Grammar(), Alpha3Grammar(), "GBR"), + (LegacyAlpha3Grammar(), Alpha3Grammar(), "US"), # 2 letters, no match + (LegacyAlpha3Grammar(), Alpha3Grammar(), "xUSA"), # inside-token reject + (LegacyAlpha3Grammar(), Alpha3Grammar(), ""), # empty +] + +COUNTRY_NUMERIC_PARITY_CASES: list[tuple[Grammar[Any], Grammar[Any], str]] = [ + (LegacyNumericGrammar(), NumericGrammar(), "840"), + (LegacyNumericGrammar(), NumericGrammar(), "4"), + (LegacyNumericGrammar(), NumericGrammar(), "004"), + (LegacyNumericGrammar(), NumericGrammar(), "840 and 124"), # two + (LegacyNumericGrammar(), NumericGrammar(), "1234"), # 4 digits, no match + (LegacyNumericGrammar(), NumericGrammar(), "US"), # letters, no match + (LegacyNumericGrammar(), NumericGrammar(), ""), # empty +] + +COUNTRY_NAME_PARITY_CASES: list[tuple[Grammar[Any], Grammar[Any], str]] = [ + (LegacyNameGrammar(), NameGrammar(), "United States"), # original case preserved + (LegacyNameGrammar(), NameGrammar(), " united states "), # whitespace + case fold + (LegacyNameGrammar(), NameGrammar(), "USA"), # not a name + (LegacyNameGrammar(), NameGrammar(), "Alemania"), # localized name + (LegacyNameGrammar(), NameGrammar(), "Burma"), # historical name + (LegacyNameGrammar(), NameGrammar(), "840"), # numeric, not name + (LegacyNameGrammar(), NameGrammar(), "XYZ"), # unknown name + (LegacyNameGrammar(), NameGrammar(), ""), # empty +] + + +@pytest.mark.property +@pytest.mark.parametrize(("legacy", "new", "text"), DATE_ISO_PARITY_CASES) +def test_date_iso8601_parity( + legacy: Grammar[Any], new: Grammar[Any], text: str +) -> None: + """Byte-identical RecognitionMatch parity for migrated Date ISO-8601 grammar.""" + assert_grammar_parity(legacy, new, text) + + +@pytest.mark.property +@pytest.mark.parametrize(("legacy", "new", "text"), DATE_US_PARITY_CASES) +def test_date_us_parity(legacy: Grammar[Any], new: Grammar[Any], text: str) -> None: + """Byte-identical RecognitionMatch parity for migrated Date US grammar.""" + assert_grammar_parity(legacy, new, text) + + +@pytest.mark.property +@pytest.mark.parametrize(("legacy", "new", "text"), DATE_EUROPEAN_PARITY_CASES) +def test_date_european_parity( + legacy: Grammar[Any], new: Grammar[Any], text: str +) -> None: + """Byte-identical RecognitionMatch parity for migrated Date European grammar.""" + assert_grammar_parity(legacy, new, text) + + +@pytest.mark.property +@pytest.mark.parametrize(("legacy", "new", "text"), DATE_SLASH_PARITY_CASES) +def test_date_slash_iso_parity( + legacy: Grammar[Any], new: Grammar[Any], text: str +) -> None: + """Byte-identical RecognitionMatch parity for migrated Date slash-ISO grammar.""" + assert_grammar_parity(legacy, new, text) + + +@pytest.mark.property +@pytest.mark.parametrize(("legacy", "new", "text"), EMAIL_STANDARD_PARITY_CASES) +def test_email_standard_parity( + legacy: Grammar[Any], new: Grammar[Any], text: str +) -> None: + """Byte-identical RecognitionMatch parity for migrated Email standard grammar.""" + assert_grammar_parity(legacy, new, text) + + +@pytest.mark.property +@pytest.mark.parametrize(("legacy", "new", "text"), EMAIL_OBFUSCATED_PARITY_CASES) +def test_email_obfuscated_parity( + legacy: Grammar[Any], new: Grammar[Any], text: str +) -> None: + """Byte-identical RecognitionMatch parity for migrated Email obfuscated grammar.""" + assert_grammar_parity(legacy, new, text) + + +@pytest.mark.property +@pytest.mark.parametrize(("legacy", "new", "text"), EMAIL_LOCALHOST_PARITY_CASES) +def test_email_localhost_parity( + legacy: Grammar[Any], new: Grammar[Any], text: str +) -> None: + """Byte-identical RecognitionMatch parity for migrated Email localhost grammar.""" + assert_grammar_parity(legacy, new, text) + + +@pytest.mark.property +@pytest.mark.parametrize(("legacy", "new", "text"), IPV4_PARITY_CASES) +def test_ipv4_parity(legacy: Grammar[Any], new: Grammar[Any], text: str) -> None: + """Byte-identical RecognitionMatch parity for migrated IPv4 grammar.""" + assert_grammar_parity(legacy, new, text) + + +@pytest.mark.property +@pytest.mark.parametrize(("legacy", "new", "text"), IPV6_PARITY_CASES) +def test_ipv6_parity(legacy: Grammar[Any], new: Grammar[Any], text: str) -> None: + """Byte-identical RecognitionMatch parity for migrated IPv6 grammar.""" + assert_grammar_parity(legacy, new, text) + + +@pytest.mark.property +@pytest.mark.parametrize(("legacy", "new", "text"), ISBN13_PARITY_CASES) +def test_isbn13_parity(legacy: Grammar[Any], new: Grammar[Any], text: str) -> None: + """Byte-identical RecognitionMatch parity for migrated ISBN-13 grammar.""" + assert_grammar_parity(legacy, new, text) + + +@pytest.mark.property +@pytest.mark.parametrize(("legacy", "new", "text"), ISBN10_PARITY_CASES) +def test_isbn10_parity(legacy: Grammar[Any], new: Grammar[Any], text: str) -> None: + """Byte-identical RecognitionMatch parity for migrated ISBN-10 grammar.""" + assert_grammar_parity(legacy, new, text) + + +@pytest.mark.property +@pytest.mark.parametrize(("legacy", "new", "text"), COUNTRY_ALPHA2_PARITY_CASES) +def test_country_alpha2_parity( + legacy: Grammar[Any], new: Grammar[Any], text: str +) -> None: + """Byte-identical RecognitionMatch parity for migrated Country alpha-2 grammar.""" + assert_grammar_parity(legacy, new, text) + + +@pytest.mark.property +@pytest.mark.parametrize(("legacy", "new", "text"), COUNTRY_ALPHA3_PARITY_CASES) +def test_country_alpha3_parity( + legacy: Grammar[Any], new: Grammar[Any], text: str +) -> None: + """Byte-identical RecognitionMatch parity for migrated Country alpha-3 grammar.""" + assert_grammar_parity(legacy, new, text) + + +@pytest.mark.property +@pytest.mark.parametrize(("legacy", "new", "text"), COUNTRY_NUMERIC_PARITY_CASES) +def test_country_numeric_parity( + legacy: Grammar[Any], new: Grammar[Any], text: str +) -> None: + """Byte-identical RecognitionMatch parity for migrated Country numeric grammar.""" + assert_grammar_parity(legacy, new, text) + + +@pytest.mark.property +@pytest.mark.parametrize(("legacy", "new", "text"), COUNTRY_NAME_PARITY_CASES) +def test_country_name_parity( + legacy: Grammar[Any], new: Grammar[Any], text: str +) -> None: + """Byte-identical RecognitionMatch parity for migrated Country name grammar.""" + assert_grammar_parity(legacy, new, text) + + +# NOTE: US/European date grammars merged two separate ``finditer`` loops +# (4-digit then 2-digit year) into a single ``(\\d{4}|\\d{2})`` alternation. +# Legacy ``recognize()`` returned matches grouped by year length +# (all 4-digit first, then 2-digit), while the staged pipeline returns +# document order. The engine sorts by ``start`` before dedup, so +# end-to-end ``canonicalize()`` is identical; direct ``recognize()`` +# order is now document-order. This test locks the new contract and proves the +# sorted multiset is still parity-equivalent. + + +@pytest.mark.property +def test_us_date_document_order() -> None: + """Staged US grammar emits document order; legacy grouped by year length.""" + text = "01/02/26 foo 01/02/2026" + legacy = LegacyUSDateGrammar() + new = USDateGrammar() + legacy_matches = legacy.recognize(text) + new_matches = new.recognize(text) + # Staged is document order + assert [m.raw_text for m in new_matches] == ["01/02/26", "01/02/2026"] + # Legacy is grouped (4-digit first) + assert [m.raw_text for m in legacy_matches] == ["01/02/2026", "01/02/26"] + # Sorted by start they are equivalent + assert sorted(new_matches, key=lambda m: m.start) == sorted( + legacy_matches, key=lambda m: m.start + ) + + +@pytest.mark.property +def test_european_date_document_order() -> None: + """Staged European grammar emits document order; legacy grouped.""" + text = "26/07/26 foo 26/07/2026" + legacy = LegacyEuropeanDateGrammar() + new = EuropeanDateGrammar() + legacy_matches = legacy.recognize(text) + new_matches = new.recognize(text) + assert [m.raw_text for m in new_matches] == ["26/07/26", "26/07/2026"] + assert [m.raw_text for m in legacy_matches] == ["26/07/2026", "26/07/26"] + assert sorted(new_matches, key=lambda m: m.start) == sorted( + legacy_matches, key=lambda m: m.start + ) + + +@pytest.mark.property +def test_ipv6_document_order() -> None: + """Staged IPv6 emits document order; legacy grouped full before compressed. + + Compressed ``::1`` precedes full ``2001:0db8:85a3:...`` in this input. + Legacy returns [full, compressed]; staged returns [compressed, full]. + Sorted by start they are equivalent. + """ + text = "::1 foo 2001:0db8:85a3:0000:0000:8a2e:0370:7334" + legacy = LegacyIPv6Grammar() + new = IPv6Grammar() + legacy_matches = legacy.recognize(text) + new_matches = new.recognize(text) + assert [m.raw_text for m in new_matches] == [ + "::1", + "2001:0db8:85a3:0000:0000:8a2e:0370:7334", + ] + assert [m.raw_text for m in legacy_matches] == [ + "2001:0db8:85a3:0000:0000:8a2e:0370:7334", + "::1", + ] + assert sorted(new_matches, key=lambda m: m.start) == sorted( + legacy_matches, key=lambda m: m.start + ) diff --git a/tests/unit/test_boundary_guards.py b/tests/unit/test_boundary_guards.py new file mode 100644 index 00000000..ed9aef7a --- /dev/null +++ b/tests/unit/test_boundary_guards.py @@ -0,0 +1,75 @@ +"""BoundaryGuard unit tests — 10 distinct lookarounds, one family.""" + +from __future__ import annotations + +import re + +import pytest + +from paxman.core.grammar.boundary import BoundaryGuard + +pytestmark = pytest.mark.unit + + +def test_word_sign_guard_blocks_inside_token() -> None: + g = BoundaryGuard.word_sign() # (? None: + # SIUnit includes ° in the lookaround; Currency does not. + g_word = BoundaryGuard.word_sign() + g_degree = BoundaryGuard.degree_word_sign() + assert g_word.lookbehind != g_degree.lookbehind + assert "°" in g_degree.lookbehind + + +def test_scheme_char_guard_for_url() -> None: + g = BoundaryGuard.scheme_char() # (? None: + g = BoundaryGuard.e164() # (? None: + g = BoundaryGuard.digit() # (? None: + g = BoundaryGuard.word_only() # (? None: + g = BoundaryGuard.phone_national() + # A bare national number at string start matches. + assert g.wrap("555").search("555") is not None + # A national number preceded by "+1 " is blocked by the 4-lookbehind chain. + assert g.wrap("555").search("+1 555") is None + # A national number preceded by a word boundary (space) matches. + assert g.wrap("555").search("call 555") is not None + + +def test_ipv6_token_guard() -> None: + g = BoundaryGuard.ipv6_token() + # At string start matches. + assert g.wrap("2001:db8").search("2001:db8") is not None + # Preceded by a non-delimiter char is blocked. + assert g.wrap("2001:db8").search("x2001:db8") is None + # Preceded by a delimiter (space) and followed by a delimiter (comma) matches. + assert g.wrap("2001:db8").search(" 2001:db8,") is not None diff --git a/tests/unit/test_lexicon_alternation.py b/tests/unit/test_lexicon_alternation.py new file mode 100644 index 00000000..616c9abc --- /dev/null +++ b/tests/unit/test_lexicon_alternation.py @@ -0,0 +1,60 @@ +"""LexiconAlternation unit tests.""" + +from __future__ import annotations + +import pytest + +from paxman.capabilities.Currency.notation import CurrencyNotation +from paxman.core.grammar.boundary import BoundaryGuard +from paxman.core.grammar.lexicon import LexiconAlternation +from paxman.core.grammar.stages import LexiconStage, PipelineState, WholeInputLookup + +pytestmark = pytest.mark.unit + + +def test_longest_first_ordering() -> None: + alt = LexiconAlternation(tokens=["$", "US$", "A$"], longest_first=True) + # US$ (3) before A$ (2) before $ (1); qualified-first tie-break is secondary + assert alt.ordered_tokens[0] == "US$" + assert alt.alternation.startswith("US\\$") + + +def test_qualified_first_within_same_length() -> None: + alt = LexiconAlternation(tokens=["A$", "$$"], longest_first=True) + # Same length (2), so qualified-first tie-break determines order: "A$" before "$$" + assert alt.ordered_tokens[0] == "A$" + assert alt.ordered_tokens[1] == "$$" + + +def test_alternation_is_escaped() -> None: + alt = LexiconAlternation(tokens=["$", "("], longest_first=True) + assert r"\$" in alt.alternation + assert r"\(" in alt.alternation + + +def test_lexicon_stage_emits_matches_with_boundary() -> None: + stage = LexiconStage[CurrencyNotation]( + tokens=["$", "US$"], + boundary=BoundaryGuard.word_sign(), + longest_first=True, + notation_fn=lambda token: CurrencyNotation(text=token, shape="symbol"), + ) + state = PipelineState(text="Pay US$ and $", matches=[], scratch={}) + out = stage.run(state) + assert len(out.matches) == 2 + assert out.matches[0].raw_text == "US$" + assert out.matches[1].raw_text == "$" + + +def test_whole_input_lookup_emits_original_trimmed_case() -> None: + stage: WholeInputLookup[CurrencyNotation] = WholeInputLookup( + keys={"us", "eur"}, # normalized keys + notation_fn=lambda trimmed: CurrencyNotation(text=trimmed, shape="code"), + normalizer=lambda s: s.lower(), + ) + state = PipelineState(text=" Us ", matches=[], scratch={}) + out = stage.run(state) + assert len(out.matches) == 1 + assert out.matches[0].raw_text == "Us" # original trimmed, not "us" + assert out.matches[0].start == 2 + assert out.matches[0].end == 4 diff --git a/tests/unit/test_phone_helpers.py b/tests/unit/test_phone_helpers.py new file mode 100644 index 00000000..38ea00a4 --- /dev/null +++ b/tests/unit/test_phone_helpers.py @@ -0,0 +1,43 @@ +"""Coverage for Phone grammar helper strip_separators (both branches).""" + +from __future__ import annotations + +import pytest + +from paxman.capabilities.Phone.grammar._common import strip_separators +from paxman.capabilities.Phone.grammar.e164_recognition import ( + strip_separators as e164_strip, +) +from paxman.capabilities.Phone.grammar.international_00_recognition import ( + strip_separators as intl_strip, +) +from paxman.capabilities.Phone.grammar.national_recognition import ( + strip_separators as national_strip, +) +from paxman.capabilities.Phone.grammar.tel_uri_recognition import ( + strip_separators as tel_strip, +) + +pytestmark = pytest.mark.unit + + +def test_strip_separators_plus_true() -> None: + """PLUS branch strips plus and separators.""" + # All four grammars re-export the same shared helper — verify identity. + assert e164_strip is strip_separators + assert intl_strip is strip_separators + assert national_strip is strip_separators + assert tel_strip is strip_separators + assert strip_separators("+1 (555) 123-4567", plus=True) == "15551234567" + assert strip_separators("00 44 20 7946 0958", plus=True) == "00442079460958" + assert strip_separators("tel:+1-201-555-0123", plus=True) == "tel:12015550123" + + +def test_strip_separators_plus_false() -> None: + """Non-plus branch strips only separators, keeps plus.""" + # Directly exercises the fallback line `return value.translate(...)` in _common.py. + assert strip_separators("a (b) - c", plus=False) == "abc" + assert strip_separators("+1 (555) 123-4567", plus=False) == "+15551234567" + assert strip_separators("00 44 20 7946 0958", plus=False) == "00442079460958" + assert strip_separators("(555) 123-4567", plus=False) == "5551234567" + assert strip_separators("tel:+1-201-555-0123", plus=False) == "tel:+12015550123" diff --git a/tests/unit/test_pipeline_stages.py b/tests/unit/test_pipeline_stages.py new file mode 100644 index 00000000..5f9a94a5 --- /dev/null +++ b/tests/unit/test_pipeline_stages.py @@ -0,0 +1,66 @@ +"""Unit tests for PipelineState and PipelineGrammar skeleton.""" + +from __future__ import annotations + +import re + +import pytest + +from paxman.capabilities.Date.notation import DateNotation +from paxman.core.domain import Grammar +from paxman.core.grammar import PipelineGrammar, PipelineState +from paxman.core.grammar.stages import RegexStage, StandardPre + +pytestmark = pytest.mark.unit + + +def _date_notation(m: re.Match[str]) -> DateNotation: + return DateNotation(N1=m.group(1), N2=m.group(2), N3=m.group(3)) + + +class _ProbeGrammar(PipelineGrammar[DateNotation]): + """Minimal PipelineGrammar for skeleton test.""" + + name = "probe_recognition" + semantics = "probe_recognition" + pre = StandardPre(empty_guard=True) + regex = RegexStage( + r"(? None: + state = PipelineState(text="hello", matches=[], scratch={}) + assert state.text == "hello" + with pytest.raises(AttributeError): + state.text = "mutated" # type: ignore[misc] + + +def test_pipeline_grammar_is_grammar_subclass() -> None: + g = _ProbeGrammar() + assert isinstance(g, Grammar) + assert g.name == "probe_recognition" + assert g.semantics == "probe_recognition" + + +def test_pipeline_grammar_recognize_delegates_to_stages() -> None: + g = _ProbeGrammar() + results = g.recognize("2026-01-15 foo 2026/01/15") + assert len(results) == 1 + assert results[0].raw_text == "2026-01-15" + assert results[0].start == 0 + assert results[0].end == 10 + + +def test_empty_input_early_exit_via_pre() -> None: + g = _ProbeGrammar() + assert g.recognize("") == [] + assert g.recognize(" ") == [] + + +def test_grammar_with_no_stages_returns_empty() -> None: + class _EmptyGrammar(PipelineGrammar[DateNotation]): + name = "empty_recognition" + semantics = "empty_recognition" + + assert _EmptyGrammar().recognize("2026-01-15") == []