diff --git a/packages/overture-schema-codegen/docs/design.md b/packages/overture-schema-codegen/docs/design.md index b58dad1b7..c8950d958 100644 --- a/packages/overture-schema-codegen/docs/design.md +++ b/packages/overture-schema-codegen/docs/design.md @@ -317,18 +317,33 @@ Four dispatch mechanisms: ### Check Builder `pyspark/check_builder.py` walks `FieldSpec` trees to produce `Check` and `ModelCheck` -IR. Resolves the mapping from nested field paths to PySpark array iteration patterns, -producing a `FieldPath` (`ScalarPath` or `ArrayPath`) on each `Check`: - -- **Scalar field** -- `ScalarPath`; renders as `F.col("field")` -- **Top-level array** -- `ArrayPath` with one `ArraySegment`; renders as - `array_check("field", lambda el: ...)` -- **Field inside an array element** -- `ArrayPath` with struct navigation after the - array segment; renders as `array_check("array_col", lambda el: el["field"])` -- **Nested array inside an array** -- `ArrayPath` with multiple `ArraySegment`s; - renders as `nested_array_check("outer", lambda el: array_check(el["inner"], ...))` -- **Multiple nesting levels** -- chained `nested_array_check` with struct segments - navigating between array iterations +IR. Resolves the mapping from nested field paths to PySpark iteration patterns, +producing a `FieldPath` (`Direct` or `Iterated`) on each `Check`. A `Direct` locates +a value reached with no iteration; an `Iterated` mixes struct segments with one or +more iterating segments (`ArraySegment` for a `list`, `MapSegment` for a `dict[K, +V]` projected to its keys or values); at render time each iterating segment -- +named or anonymous alike -- becomes its own render frame (one `array_check`/ +`map_*_check` call). A container nested directly inside another with no field +name between them (`list[list[X]]`, `dict[K, list[X]]`, `list[dict]`, a map +projected inside an array element, an array inside a map value) is an +*anonymous* iterating segment (`is_anonymous`) -- its render frame carries no +struct descent from the previous one. `promote_terminal` performs the entry +into a container at the point the walker reaches it, replacing a struct +terminal with a named iterating segment or appending an anonymous one when the +terminal already iterates: + +- **Scalar field** -- `Direct`; renders as `F.col("field")` +- **Top-level array or map** -- `Iterated` with one named iterating segment; + renders as `array_check("field", lambda el: ...)` or + `map_keys_check`/`map_values_check("field", lambda k_or_v: ...)` +- **Field inside an array or map element** -- struct segments after the iterating + segment; renders as `array_check("col", lambda el: el["field"])` +- **`list[list[X]]`, `dict[K, list[X]]`, a nested map, or a map projected inside + an array** -- an anonymous or named second iterating segment adds another + render frame, rendered by folding the flattening variant (`nested_array_check`, + `nested_map_{keys,values}_check`) around the inner render frame's helper +- **Multiple nesting levels** -- chained flattening helpers with struct segments + navigating between render frames Union handling: variant-specific fields are annotated with `ColumnGuard` or `ElementGuard` discriminator gates. `Check.guards` is AND-composed at render time. diff --git a/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/_render_common.py b/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/_render_common.py index 6747213f8..4ef56540c 100644 --- a/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/_render_common.py +++ b/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/_render_common.py @@ -31,7 +31,7 @@ from jinja2 import Environment, FileSystemLoader -from overture.schema.system.field_path import ArrayPath, MapProjection +from overture.schema.system.field_path import Iterated, MapProjection from .check_ir import Check, Guard, ModelCheck from .constraint_dispatch import ForbidIf, RequireIf, model_constraint_function @@ -77,21 +77,27 @@ "check_struct_unique": "_unique", } -_MAP_RUNTIME_HELPERS: dict[MapProjection, str] = { - MapProjection.KEY: "map_keys_check", - MapProjection.VALUE: "map_values_check", +_MAP_RUNTIME_HELPERS: dict[tuple[MapProjection, bool], str] = { + (MapProjection.KEY, False): "map_keys_check", + (MapProjection.VALUE, False): "map_values_check", + (MapProjection.KEY, True): "nested_map_keys_check", + (MapProjection.VALUE, True): "nested_map_values_check", } -def map_runtime_helper(projection: MapProjection) -> str: +def map_runtime_helper(projection: MapProjection, *, flatten: bool = False) -> str: """Map a projection to its PySpark column-patterns helper name. `MapProjection.KEY` -> `map_keys_check`; - `MapProjection.VALUE` -> `map_values_check`. This is a pyspark-layer + `MapProjection.VALUE` -> `map_values_check`. When *flatten* is set (the + map holds further iteration, e.g. `dict[K, list]`, so the projected + element check returns an `array`), the flattening variant is + named instead (`nested_map_keys_check` / `nested_map_values_check`) -- + the map analogue of `nested_array_check`. This is a pyspark-layer concern; the mapping lives here rather than on `MapProjection` itself (a system-package enum) to avoid a layering violation. """ - return _MAP_RUNTIME_HELPERS[projection] + return _MAP_RUNTIME_HELPERS[(projection, flatten)] _TEMPLATES_DIR = Path(__file__).parent / "templates" @@ -209,8 +215,13 @@ def _model_check_base_label(check: ModelCheck) -> str: carries a single target field (multi-field decorators split at dispatch time). - Other kinds (`require_any_of`, `radio_group`, `min_fields_set`) - name the whole constraint; on `ArrayPath` targets they use the + name the whole constraint; on `Iterated` targets they use the path itself so anchors are distinguishable across nestings. + + Every `Iterated` target (array, map, or mixed) uses the iterated + formula -- the anchor-disambiguation reason that motivates it for + arrays applies identically to maps and mixed paths. `Direct` targets + keep the row-root formula. """ match check.descriptor: case RequireIf(): @@ -218,11 +229,11 @@ def _model_check_base_label(check: ModelCheck) -> str: case ForbidIf(): kind_suffix = "_forbidden" case _: - if isinstance(check.target, ArrayPath): + if isinstance(check.target, Iterated): return str(check.target) return check_name(model_constraint_function(check.descriptor)) target = check.descriptor.field_names[0] - if not isinstance(check.target, ArrayPath): + if not isinstance(check.target, Iterated): return f"{target}{kind_suffix}" return f"{check.target}.{target}{kind_suffix}" diff --git a/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/check_builder.py b/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/check_builder.py index d89f9a586..4a35a09d1 100644 --- a/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/check_builder.py +++ b/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/check_builder.py @@ -7,10 +7,10 @@ `Check` (required first, then enum, then dispatched constraints), deduplicate, and split column-level checks into separate suffixed checks. - Target resolution: a shape walker descends each field's `FieldShape` - tree, building the `ScalarPath` or `ArrayPath` target by appending + tree, building the `Direct` or `Iterated` target by appending segments as it goes -- so the path read in the code is the path that - lands in the IR. Entering a `list[...]` layer promotes the path's - terminal struct segment to an iterated `ArraySegment`. + lands in the IR. Entering a `list[...]` or `dict[K, V]` layer promotes + the path's terminal segment to an iterated `ArraySegment` / `MapSegment`. - Subtype gating: annotate variant-specific fields with discriminator `Guard`s, synthesize forbid_if/require_if for absent or required variants, and gate check_required under nullable struct ancestors. @@ -25,15 +25,13 @@ from typing_extensions import assert_never from overture.schema.system.field_path import ( - ArrayPath, ArraySegment, + Direct, FieldPath, - MapPath, + Iterated, MapProjection, MapSegment, - ScalarPath, - promote_terminal_array, - promote_terminal_map, + promote_terminal, ) from overture.schema.system.model_constraint import ( FieldEqCondition, @@ -55,12 +53,8 @@ UnionRef, ) from ..extraction.field_walk import ( - all_constraints, enum_source, - has_array_layer, - terminal_of, terminal_primitive, - terminal_scalar, ) from ..extraction.literal_alternatives import LiteralAlternatives from ..extraction.specs import FieldSpec, ModelSpec, RecordSpec, UnionSpec @@ -253,14 +247,6 @@ def _walk_field_shape( case NewTypeShape(name=name, inner=inner): nt_descriptors = dispatch_newtype(name) if nt_descriptors is not None: - if isinstance(path.segments[-1], ArraySegment): - # A NewType with a dispatch override nested under a list - # layer has no schema field; raise to keep the gap loud - # rather than emit an untested target (cf. list[list[Union]]). - raise NotImplementedError( - f"NewType with a dispatch override ({name}) nested " - "under a list layer is not supported" - ) descriptors = list(nt_descriptors) if required: descriptors.insert(0, _required_descriptor(required_gate)) @@ -300,7 +286,7 @@ def _walk_field_shape( ) sub_checks, terminal = _walk_field_shape( element, - promote_terminal_array(path), + promote_terminal(path), base_type=base_type, required=False, required_gate=required_gate, @@ -308,19 +294,11 @@ def _walk_field_shape( ) return [*checks, *sub_checks], terminal - case UnionRef(): - terminal_seg = path.segments[-1] - if isinstance(terminal_seg, ArraySegment) and terminal_seg.iter_count > 1: - # `list[list[Union]]` would build a multi-iter union target, - # but no schema field has that shape. The walker raises to - # keep the gap loud rather than silently emit one. - raise NotImplementedError( - "Union nested under multiple list layers " - "(list[list[Union]]) is not supported" - ) - return _ref_terminal_checks(shape, path, required, required_gate) - - case ModelRef(): + case UnionRef() | ModelRef(): + # A union or model reached under any array/map nesting (including + # `list[list[Union]]`) descends the same way: the fold wraps each + # variant-gated field check at the innermost element, where the + # `ElementGuard`'s discriminator co-locates with the leaf accessor. return _ref_terminal_checks(shape, path, required, required_gate) case Primitive() | LiteralScalar() | AnyScalar(): @@ -338,7 +316,7 @@ def _walk_field_shape( # required check and any map-level constraints (currently always # empty -- map-level length constraints are rejected at # extraction). The key and value layers are walked separately so - # their per-key/per-value constraints land on `MapPath` targets. + # their per-key/per-value constraints land on `Iterated` targets. # A `ModelRef`/`UnionRef` projection hands back a `_ShapeTerminal` # for the caller to descend into, exactly as a `list[Model]` # element does. @@ -357,8 +335,17 @@ def _walk_field_shape( value_shape, path, MapProjection.VALUE ) if key_terminal is not None and value_terminal is not None: + # Not a representational limit: the taxonomy encodes + # `a{key}.kfield` and `a{value}.vfield` independently. The + # barrier is that `_walk_field_shape` returns a single + # `_ShapeTerminal`, so the FieldSpec recursion can descend only + # one projection's sub-model -- a `dict[Model, Model]` needs + # both descended. Lifting it means returning two terminals. raise NotImplementedError( - "map with a model key and a model value is not supported" + "dict[Model, Model] reaches a sub-model through both its " + "key and value projection, but _walk_field_shape returns a " + "single terminal, so only one projection's sub-model can be " + "descended" ) terminal = value_terminal if value_terminal is not None else key_terminal return [*field_checks, *key_checks, *value_checks], terminal @@ -410,108 +397,31 @@ def _terminal_scalar_checks( return [] -@dataclass(frozen=True) -class MapProjectionVerdict: - """Whether a map's projected key/value shape is representable as a `MapPath`. - - `reason` names why an unrepresentable shape was rejected (for the - `NotImplementedError` message); it is `None` when `representable` is True. - `has_value_to_validate` reports whether the projected shape carries a - constraint or descends into a model -- the loud/quiet discriminator: an - unrepresentable shape with something to validate raises, an unrepresentable - shape with nothing to validate is silently dropped. - """ - - representable: bool - reason: str | None - has_value_to_validate: bool - - -def classify_map_projection( - sub_shape: FieldShape, - map_path: FieldPath, -) -> MapProjectionVerdict: - """Classify a map's projected key/value shape against the representable bound. - - The single source of truth for which map projections a `MapPath` can - locate. The representable shape: a scalar terminal or `ModelRef`/`UnionRef` - terminal, reached WITHOUT array iteration (`map_path` is not an - `ArrayPath`), with no `ArrayOf` layer in the projected shape. Both - `_map_projection_checks` (shape-level) and the path-level guards in - `field_path.py` (`promote_terminal_map` rejecting an `ArrayPath`) enforce - this boundary; this classifier states it once so the prohibitions agree by - construction rather than by parallel maintenance. - - Two shapes fall outside the bound and have no `MapPath`: - - - a map reached through an array (`list[dict[K, V]]`, a `map_path` that is - an `ArrayPath`), whose key/value can't anchor a struct-prefixed `MapPath`; - - a key/value carrying an array layer (`dict[K, list[V]]`), whose scalar - terminal sits under an `ArrayOf` that `terminal_scalar` would unwrap. - """ - is_ref_terminal = isinstance(terminal_of(sub_shape), (ModelRef, UnionRef)) - has_value_to_validate = bool(all_constraints(sub_shape)) or is_ref_terminal - if isinstance(map_path, ArrayPath): - return MapProjectionVerdict( - representable=False, - reason="map reached through an array is not representable", - has_value_to_validate=has_value_to_validate, - ) - if has_array_layer(sub_shape): - return MapProjectionVerdict( - representable=False, - reason="map value carrying a list layer (dict[K, list[V]]) is not representable", - has_value_to_validate=has_value_to_validate, - ) - if not is_ref_terminal and terminal_scalar(sub_shape) is None: - return MapProjectionVerdict( - representable=False, - reason="constraint on a non-scalar terminal", - has_value_to_validate=has_value_to_validate, - ) - return MapProjectionVerdict( - representable=True, reason=None, has_value_to_validate=has_value_to_validate - ) - - def _map_projection_checks( sub_shape: FieldShape, map_path: FieldPath, projection: MapProjection, ) -> tuple[list[Check], _ShapeTerminal | None]: - """Walk a map's key or value shape, emitting checks on a `MapPath` target. - - Supports two shapes reached without array iteration: a scalar terminal - (`dict[K, scalar]` -- per-key/value constraints land on a bare `MapPath`) - and a `ModelRef`/`UnionRef` terminal (`dict[K, Model]` -- the returned - `_ShapeTerminal` lets the caller descend into the model's fields and - constraints on a `MapPath` leaf, mirroring a `list[Model]` element). - - `classify_map_projection` is the arbiter of which shapes are - representable. An unrepresentable shape carrying something to validate - (`has_value_to_validate`) raises `NotImplementedError` to keep the dropped - check loud; an unrepresentable shape with nothing to validate yields no - checks. The constraint -- not the shape alone -- is what stays loud, - matching the silent treatment of unconstrained maps. + """Walk a map's key or value shape, emitting checks on an `Iterated` target. + + Promotes *map_path*'s terminal into a `MapSegment` projecting the chosen + side, then walks the projected shape unconditionally. Every map/array + nesting -- `dict[K, scalar]` (per-key/value constraints on a bare map + frame), `dict[K, Model]` (the returned `_ShapeTerminal` lets the caller + descend the value model on a map leaf), `dict[K, list]`, `dict[K, dict]`, + and a map reached through an array -- is now representable, so the walk + needs no representability gate: an unconstrained shape simply yields no + checks. """ - verdict = classify_map_projection(sub_shape, map_path) - if not verdict.representable: - if verdict.has_value_to_validate: - raise NotImplementedError( - f"map {projection.value} on an unsupported shape " - f"({verdict.reason}) is not supported ({sub_shape!r})" - ) - return [], None primitive = terminal_primitive(sub_shape) - sub_checks, terminal = _walk_field_shape( + return _walk_field_shape( sub_shape, - promote_terminal_map(map_path, projection), + promote_terminal(map_path, projection=projection), base_type=primitive.base_type if primitive is not None else None, required=False, required_gate=None, carried_element=[], ) - return sub_checks, terminal def _ref_terminal_checks( @@ -540,7 +450,7 @@ def _ref_terminal_checks( def _build_field_checks( field_spec: FieldSpec, - prefix: FieldPath = ScalarPath(), + prefix: FieldPath = Direct(), *, nullable_gate: FieldPath | None = None, arm: str | None = None, @@ -552,9 +462,10 @@ def _build_field_checks( shared. It propagates to any model constraints discovered through this field's sub-models so per-arm test modules can filter them correctly. """ - # `prefix` is a ScalarPath/ArrayPath, or a MapPath when descending into - # a `dict[K, Model]` value model -- all three define `append_struct`, - # which extends the path's struct leaf with this field's name. + # `prefix` is a `Direct` or an `Iterated` (the latter when descending + # into a list element or a `dict[K, Model]` value model) -- both define + # `append_struct`, which extends the path's struct leaf with this field's + # name. path = prefix.append_struct(field_spec.name) checks, terminal = _walk_field_shape( field_spec.shape, @@ -599,7 +510,7 @@ def _build_field_checks( def _recurse_into_model( model_spec: RecordSpec, - prefix: FieldPath = ScalarPath(), + prefix: FieldPath = Direct(), is_optional: bool = False, nullable_gate: FieldPath | None = None, *, @@ -608,7 +519,7 @@ def _recurse_into_model( """Walk a MODEL-kind field's children plus its model-level constraints. `prefix` is the terminal path the shape walker reached the `ModelRef` - at, defaulting to the empty `ScalarPath()` at the row root. Its terminal + at, defaulting to the empty `Direct()` at the row root. Its terminal segment is an `ArraySegment` (the field is a list) or a `MapSegment` (the field is a `dict[K, Model]` reached through its key/value projection) exactly when the model is reached through iteration, which @@ -640,33 +551,35 @@ def _recurse_into_model( model_checks.extend(sub_model_checks) if model_spec.constraints: - constraint_gate = ( - prefix - if is_optional and not field_is_iterated and isinstance(prefix, ArrayPath) - else None - ) + # The constraint applies wherever the model is reached, so it inherits + # the same nullable gate as the model's fields: `child_gate` is the + # optional-ancestor path (or the model's own optional prefix) that must + # be non-null for the constraint to apply, and `None` once inside any + # iterated container (the fold handles per-element nullability). The + # renderer wraps a Direct-target constraint in `F.when(gate.isNotNull())` + # and an Iterated-target one element-relatively. sub_model_constraint_checks = _dispatch_model_constraints( model_spec.constraints, model_spec.fields, target=_model_constraint_target(prefix), arm=arm, - gate=constraint_gate, + gate=child_gate, ) - if sub_model_constraint_checks: - _guard_struct_nested_anchor(prefix, model_spec.name) model_checks.extend(sub_model_constraint_checks) return field_checks, model_checks def _is_struct_only_prefix(prefix: FieldPath) -> bool: - """Non-root struct path with no array traversal. + """Non-root struct path with no iteration. - True when `prefix` has one or more struct segments but no array + True when `prefix` has one or more struct segments but no array/map iteration -- meaning discriminator column access and model-constraint targeting cannot use the prefix without resolving it into a - struct-qualified path, which the current renderer does not support. + struct-qualified path, which the current renderer does not support. A + `Direct` with segments is the only struct-only prefix; any `Iterated` + (array- or map-reached) is a valid anchor. """ - return not isinstance(prefix, ArrayPath) and bool(prefix.segments) + return isinstance(prefix, Direct) and bool(prefix.segments) def _reject_struct_only_prefix(prefix: FieldPath, message: str) -> None: @@ -682,23 +595,24 @@ def _reject_struct_only_prefix(prefix: FieldPath, message: str) -> None: def _guard_struct_nested_anchor(prefix: FieldPath, name: str) -> None: - """Raise when emitting a model constraint at a struct-only prefix. - - See `_model_constraint_target`: in that case the constraint's target - collapses to the row root, which is wrong for any non-skipped - constraint. Today only `NoExtraFieldsConstraint` reaches here (and - dispatches to None); a real descriptor at this depth is a renderer - gap, not a normal case. A `MapPath` is exempt -- it is a valid anchor - (`_model_constraint_target` keeps it, and the renderer wraps the check - in `map_values_check`/`map_keys_check`). + """Raise when a struct-nested UNION emits union-level or exclusivity checks. + + A plain model constraint at a struct-only prefix is supported: the target + is the struct prefix and the renderer qualifies field references + (`F.col("details.foo")`, see `_model_constraint_target`). A discriminated + UNION reached through a plain struct is not: its synthesized exclusivity + checks and union-level constraints interlock with the variant-field + `ColumnGuard`s (`_guard_struct_nested_variant_fields`), which render the + discriminator as a top-level column rather than a struct-qualified path. + Gating the whole union case loudly keeps that mis-columning from shipping. + A map-reached `Iterated` prefix is a valid anchor, so `_is_struct_only_prefix` + -- `False` for any `Iterated` -- exempts it automatically. """ - if isinstance(prefix, MapPath): - return _reject_struct_only_prefix( prefix, - f"Model constraint on struct-nested {name!r} " - f"(reached at {prefix!r}) -- the renderer has no anchor " - "for nested-struct model constraints.", + f"Model constraint on struct-nested union {name!r} " + f"(reached at {prefix!r}) -- the discriminator gating renders " + "as a top-level column, not a struct-qualified path.", ) @@ -721,18 +635,64 @@ def _guard_struct_nested_variant_fields(prefix: FieldPath, name: str) -> None: ) +def _iteration_depth(path: FieldPath) -> int: + """Number of iteration frames (`Array`/`Map` segments) in *path*. + + Each iterating segment -- named or anonymous -- is one lambda frame in + the renderer's fold, so the count is the depth at which the innermost + element variable is bound. A `Direct` path binds no element variable + and has depth 0. + """ + if isinstance(path, Iterated): + return sum( + 1 for s in path.segments if isinstance(s, (ArraySegment, MapSegment)) + ) + return 0 + + +def _guard_variant_field_past_element( + prefix: FieldPath, checks: list[Check], name: str +) -> None: + """Raise when an `ElementGuard`'d variant field iterates past its discriminator. + + An `ElementGuard` carries the discriminator of a union reached at + *prefix*, and the renderer applies it at the innermost iteration + variable (`_render_iterated_check_expr`). That placement is correct only + when the guarded check binds the same element as the discriminator -- + i.e. the check iterates no further than *prefix*. When a variant field is + itself an iterated container (e.g. `list[list[Union{codes: list[int]}]]`, + where `codes[]` adds a third iteration past the union element), the + innermost variable is that deeper element, where the discriminator does + not live. The renderer has no per-guard depth info to place the guard at + the discriminator's shallower iteration level, so the render would + silently gate on the wrong element. Raise instead. + """ + prefix_depth = _iteration_depth(prefix) + for ck in checks: + if _iteration_depth(ck.target) > prefix_depth: + raise NotImplementedError( + f"Discriminated union {name!r}: variant field check reaches " + f"its value through iteration beyond the discriminator's " + f"element (discriminator element at {prefix!r}, check target " + f"{ck.target!r}). The ElementGuard is applied at the innermost " + f"iteration variable, so it cannot be placed at the " + f"discriminator's iteration level when the variant field " + f"iterates further." + ) + + def _recurse_into_union( union_spec: UnionSpec, - prefix: FieldPath = ScalarPath(), + prefix: FieldPath = Direct(), *, arm: str | None = None, ) -> tuple[list[Check], list[ModelCheck]]: """Walk a UNION-kind field's variants, gathering Checks and ModelChecks. `prefix` is the terminal path the shape walker reached the `UnionRef` - at; the union's variant fields live directly under it. An `ArrayPath` - prefix means the union is reached through array iteration, so variant - gates are element-level and model constraints target that path. + at; the union's variant fields live directly under it. An `Iterated` + prefix means the union is reached through array or map iteration, so + variant gates are element-level and model constraints target that path. `arm` is the outer union arm whose variant-specific field reached this inner union. It tags any model constraints discovered here so they @@ -757,34 +717,29 @@ def _recurse_into_union( def _model_constraint_target(prefix: FieldPath) -> FieldPath: - """Where a model constraint's check should be anchored. - - Three supported cases: - - - `ArrayPath` -- constraints on a sub-model reached through array - iteration target the array path (so the renderer wraps the check - in `array_check`). - - `MapPath` -- constraints on a `dict[K, Model]` value model target the - map path (so the renderer wraps the check in `map_values_check`), - mirroring the array case. - - Empty or struct-only `ScalarPath` -- constraints anchor at the row - root. Pure struct nesting (e.g. `Names` reached at - `ScalarPath('names')`) collapses here because the renderer has no - anchor for nested-struct model constraints. The only constraint kind - currently reachable through pure struct nesting is - `NoExtraFieldsConstraint`, which `dispatch_model_constraint` - discards before the target is consulted, so the collapse is - observationally inert today; a non-skipped constraint at this depth - would surface as a wrong-anchor bug. + """Where a model constraint's check should be anchored -- the prefix itself. + + The check anchors exactly where the shape walker reached the constrained + model, so this is the identity on `prefix`: + + - `Iterated` -- a sub-model reached through array or map iteration; the + renderer wraps the check in the iteration fold (`array_check` for an + array-reached model, `map_values_check` for a `dict[K, Model]` value + model), and field references become element-relative accessors. + - Struct-only `Direct` (e.g. `Details` reached at `Direct('details')`) -- + a sub-model reached through a plain struct field; the renderer qualifies + every field reference with the struct prefix (`F.col("details.foo")`). + - Empty `Direct` -- a row-root constraint; field references are top-level + columns. """ - return prefix if isinstance(prefix, (ArrayPath, MapPath)) else ScalarPath() + return prefix def _dispatch_model_constraints( constraints: tuple[ModelConstraint, ...], fields: list[FieldSpec], *, - target: FieldPath = ScalarPath(), + target: FieldPath = Direct(), arm: str | None = None, gate: FieldPath | None = None, ) -> list[ModelCheck]: @@ -813,7 +768,7 @@ def _singleton_arm(values: tuple[str, ...]) -> str | None: def _field_checks_for_union( spec: UnionSpec, value_by_class: dict[type[BaseModel], str], - prefix: FieldPath = ScalarPath(), + prefix: FieldPath = Direct(), *, arm: str | None = None, ) -> tuple[list[Check], list[ModelCheck]]: @@ -825,8 +780,10 @@ def _field_checks_for_union( discriminator is irrelevant to per-arm test filtering, which always keys on the outermost union's discriminator. """ + # A union reached through any iterated container (array or map element) + # is element-gated; only a row-/struct-level union uses a column gate. guard_cls: type[Guard] = ( - ElementGuard if isinstance(prefix, ArrayPath) else ColumnGuard + ElementGuard if isinstance(prefix, Iterated) else ColumnGuard ) field_checks: list[Check] = [] model_checks: list[ModelCheck] = [] @@ -855,6 +812,8 @@ def _field_checks_for_union( # then an `ElementGuard` from the nested union the field # lives in). guard: Guard = guard_cls(discriminator=discriminator, values=values) + if isinstance(guard, ElementGuard): + _guard_variant_field_past_element(prefix, checks, spec.name) checks = [replace(ck, guards=(guard, *ck.guards)) for ck in checks] field_checks.extend(checks) return field_checks, model_checks @@ -863,7 +822,7 @@ def _field_checks_for_union( def _model_checks_for_union( spec: UnionSpec, arm_by_class: dict[type[BaseModel], str], - target: FieldPath = ScalarPath(), + target: FieldPath = Direct(), *, arm: str | None = None, ) -> list[ModelCheck]: @@ -904,7 +863,7 @@ def _model_checks_for_union( def _exclusivity_checks_for_union( spec: UnionSpec, value_by_class: dict[type[BaseModel], str], - target: FieldPath = ScalarPath(), + target: FieldPath = Direct(), *, arm: str | None = None, ) -> list[ModelCheck]: @@ -992,7 +951,7 @@ def build_checks( ) -> tuple[list[Check], list[ModelCheck]]: """Build all check IR for a feature spec. - Roots the walk at the empty `ScalarPath()` and delegates to the same + Roots the walk at the empty `Direct()` and delegates to the same helpers used at every nested level (`_recurse_into_union` for unions, `_recurse_into_model` for models), so the row-root and nested cases share one path. diff --git a/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/check_ir.py b/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/check_ir.py index b418c37c2..21930470e 100644 --- a/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/check_ir.py +++ b/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/check_ir.py @@ -2,10 +2,10 @@ Sum types describe each check's structural placement: -- `Check.target: FieldPath` -- a `ScalarPath` or `ArrayPath` locating +- `Check.target: FieldPath` -- a `Direct` or `Iterated` locating where the descriptor's expression is evaluated. The choice of variant - signals whether the renderer wraps the expression in `array_check` / - `nested_array_check`. + signals whether the renderer wraps the expression in an iteration fold + (`array_check` / `map_values_check` / their nested variants). - `Guard` -- a single discriminator gate. `Check.guards` is a tuple of `Guard`s AND-composed at render time; nested-union gating composes one `ColumnGuard` with one `ElementGuard`. @@ -19,10 +19,9 @@ from typing import TypeAlias from overture.schema.system.field_path import ( - ArrayPath, + Direct, FieldPath, - MapPath, - ScalarPath, + Iterated, StructSegment, ) @@ -72,22 +71,20 @@ def _top_level(name: str) -> str: def _path_top_column(path: FieldPath) -> str | None: - """Top-level row column for a `FieldPath`, or `None` for an empty `ScalarPath`. + """Top-level row column for a `FieldPath`, or `None` for an empty `Direct`. Collapses dotted struct navigation to its first segment -- the granularity - at which `validate_model` detects column absence. `ArrayPath.column_path` - and `MapPath.map_column` may be dotted when the iterated column is nested - inside a struct (e.g. `names.rules`); this strips to `names`. + at which `validate_model` detects column absence. `Iterated.outer_column` + may be dotted when the iterated column is nested inside a struct (e.g. + `names.rules`); this strips to `names`. """ match path: - case ScalarPath(segments=(StructSegment(name=first), *_)): + case Direct(segments=(StructSegment(name=first), *_)): return first - case ScalarPath(): + case Direct(): return None - case ArrayPath(): - return _top_level(path.column_path) - case MapPath(): - return _top_level(path.map_column) + case Iterated(): + return _top_level(path.outer_column) case _: raise TypeError(f"Unhandled FieldPath variant: {type(path).__name__}") @@ -105,11 +102,11 @@ def read_columns(self) -> frozenset[str]: """Top-level row columns this check's expression dereferences. Includes the target's outermost column, any `ColumnGuard` discriminator - (rendered as `F.col(...)`), and any descriptor gate on a `ScalarPath` + (rendered as `F.col(...)`), and any descriptor gate on a `Direct` target (rendered as `F.col("{gate}").isNotNull()`). `ElementGuard` discriminators are excluded -- they reference `el[...]`, an element-relative accessor, not a row-level column. Descriptor gates on - `ArrayPath` targets are also excluded -- they are applied element-relatively + `Iterated` targets are also excluded -- they are applied element-relatively via `element_relative_gate`. """ cols: set[str] = set() @@ -124,7 +121,7 @@ def read_columns(self) -> frozenset[str]: pass # element-relative: not a row-level read case _: raise TypeError(f"Unhandled Guard variant: {type(guard).__name__}") - if isinstance(self.target, ScalarPath): + if isinstance(self.target, Direct): for desc in self.descriptors: if desc.gate is not None: gate_col = _path_top_column(desc.gate) @@ -138,9 +135,9 @@ class ModelCheck: """A model-level validation check (cross-field constraint). `target` locates the model the constraint applies to: an empty - `ScalarPath()` for row-root constraints, or an `ArrayPath` when the - constrained model is reached by iterating one or more arrays. The - default `ScalarPath()` makes the row-root case ergonomic at + `Direct()` for row-root constraints, or an `Iterated` when the + constrained model is reached by iterating one or more arrays or maps. + The default `Direct()` makes the row-root case ergonomic at construction sites and is the common case; `Check.target` has no sensible default and is required. @@ -162,7 +159,7 @@ class ModelCheck: """ descriptor: ModelConstraintDescriptor - target: FieldPath = ScalarPath() + target: FieldPath = Direct() arm: str | None = None gate: FieldPath | None = None @@ -170,31 +167,38 @@ class ModelCheck: def read_columns(self) -> frozenset[str]: """Top-level row columns this model check's expression dereferences. - For row-root constraints (`ScalarPath` target): all `field_names` from + For row-root constraints (`Direct` target): all `field_names` from the constraint (collapsed to top-level column) and, for `RequireIf`/ `ForbidIf`, the condition field (both rendered as `F.col(...)`). - For array/map targets: only the outermost container column is a - row-level read (`array_check("col", ...)` / `map_values_check("col", - ...)`). The `field_names` and condition field are accessed as - element-relative `el[...]` / `inner[...]` accessors inside the - lambda -- not as `F.col(...)` -- so they do not contribute top-level - column reads. + For `Iterated` (array/map) targets: only the outermost container + column is a row-level read (`array_check("col", ...)` / + `map_values_check("col", ...)`). The `field_names` and condition field + are accessed as element-relative `el[...]` / `inner[...]` accessors + inside the lambda -- not as `F.col(...)` -- so they do not contribute + top-level column reads. - `gate` is excluded: for array targets it is element-relative; for scalar - targets the renderer asserts it is `None`. The `arm` field carries no - column information. + `gate` is excluded: for `Iterated` targets it is element-relative; for + `Direct` targets the renderer asserts it is `None`. The `arm` field + carries no column information. """ cols: set[str] = set() desc = self.descriptor - # Array/map targets wrap everything in array_check/map_values_check; + # Iterated targets wrap everything in array_check/map_values_check; # field references inside the lambda are element-relative, not row-level. # Only the container column itself is a top-level read. - if not isinstance(self.target, ScalarPath): + if isinstance(self.target, Iterated): container_col = _path_top_column(self.target) if container_col is not None: cols.add(container_col) return frozenset(cols) + # Struct-nested target (non-empty Direct): every field and condition + # reference qualifies to `.`, and the gate is a struct + # prefix of the target, so the sole top-level column read is the + # target's first segment. + struct_top = _path_top_column(self.target) + if struct_top is not None: + return frozenset({struct_top}) # Row-root target: field_names and condition field render as F.col(...). match desc: case ( diff --git a/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/renderer.py b/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/renderer.py index 44c5d85d2..9694eaf4f 100644 --- a/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/renderer.py +++ b/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/renderer.py @@ -4,14 +4,16 @@ import re from collections.abc import Mapping +from dataclasses import dataclass from enum import Enum from overture.schema.system.field_path import ( - ArrayPath, + ArraySegment, + Direct, FieldPath, - MapPath, + Iterated, MapProjection, - ScalarPath, + MapSegment, ) from overture.schema.system.geometric import GeometryType @@ -104,16 +106,23 @@ def _render_condition( in_array: bool = False, struct_path: tuple[str, ...] = (), var: str = "el", + column_prefix: tuple[str, ...] = (), ) -> str: """Render a parsed condition to a PySpark Column expression string. `struct_path` is the leaf the constrained model was reached at; the condition field lives beside the target field on that same model, so its reference must navigate the same leaf (e.g. `el["inner"]["subtype"]`, - not `el["subtype"]`). + not `el["subtype"]`). `column_prefix` plays the same role for a + struct-nested (non-iterated) model, qualifying the condition to + `F.col("details.subtype")`. """ ref = _render_field_ref( - parsed.field_name, in_array=in_array, struct_path=struct_path, var=var + parsed.field_name, + in_array=in_array, + struct_path=struct_path, + var=var, + column_prefix=column_prefix, ) op = "!=" if parsed.negated else "==" # A bare `== True` / `== False` -- from any boolean condition, whether @@ -134,16 +143,21 @@ def _render_field_ref( in_array: bool, struct_path: tuple[str, ...] = (), var: str = "el", + column_prefix: tuple[str, ...] = (), ) -> str: """Render a field reference as F.col("x"), el["x"], or el["struct"]["x"]. `F.col` accepts dotted names directly so the top-level form keeps - `field_name` intact. The in-array form descends a struct via - `el[...]`, which requires the dotted name to be split into segments - before applying `struct_path` and the field's own parts. + `field_name` intact. `column_prefix` names the struct segments the model + was reached through (a struct-nested model constraint at + `Direct('details')`), so its fields resolve to `F.col("details.foo")`; it + is empty for a row-root constraint. The in-array form descends a struct via + `el[...]`, which requires the dotted name to be split into segments before + applying `struct_path` and the field's own parts. """ if not in_array: - return f'F.col("{field_name}")' + qualified = ".".join((*column_prefix, field_name)) + return f'F.col("{qualified}")' parts = (*struct_path, *field_name.split(".")) return _element_accessor(var, parts) @@ -224,66 +238,153 @@ def _wrap_element_gate(body: str, var: str, gate_parts: tuple[str, ...]) -> str: return f"F.when({gate_accessor}.isNotNull(), {body})" -def _wrap_in_array_iteration( - column_path: str, - inner_struct_paths: tuple[tuple[str, ...], ...], +def _map_iter_var(projection: MapProjection) -> str: + """Lambda variable name for a map projection: `k` for keys, `v` for values.""" + return "k" if projection is MapProjection.KEY else "v" + + +@dataclass(frozen=True, slots=True) +class RenderFrame: + """One iteration frame enriched with its lambda var and runtime helper. + + Attributes + ---------- + prefix_structs + Struct segment names between the previous iterating segment (or the + start of the path) and this one. For the outermost frame this is the + column struct prefix; joined with `segment.name` it forms the frame's + `F.col(...)` column. For an inner named frame it is the descent from + the previous element; for an anonymous frame it is empty. + segment + The iterating segment (`ArraySegment` or `MapSegment`) this frame + iterates. Anonymous when the parent element is itself the container. + is_innermost + Whether this is the leaf-most iteration (the base runtime helper is + used; outer frames use the `nested_` flattening helper). + var_name + The lambda parameter name (`el` / `el2` / `inner` for arrays, + `k` / `v` for maps). + helper_name + The `column_patterns` helper this frame calls. + """ + + prefix_structs: tuple[str, ...] + segment: ArraySegment | MapSegment + is_innermost: bool + var_name: str + helper_name: str + + @property + def descent(self) -> tuple[str, ...]: + """Struct accessor from the previous element to this container. + + Empty for an anonymous frame (the parent element already IS this + container); `prefix_structs + segment.name` for a named frame. + """ + if self.segment.is_anonymous: + return () + return (*self.prefix_structs, self.segment.name) + + @property + def column(self) -> str: + """Dotted `F.col(...)` name for the outermost frame.""" + return ".".join((*self.prefix_structs, self.segment.name)) + + +def _render_frames(target: Iterated) -> tuple[RenderFrame, ...]: + """Enrich each iteration of *target* with its lambda var and runtime helper. + + One `RenderFrame` per iteration -- every iterating segment, named and + anonymous, since each is its own `array_check` / `map_*_check` call. + Built once and consumed by the fold, `_pattern_imports_for`, and the + model-constraint context so var and helper names never drift (a hazard + with mixed nesting where two map frames both want `v`). + + Array frames use `el` / `el2` / `inner` (indexed by overall iteration + position) with `array_check` (innermost) or `nested_array_check`; map + frames use `k` / `v` with `map_{keys,values}_check` (innermost) or the + `nested_map_*` flattening variant. + """ + raw: list[tuple[tuple[str, ...], ArraySegment | MapSegment]] = [] + prefix: list[str] = [] + for seg in target.segments: + if isinstance(seg, (ArraySegment, MapSegment)): + raw.append((tuple(prefix), seg)) + prefix = [] + else: + prefix.append(seg.name) + total = len(raw) + frames: list[RenderFrame] = [] + for i, (prefix_structs, seg) in enumerate(raw): + is_innermost = i == total - 1 + if isinstance(seg, ArraySegment): + var = _iter_var_name(i, total) + helper = "array_check" if is_innermost else "nested_array_check" + else: + var = _map_iter_var(seg.projection) + helper = map_runtime_helper(seg.projection, flatten=not is_innermost) + frames.append( + RenderFrame( + prefix_structs=prefix_structs, + segment=seg, + is_innermost=is_innermost, + var_name=var, + helper_name=helper, + ) + ) + return tuple(frames) + + +def _wrap_in_iteration( + frames: tuple[RenderFrame, ...], body: str, *, gate_parts: tuple[str, ...] = (), ) -> str: - """Wrap `body` in nested array_check / nested_array_check frames. - - One frame per iteration: `column_path` names the outermost array - column, `inner_struct_paths` gives the struct accessor from each - iteration's element to the next array (its length plus one is the - iteration count). `body` is the innermost lambda body. `gate_parts`, - when set, wraps the outermost lambda body in a nullable-parent - element gate. - - The recursion descends one frame per call, carrying the frame index - and its lambda variable; the innermost frame is `array_check`, every - outer frame `nested_array_check`. + """Fold *frames* outermost->innermost into nested iteration helper calls. + + The outermost frame targets its `column` (an `F.col` string); each inner + frame targets an element accessor built from the outer frame's var and the + inner frame's `descent`. The innermost frame carries `body`. `gate_parts`, + when set, wraps the OUTERMOST frame's body in a nullable-parent element + gate (`element_relative_gate` is relative to the outer array element); + element guards are applied to `body` at the innermost var by the caller, + the two wrap points staying distinct. """ - total = 1 + len(inner_struct_paths) - def frame(idx: int, accessor: str, var: str) -> str: - if idx == total - 1: + def build(i: int, accessor: str) -> str: + frame = frames[i] + if frame.is_innermost: inner = body - fn = "array_check" else: - child_var = _iter_var_name(idx + 1, total) - child_accessor = _element_accessor(var, inner_struct_paths[idx]) - inner = frame(idx + 1, child_accessor, child_var) - fn = "nested_array_check" - if idx == 0 and gate_parts: - inner = _wrap_element_gate(inner, var, gate_parts) - return f"{fn}({accessor}, lambda {var}: {inner})" + child = frames[i + 1] + inner = build(i + 1, _element_accessor(frame.var_name, child.descent)) + if i == 0 and gate_parts: + inner = _wrap_element_gate(inner, frame.var_name, gate_parts) + return f"{frame.helper_name}({accessor}, lambda {frame.var_name}: {inner})" - return frame(0, f'"{column_path}"', "el") + return build(0, f'"{frames[0].column}"') -def _render_array_check_expr( - target: ArrayPath, +def _render_iterated_check_expr( + target: Iterated, desc: ExpressionDescriptor, *, element_guards: tuple[ElementGuard, ...] = (), gate_parts: tuple[str, ...] = (), ) -> str: - """Render an ArrayPath target to an array_check / nested_array_check expression. + """Render an `Iterated` target to a nested iteration-fold expression. Element guards are applied at the innermost iteration variable. This assumes each guard's discriminator lives on the same struct level as - the leaf accessor -- which is true today because `ElementGuard`s only - arise from a union variant whose discriminator field is the - immediately enclosing array element. A future case where a check is - reached through further iteration *inside* a discriminated union - element would need per-guard depth info to apply the guard at the - correct frame. + the leaf accessor -- true today because `ElementGuard`s only arise from + a union variant whose discriminator field is the immediately enclosing + array element. A future case where a check is reached through further + iteration *inside* a discriminated union element would need per-guard + depth info to apply the guard at the correct frame. """ - inner_struct_paths = target.iter_struct_paths - iteration_count = 1 + len(inner_struct_paths) - - innermost_var = _iter_var_name(iteration_count - 1, iteration_count) + frames = _render_frames(target) + innermost_var = frames[-1].var_name leaf_accessor = _element_accessor(innermost_var, target.leaf) body = _render_expr_call(desc, leaf_accessor) @@ -292,40 +393,7 @@ def _render_array_check_expr( body, guard.values, guard.discriminator, in_array=True, var=innermost_var ) - return _wrap_in_array_iteration( - target.column_path, inner_struct_paths, body, gate_parts=gate_parts - ) - - -def _map_iter_var(projection: MapProjection) -> str: - """Lambda variable name for a map projection: `k` for keys, `v` for values.""" - return "k" if projection is MapProjection.KEY else "v" - - -def _wrap_in_map_iteration(target: MapPath, body: str) -> str: - """Wrap `body` in a map_keys_check / map_values_check projection lambda. - - The map helper projects the map (`F.map_keys` / `F.map_values`) and - applies the lambda to each projected key or value, the map analogue of - `_wrap_in_array_iteration`. `body` references the projected element via - the same `_map_iter_var(target.projection)` name this builds the lambda - parameter from. - """ - helper = map_runtime_helper(target.projection) - var = _map_iter_var(target.projection) - return f'{helper}("{target.map_column}", lambda {var}: {body})' - - -def _render_map_check_expr(target: MapPath, desc: ExpressionDescriptor) -> str: - """Render a MapPath target to a map_keys_check / map_values_check expression. - - A non-empty `target.leaf` navigates into a `dict[K, Model]` value struct - (`v["field"]`), mirroring an array element's leaf accessor; an empty leaf - applies the check to the projected scalar itself. - """ - var = _map_iter_var(target.projection) - body = _render_expr_call(desc, _element_accessor(var, target.leaf)) - return _wrap_in_map_iteration(target, body) + return _wrap_in_iteration(frames, body, gate_parts=gate_parts) def _render_variant_expr( @@ -354,25 +422,20 @@ def _render_column_gate(expr: str, gate: FieldPath) -> str: def _model_check_func_name(check: ModelCheck, idx: int) -> str: """Build the private function name for a model-constraint check. - Array and map targets prefix the column path -- using the full encoded - `FieldPath` when the check is reached via inner iteration or leaf struct - navigation, otherwise the outer column name alone -- so collisions - across nested contexts get distinct identifiers. Row-root targets emit - `_{fn}_{idx}_check`. + An `Iterated` (array/map) target prefixes the column path -- using the + full encoded `FieldPath` when the check is reached via inner iteration or + leaf struct navigation, otherwise the outer column name alone -- so + collisions across nested contexts get distinct identifiers. Row-root + (`Direct`) targets emit `_{fn}_{idx}_check`. """ fn = model_constraint_function(check.descriptor) - match check.target: - case ArrayPath() as target: - has_nested_path = bool(target.iter_struct_paths) or bool(target.leaf) - prefix_source = str(target) if has_nested_path else target.column_path - prefix = sanitize_field_name(prefix_source) - return f"_{prefix}_{fn}_{idx}_check" - case MapPath() as target: - prefix_source = str(target) if target.leaf else target.map_column - prefix = sanitize_field_name(prefix_source) - return f"_{prefix}_{fn}_{idx}_check" - case _: - return f"_{fn}_{idx}_check" + target = check.target + if isinstance(target, Iterated): + has_nested_path = bool(target.iter_struct_paths) or bool(target.leaf) + prefix_source = str(target) if has_nested_path else target.outer_column + prefix = sanitize_field_name(prefix_source) + return f"_{prefix}_{fn}_{idx}_check" + return f"_{fn}_{idx}_check" def _check_shape_token(target: FieldPath) -> str: @@ -380,11 +443,11 @@ def _check_shape_token(target: FieldPath) -> str: Mirrors the member names of `overture.schema.pyspark.check.CheckShape`; the check-function template prefixes `CheckShape.` to the result. An - `ArrayPath` or `MapPath` target renders to an `array` - expression (the map helper iterates the projected keys/values), every - other path to a nullable string. + `Iterated` target renders to an `array` expression (array + iteration, or a map helper iterating the projected keys/values), a + `Direct` target to a nullable string. """ - return "ARRAY" if isinstance(target, (ArrayPath, MapPath)) else "SCALAR" + return "ARRAY" if isinstance(target, Iterated) else "SCALAR" def _render_check_expr(check: Check, descriptor_idx: int) -> str: @@ -394,34 +457,32 @@ def _render_check_expr(check: Check, descriptor_idx: int) -> str: element_guards = tuple(g for g in check.guards if isinstance(g, ElementGuard)) match check.target: - case ScalarPath(): + case Direct(): expr = _render_expr_call(desc, f'F.col("{check.target}")') if desc.gate: expr = _render_column_gate(expr, desc.gate) - case ArrayPath(): + case Iterated(): gate_parts: tuple[str, ...] = () if desc.gate is not None: # check_builder zeros the nullable gate when descending into - # a list (see `_recurse_into_model`), so a gate paired with - # an ArrayPath target should never occur today. If it does, - # the column-level fallback below would silently hide a - # codegen bug -- raise instead. + # any iterated container (see `_recurse_into_model`), so a + # gate paired with an Iterated target should never occur + # today. If it does, the column-level fallback below would + # silently hide a codegen bug -- raise instead. element_relative = check.target.element_relative_gate(desc.gate) if element_relative is None: raise AssertionError( - f"ArrayPath target with column-level gate is not " + f"Iterated target with column-level gate is not " f"produced by check_builder (gate={desc.gate!r}, " f"target={check.target!r})" ) gate_parts = element_relative - expr = _render_array_check_expr( + expr = _render_iterated_check_expr( check.target, desc, element_guards=element_guards, gate_parts=gate_parts, ) - case MapPath(): - expr = _render_map_check_expr(check.target, desc) case _: raise TypeError( f"Unhandled FieldPath variant: {type(check.target).__name__}" @@ -473,25 +534,42 @@ def _render_model_constraint_function_context(row: ModelCheckRow) -> dict[str, o check = row.check desc = check.descriptor target = check.target - match target: - case ArrayPath(): - in_array = True - var = "inner" if target.iter_struct_paths else "el" - struct_path: tuple[str, ...] = target.leaf - case MapPath(): - # The map's values are iterated like an array element, so field - # references use the element accessor (`v["foo"]`) under the - # projected variable. - in_array = True - var = _map_iter_var(target.projection) - struct_path = target.leaf - case _: - in_array = False - var, struct_path = "el", () + # Build the render frames once; both the field-reference context (var / + # struct_path) and the iteration wrap below read from them so nothing drifts. + frames: tuple[RenderFrame, ...] = () + column_prefix: tuple[str, ...] = () + if isinstance(target, Iterated): + # The innermost element (array element or projected map value) is + # iterated, so field references use the element accessor + # (`inner["foo"]`, `v["foo"]`) under the innermost lambda variable. + frames = _render_frames(target) + in_array = True + var = frames[-1].var_name + struct_path: tuple[str, ...] = target.leaf + else: + # A struct-nested model constraint (`Direct` with segments) qualifies + # every field reference with the struct prefix (`F.col("details.foo")`); + # a row-root constraint (empty `Direct`) leaves the prefix empty. + in_array = False + var, struct_path = "el", () + column_prefix = tuple(s.name for s in target.segments) def _field_ref(field_name: str) -> str: return _render_field_ref( - field_name, in_array=in_array, struct_path=struct_path, var=var + field_name, + in_array=in_array, + struct_path=struct_path, + var=var, + column_prefix=column_prefix, + ) + + def _condition_ref(parsed: FieldEq) -> str: + return _render_condition( + parsed, + in_array=in_array, + struct_path=struct_path, + var=var, + column_prefix=column_prefix, ) fn = model_constraint_function(desc) @@ -508,23 +586,14 @@ def _cols_and_names(field_names: tuple[str, ...]) -> tuple[str, str]: case RequireAnyTrue(): parsed_conditions = [require_field_eq(c) for c in desc.conditions] conds_list = ( - "[" - + ", ".join( - _render_condition( - p, in_array=in_array, struct_path=struct_path, var=var - ) - for p in parsed_conditions - ) - + "]" + "[" + ", ".join(_condition_ref(p) for p in parsed_conditions) + "]" ) names_list = py_literal([p.field_name for p in parsed_conditions]) inner_expr = f"{fn}({conds_list}, {names_list})" case RequireIf() | ForbidIf(): target_name = desc.field_names[0] parsed = require_field_eq(desc.condition) - condition_expr = _render_condition( - parsed, in_array=in_array, struct_path=struct_path, var=var - ) + condition_expr = _condition_ref(parsed) condition_desc = _render_condition_desc(parsed) target_ref = _field_ref(target_name) inner_expr = ( @@ -536,8 +605,12 @@ def _cols_and_names(field_names: tuple[str, ...]) -> tuple[str, str]: case _: raise TypeError(f"Unhandled model constraint descriptor: {desc!r}") - if isinstance(target, ArrayPath): + if isinstance(target, Iterated): if check.gate is not None: + # A gate reaches only an array-first target: check_builder zeros + # the gate for any iterated container, so a map-reached model + # carries none, and `element_relative_gate` asserts the array-first + # precondition. The wrap assumes a single array level. assert not target.iter_struct_paths, ( f"gated ModelCheck with a nested-array target ({target!r}) is unsupported; " f"the element-gate wrap assumes a single array level" @@ -545,28 +618,26 @@ def _cols_and_names(field_names: tuple[str, ...]) -> tuple[str, str]: element_relative = target.element_relative_gate(check.gate) assert element_relative is not None, ( f"ModelCheck gate={check.gate!r} is not reachable as an element-level " - f"accessor on target={target!r}; gates on ModelChecks must be ArrayPaths " + f"accessor on target={target!r}; gates on ModelChecks must be Iterated " f"entering the same outer array as the target" ) inner_expr = _wrap_element_gate(inner_expr, var, element_relative) - expr = _wrap_in_array_iteration( - target.column_path, target.iter_struct_paths, inner_expr + expr = _wrap_in_iteration(frames, inner_expr) + elif check.gate is not None: + # A struct-nested model reached through an optional ancestor: skip the + # constraint when that ancestor is null (accessing a field of a null + # struct yields null, which would otherwise trip the constraint on an + # absent model). The gate is a struct prefix of the target, so it reads + # the target's top-level column and `_render_column_gate` renders + # `F.when(F.col("details").isNotNull(), ...)`. A gate on an empty + # (row-root) `Direct` target is meaningless -- check_builder never emits + # one -- so guard it rather than render a nonsensical `F.when`. + assert isinstance(target, Direct) and target.segments, ( + f"ModelCheck gate={check.gate!r} on a row-root Direct target={target!r}; " + f"a gate only pairs with a struct-nested or iterated model" ) - elif isinstance(target, MapPath): - # A `dict[K, Model]` value-model constraint wraps in map_values_check - # (or map_keys_check), iterating the projected values like an array. - # check_builder zeros the gate for iterated containers, so no gate - # reaches here. - assert check.gate is None, ( - f"ModelCheck gate={check.gate!r} paired with MapPath target={target!r}; " - f"map iteration handles value nullability, so a gate is unexpected" - ) - expr = _wrap_in_map_iteration(target, inner_expr) + expr = _render_column_gate(inner_expr, check.gate) else: - assert check.gate is None, ( - f"ModelCheck gate={check.gate!r} paired with non-ArrayPath target={target!r}; " - f"a gate only makes sense when the constrained model is inside an array" - ) expr = inner_expr return _check_function_context( @@ -613,17 +684,15 @@ def _needs_geometry_type_import(field_checks: list[Check]) -> bool: def _pattern_imports_for(target: FieldPath) -> set[str]: - """Column-pattern helpers needed to iterate `target`.""" - match target: - case ArrayPath(): - names = {"array_check"} - if target.iter_struct_paths: - names.add("nested_array_check") - return names - case MapPath(): - return {map_runtime_helper(target.projection)} - case _: - return set() + """Column-pattern helpers needed to iterate `target`. + + Reads the helper names off `_render_frames` -- the single source of the + frame->helper mapping the fold also consumes -- so the imports never drift + from the emitted calls. A `Direct` target needs none. + """ + if isinstance(target, Iterated): + return {frame.helper_name for frame in _render_frames(target)} + return set() def _collect_column_pattern_imports( diff --git a/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/test_data/scaffold.py b/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/test_data/scaffold.py index 4d48dde05..a911fb4b7 100644 --- a/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/test_data/scaffold.py +++ b/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/test_data/scaffold.py @@ -13,10 +13,12 @@ from typing import Any from overture.schema.system.field_path import ( - ArrayPath, ArraySegment, FieldPath, FieldSegment, + Iterated, + MapSegment, + terminal_run_start, ) from ...extraction.field_walk import ( @@ -71,6 +73,19 @@ class _ElementDiscriminator: depth: int +def _is_anonymous_iter(seg: FieldSegment) -> bool: + """True when *seg* iterates a container nested directly inside another. + + In a run of nested containers the first takes the field's name; each + further level is *anonymous*, because no field name introduces it -- the + parent element is itself the next container. For `grid: list[list[int]]` + the path `grid[][]` is a named `ArraySegment("grid")` followed by an + anonymous `ArraySegment("")`; this returns False for the first and True + for the second, the "extra" iteration past the named `grid`. + """ + return isinstance(seg, (ArraySegment, MapSegment)) and seg.is_anonymous + + def _find_field_spec(fields: list[FieldSpec], name: str) -> FieldSpec | None: """Find a FieldSpec by name in a list.""" for f in fields: @@ -82,19 +97,32 @@ def _find_field_spec(fields: list[FieldSpec], name: str) -> FieldSpec | None: def leaf_list_depth(field_path: FieldPath, spec: ModelSpec) -> int: """Return the unaccounted-for list depth of the leaf field. - Walks the spec's field tree along *field_path* and returns the - leaf's `list_depth(shape)` minus any `iter_count` on the terminal - path segment. Paths whose terminal segment is itself an array - target the array's elements, so the mutation already operates one - level deep. Returns 0 when *field_path* is empty or when any - segment fails to resolve against *spec* (e.g. union arms that - don't share the path's intermediate fields). + Walks the spec's field tree along *field_path* and returns the leaf's + `list_depth(shape)` minus the path's own trailing iteration depth at + the leaf. The leaf is the last *named* segment -- any anonymous + `ArraySegment`s after it are further list-nesting of that same field, + not a lookup of their own, and are skipped both when descending the + field tree and when counting how much depth the path already covers. + Paths whose terminal segment is itself an array target the array's + elements, so the mutation already operates one level deep. Returns 0 + when *field_path* is empty or when any segment fails to resolve + against *spec* (e.g. union arms that don't share the path's + intermediate fields). """ segments = field_path.segments if not segments: return 0 + + leaf_index = terminal_run_start(segments) + leaf_seg = segments[leaf_index] + terminal_iter = ( + (len(segments) - leaf_index) if isinstance(leaf_seg, ArraySegment) else 0 + ) + fields = list(spec.fields) - for seg in segments[:-1]: + for seg in segments[:leaf_index]: + if isinstance(seg, ArraySegment) and seg.is_anonymous: + continue field = _find_field_spec(fields, seg.name) if field is None: return 0 @@ -102,11 +130,10 @@ def leaf_list_depth(field_path: FieldPath, spec: ModelSpec) -> int: if model_ref is None: return 0 fields = model_ref.model.fields - leaf_seg = segments[-1] + leaf = _find_field_spec(fields, leaf_seg.name) if leaf is None: return 0 - terminal_iter = leaf_seg.iter_count if isinstance(leaf_seg, ArraySegment) else 0 return max(0, list_depth(leaf.shape) - terminal_iter) @@ -151,7 +178,7 @@ def _walk_to_target( Accepts any `FieldSegment`: struct steps recurse, an `ArraySegment` wraps its inner value in lists, and a trailing `MapSegment` resolves via `value_for_field` (which populates the map with a valid entry), - so a `MapPath` target scaffolds the same way as a struct terminal. + so a map-projection target scaffolds the same way as a struct terminal. `leaf_value`, when set, replaces the synthesized value at the terminal field -- used to seed a specific valid value (e.g. a literal alternative) @@ -174,6 +201,21 @@ def _walk_to_target( f"(available: {sorted(f.name for f in fields)})" ) + # Anonymous iterating segments immediately after `seg` are further + # container-nesting of THIS SAME field (`list[list[...]]`, + # `dict[K, dict[K2, ...]]`, no intervening field name), not separate + # lookups -- `generate_base_row`/`value_for_field` resolve straight + # through every list and map layer, so peeling the run here (rather than + # recursing anonymous-segment-by-segment) keeps the base-row merge below + # anchored on the field `seg` actually names. `extra_iter` carries only the + # ARRAY levels to the wrap step; anonymous map levels need no manual + # wrapping, since `value_for_field` nests the map itself. + extra_iter = 0 + while remaining and _is_anonymous_iter(remaining[0]): + if isinstance(remaining[0], ArraySegment): + extra_iter += 1 + remaining = remaining[1:] + inner: Any if remaining: discriminator_value = ( @@ -192,7 +234,7 @@ def _walk_to_target( child_spec.fields, spec_name, discriminator=discriminator, - current_depth=current_depth + 1, + current_depth=current_depth + 1 + extra_iter, leaf_value=leaf_value, ) inner = {**generate_base_row(child_spec), **recursed} @@ -213,15 +255,15 @@ def _walk_to_target( if isinstance(seg, ArraySegment): if not remaining and has_array_layer(field_spec.shape): return {seg.name: inner} - # A single-level array (iter_count == 1) gets a constraint-valid list; - # nested `list[list[...]]` levels (iter_count > 1) carry no min_length>1 + # A single-level array (extra_iter == 0) gets a constraint-valid list; + # nested `list[list[...]]` levels (extra_iter > 0) carry no min_length>1 # or uniqueness constraint in any current schema, so minimal nesting # suffices. Add per-level constraint handling here if one ever does -- # the row would otherwise be short on the unmutated `::valid` row. - if seg.iter_count == 1: + if extra_iter == 0: return {seg.name: _array_with_target(inner, field_spec, spec_name)} wrapped: Any = inner - for _ in range(seg.iter_count): + for _ in range(1 + extra_iter): wrapped = [wrapped] return {seg.name: wrapped} if remaining and has_array_layer(field_spec.shape): @@ -318,24 +360,31 @@ def generate_model_scaffold(check: ModelCheck, spec: ModelSpec) -> dict[str, Any Two target shapes need no scaffold and return `{}`: - - a `ScalarPath` target -- a top-level model constraint, whose fields - live at the row root; - - a `MapPath` target -- a `dict[K, Model]` value-model constraint. The - mutation (`map_path=`) owns map navigation: it corrupts the base row's - single map entry in place, or stubs one when the map is absent. Unlike - an array, a dict scaffold can't replace a base-row map entry under - `deep_merge`'s recursive dict merge, so there is nothing to add here. - - An `ArrayPath` walks the path with `_walk_to_target`: every model on the - way -- including the constrained model at the leaf -- is built as a valid - base row, so the constraint under test (e.g. a scope's `require_any_of`) - is satisfied on the unmutated `::valid` row and the only violation is the - one the mutation introduces. + - a `Direct` target -- a top-level model constraint, whose fields live + at the row root; + - a map-first `Iterated` target -- a `dict[K, Model]` value-model + constraint. The mutation (`map_path=`) owns map navigation: it corrupts + the base row's single map entry in place, or stubs one when the map is + absent. Unlike an array, a dict scaffold can't replace a base-row map + entry under `deep_merge`'s recursive dict merge, so there is nothing to + add here. + + An array-first `Iterated` target walks the path with `_walk_to_target`: + every model on the way -- including the constrained model at the leaf -- + is built as a valid base row, so the constraint under test (e.g. a + scope's `require_any_of`) is satisfied on the unmutated `::valid` row and + the only violation is the one the mutation introduces. + + An `Iterated` target is array-first exactly when its outermost frame is + an `ArraySegment` (guaranteed named, so it heads `iter_frames`); a + map-first target's mutation owns navigation, matching the former + map-path case. """ - match check.target: - case ArrayPath() as target: - return _walk_to_target( - target.segments, spec.fields, spec.name, discriminator=None - ) - case _: - return {} + target = check.target + if isinstance(target, Iterated) and isinstance( + target.iter_frames[0][1], ArraySegment + ): + return _walk_to_target( + target.segments, spec.fields, spec.name, discriminator=None + ) + return {} diff --git a/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/test_renderer.py b/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/test_renderer.py index 6d245a046..a3e94bfdf 100644 --- a/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/test_renderer.py +++ b/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/test_renderer.py @@ -6,7 +6,14 @@ from typing_extensions import assert_never -from overture.schema.system.field_path import ArrayPath, MapPath, MapProjection +from overture.schema.system.field_path import ( + ArraySegment, + Direct, + FieldPath, + Iterated, + MapProjection, + MapSegment, +) from ..extraction.field import FieldShape, Primitive from ..extraction.field_walk import has_array_layer, terminal_of @@ -58,8 +65,7 @@ def _check_belongs_to_arm(check: Check, arm: str) -> bool: irrelevant to arm filtering. A check belongs to *arm* when every `ColumnGuard` admits it (guards are AND-composed). """ - column_guards = [g for g in check.guards if isinstance(g, ColumnGuard)] - return all(arm in g.values for g in column_guards) + return all(arm in g.values for g in check.guards if isinstance(g, ColumnGuard)) def _model_check_belongs_to_arm(check: ModelCheck, arm: str) -> bool: @@ -72,6 +78,63 @@ def _model_check_belongs_to_arm(check: ModelCheck, arm: str) -> bool: return check.arm is None or check.arm == arm +def _innermost_iter_segment(target: Iterated) -> ArraySegment | MapSegment: + """Return the innermost (leaf-most) named iterating segment of *target*.""" + return target.iter_frames[-1][1] + + +def _first_iter_segment(target: Iterated) -> ArraySegment | MapSegment: + """Return the outermost (first) named iterating segment of *target*.""" + return target.iter_frames[0][1] + + +def _is_map_target(target: FieldPath) -> bool: + """True when *target* reaches its value map-first: no array precedes the map. + + Reads the FIRST iterating frame, matching `check_builder` + (`_model_constraint_target` and `generate_model_scaffold` both key off + the outermost frame). A bare or struct-prefixed map projection + (`names.common{key}`, `sources.license_priority{value}`) is a map target, + corrupted in place. A map reached only after array iteration + (`items[].tags{value}`) is array-first, not a map target: its mutation + descends the array via `set_at_path`'s map grammar. Reading the innermost + frame instead would misroute the array-first case to a top-level map + mutation on the array column. + """ + return isinstance(target, Iterated) and isinstance( + _first_iter_segment(target), MapSegment + ) + + +def _is_sole_map_projection(target: Iterated) -> bool: + """True when *target* is a bare map projection with nothing trailing. + + `names.common{key}`, `sources.license_priority{value}` -- one map frame, + no struct leaf, no further iteration. These corrupt the map's single + entry in place via `mutate_map_key` / `mutate_map_value`. + """ + return not target.leaf and not target.iter_struct_paths + + +def _map_trailing_iteration_only(target: Iterated) -> bool: + """True when a map projection is followed only by anonymous iteration. + + `subs{value}[]` (dict[K, list[X]]) and `subs{value}{value}` + (dict[K, dict[K2, X]]) descend the map value into a container and reach + the constrained scalar through anonymous `[]` / `{value}` peels alone -- + no named struct navigation, no struct leaf. `set_at_path` with the full + path peels each trailing container, so the mutation writes the scalar at + the located slot. A struct leaf (`subs{value}.label`) or a named further + container (`subs{value}.items[]`) is excluded: it needs navigation + `set_at_path`'s map-first routing does not cover here. + """ + return ( + not target.leaf + and bool(target.iter_struct_paths) + and all(not prefix for prefix in target.iter_struct_paths) + ) + + def render_test_module( model_name: str, field_checks: list[Check], @@ -171,21 +234,27 @@ def _field_mutate_expr( ) -> _MutateExpr: """Render the `mutate=` expression for one field-check descriptor. - A `MapPath` target corrupts the map's single valid entry via + A sole map projection corrupts the map's single valid entry via `mutate_map_key` / `mutate_map_value`; `check_struct_unique` calls - `mutate_unique_items` at the target path; every other descriptor - injects a constraint-violating literal via `set_at_path`. + `mutate_unique_items` at the target path; every other descriptor -- + including a map value that is itself an iterated container + (`subs{value}[]`, `subs{value}{value}`) -- injects a constraint-violating + literal via `set_at_path`, whose path grammar peels each trailing + container to the constrained scalar. """ - if isinstance(check.target, MapPath): - helper = ( - "mutate_map_key" - if check.target.projection is MapProjection.KEY - else "mutate_map_value" - ) - col_repr = py_literal(check.target.map_column) - iv_repr = py_literal(invalid_value(desc)) - return _MutateExpr(f"lambda row: {helper}(row, {col_repr}, {iv_repr})", helper) - target_repr = py_literal(str(check.target)) + target = check.target + if _is_map_target(target): + assert isinstance(target, Iterated) + if _is_sole_map_projection(target): + return _map_field_mutate_expr(target, desc) + if not _map_trailing_iteration_only(target): + raise NotImplementedError( + f"map-first field check {target!r} descends a struct leaf or a " + f"named container after the map; no conformance mutation covers it" + ) + # Iteration-only trailing: fall through to set_at_path with the full + # path, which descends the map value and peels the trailing containers. + target_repr = py_literal(str(target)) if desc.function == "check_struct_unique": return _MutateExpr( f"lambda row: mutate_unique_items(row, {target_repr})", @@ -195,6 +264,24 @@ def _field_mutate_expr( return _MutateExpr(f"set_at_path({target_repr}, {py_literal(iv_val)})", None) +def _map_field_mutate_expr(target: Iterated, desc: ExpressionDescriptor) -> _MutateExpr: + """Render the `mutate=` for a sole map-projection field check. + + `mutate_map_key` / `mutate_map_value` corrupt the map's single valid entry + in place (`names.common{key}`, `sources.license_priority{value}`). The + caller guarantees a sole projection (`_is_sole_map_projection`); a map + value that iterates further routes to `set_at_path` instead. + """ + seg = _first_iter_segment(target) + assert isinstance(seg, MapSegment) + helper = ( + "mutate_map_key" if seg.projection is MapProjection.KEY else "mutate_map_value" + ) + col_repr = py_literal(target.outer_column) + iv_repr = py_literal(invalid_value(desc)) + return _MutateExpr(f"lambda row: {helper}(row, {col_repr}, {iv_repr})", helper) + + def _render_field_check_scenarios( model_name: str, field_checks: list[Check], @@ -254,14 +341,19 @@ def _render_field_check_scenarios( def _checks_array_element(check: Check) -> bool: - """True when the check fires on each element of an `ArrayPath` directly. + """True when the check fires on each element of an array target directly. The check target ends at the array (`leaf=()`), so the mutation replaces an array element rather than a struct field on one. For these checks, a `None` invalid value still needs list wrapping; for nested struct fields, `None` already sits at the right level. """ - return isinstance(check.target, ArrayPath) and not check.target.leaf + target = check.target + return ( + isinstance(target, Iterated) + and isinstance(_innermost_iter_segment(target), ArraySegment) + and not target.leaf + ) def _wrap_for_list_leaf( @@ -336,6 +428,25 @@ def _render_model_scenarios( return entries, used_mutation_fns +def _reject_non_row_root_target(target: FieldPath, mutation_fn: str) -> None: + """Raise unless *target* is the row root (empty `Direct`). + + `mutate_radio_group` and `mutate_require_any_true` take no navigation + kwarg, so they only reach fields at the row root. An `Iterated` target + (array/map) or a struct-nested `Direct` target (a model reached through a + plain struct field) would need the constraint's fields nulled/set at a + nested node the mutation can't reach, so it raises rather than silently + corrupting top-level columns. No live schema declares `radio_group` or + `require_any_true` on a nested submodel; supporting one means teaching + these mutations an `element_path` descent. + """ + if isinstance(target, Iterated) or (isinstance(target, Direct) and target.segments): + raise ValueError( + f"{mutation_fn} does not support a nested target " + f"(target={target!r}); it reaches only row-root fields" + ) + + def _render_mutation_call( mutation_fn: str, desc: ModelConstraintDescriptor, @@ -349,17 +460,14 @@ def _render_mutation_call( # Carries `conditions`, not `field_names`: the mutation disables # every condition via a per-field `{field: value}` dict rather than # the shared field-name list the other descriptors pass. + _reject_non_row_root_target(check.target, "mutate_require_any_true") return _render_require_any_true_mutation_call(mutation_fn, desc) case RequireIf() | ForbidIf(): return _render_conditional_mutation_call( mutation_fn, desc, check, fields_repr ) case RadioGroup(): - if isinstance(check.target, (ArrayPath, MapPath)): - raise ValueError( - "mutate_radio_group does not accept array_path/map_path " - f"(target={check.target!r})" - ) + _reject_non_row_root_target(check.target, "mutate_radio_group") return f"{mutation_fn}(row, {fields_repr})" case RequireAnyOf() | MinFieldsSet(): parts = _iter_kwargs_leaf(check, mutation_fn) @@ -438,22 +546,65 @@ def _render_fill_values(desc: ForbidIf) -> str | None: return "{" + ", ".join(items) + "}" -def _map_kwargs(target: MapPath, mutation_fn: str, *, allow_leaf: bool) -> list[str]: +def _composite_element_path_kwargs(target: Iterated) -> list[str]: + """The `element_path=` kwarg carrying *target*'s full mixed map/array descent. + + No scalar `array_path` / `map_path` expresses a container-after-container + boundary (a map value that is a list, or a map nested under array + iteration). The mutation helpers walk the full path generically, so emit + it verbatim. Every map frame must be a VALUE projection -- a model can't sit + on a map key -- so a KEY frame raises here rather than emitting a path the + walker would reject only at runtime (matching `_map_kwargs`'s codegen-time + guard for the map-first case). + """ + for _prefix, seg in target.iter_frames: + if isinstance(seg, MapSegment) and seg.projection is not MapProjection.VALUE: + raise ValueError( + f"element_path cannot target a map key (target={target!r}); a " + "model-level constraint on a map key is not representable as a row" + ) + return [f'element_path="{target}"'] + + +def _array_first_map_kwargs(target: Iterated) -> list[str] | None: + """Composite kwargs when a map value sits under array iteration, else None. + + Called on the array-first branch (the first frame is an `ArraySegment`). + A `MapSegment` anywhere in the frames means the target reaches a + `dict[K, Model]` value nested under array iteration (e.g. + `items[].configs{value}`); no scalar array/inner kwarg expresses the map + boundary, so emit the composite descent path. A pure-array target has no + map frame and keeps its existing scalar kwargs (returns None). + """ + if any(isinstance(seg, MapSegment) for _prefix, seg in target.iter_frames): + return _composite_element_path_kwargs(target) + return None + + +def _map_kwargs(target: Iterated, mutation_fn: str, *, allow_leaf: bool) -> list[str]: """Mutation kwargs for a `dict[K, Model]` value-model constraint. Emits `map_path=...` (the map column) and, when `allow_leaf`, an optional single-segment `struct_path=...` for a sub-model reached through one struct field inside the value model -- the map analogue of `_iter_kwargs_leaf`'s array `struct_path`. A KEY projection is - unrepresentable (a model can't be a dict key) and raises; a multi-segment - leaf, or any leaf when `allow_leaf` is False, raises too. + unrepresentable (a model can't be a dict key) and raises. A map value that + is itself iterated (`dict[K, list[Model]]`, target `subs{value}[]`) folds + its trailing container into this same named frame, so `iter_struct_paths` + is non-empty; the map value is a container, not the model, so emit the + composite descent path the mutation walks instead of `map_path=...`. A + multi-segment leaf, or any leaf when `allow_leaf` is False, raises. """ - if target.projection is not MapProjection.VALUE: + seg = _first_iter_segment(target) + assert isinstance(seg, MapSegment) + if seg.projection is not MapProjection.VALUE: raise ValueError( f"{mutation_fn} cannot target a map key (target={target!r}); a " "model-level constraint on a map key is not representable as a row" ) - kwargs = [f'map_path="{target.map_column}"'] + if target.iter_struct_paths: + return _composite_element_path_kwargs(target) + kwargs = [f'map_path="{target.outer_column}"'] leaf = target.leaf if leaf: if not allow_leaf: @@ -469,72 +620,91 @@ def _map_kwargs(target: MapPath, mutation_fn: str, *, allow_leaf: bool) -> list[ return kwargs +def _struct_nested_kwargs(target: Direct) -> list[str]: + """Container kwargs for a model constraint on a struct-nested submodel. + + A row-root constraint (empty `Direct`) needs no navigation and yields no + kwargs. A model reached through one or more plain struct fields yields + `element_path=...` -- the pure-struct descent (`_descend_to_targets` in + `mutations.py`) that scaffolds each struct on the way and applies the + mutation to the constrained model, mirroring how iterated targets pass + `array_path` / `map_path`. + """ + return [f'element_path="{target}"'] if target.segments else [] + + def _iter_kwargs_leaf(check: ModelCheck, mutation_fn: str) -> list[str]: """Container kwargs for mutations accepting `struct_path` (a trailing leaf). - For an `ArrayPath`, yields `array_path=...` and optionally + For an array target, yields `array_path=...` and optionally `struct_path=...`; inner array iteration is rejected -- these mutations - consume only the outermost array level. For a `MapPath` (a + consume only the outermost array level. For a map target (a `dict[K, Model]` value-model constraint), delegates to `_map_kwargs`, which yields `map_path=...` and an optional single-segment `struct_path`. + A struct-nested `Direct` target (a model reached through a plain struct + field) yields `element_path=...`, the pure-struct descent the mutation + walks to reach the constrained model. """ - if isinstance(check.target, MapPath): - return _map_kwargs(check.target, mutation_fn, allow_leaf=True) - if not isinstance(check.target, ArrayPath): - return [] - inner_struct_paths = check.target.iter_struct_paths - leaf_path = check.target.leaf - - if inner_struct_paths: + target = check.target + if isinstance(target, Direct): + return _struct_nested_kwargs(target) + if isinstance(_first_iter_segment(target), MapSegment): + return _map_kwargs(target, mutation_fn, allow_leaf=True) + composite = _array_first_map_kwargs(target) + if composite is not None: + return composite + if target.iter_struct_paths: raise ValueError( f"{mutation_fn} does not accept inner_array_path " - f"(inner struct paths={inner_struct_paths!r})" + f"(inner struct paths={target.iter_struct_paths!r})" ) - kwargs = [f'array_path="{check.target.column_path}"'] - if leaf_path: - if len(leaf_path) > 1: + kwargs = [f'array_path="{target.outer_column}"'] + if target.leaf: + if len(target.leaf) > 1: raise ValueError( - f"multi-segment leaf_path {leaf_path!r} not supported by " + f"multi-segment leaf_path {target.leaf!r} not supported by " f"{mutation_fn} (struct_path must be a single segment)" ) - kwargs.append(f'struct_path="{leaf_path[0]}"') + kwargs.append(f'struct_path="{target.leaf[0]}"') return kwargs def _iter_kwargs_inner(check: ModelCheck, mutation_fn: str) -> list[str]: """Container kwargs for mutations accepting `inner_array_path`. - For an `ArrayPath`, yields `array_path=...` and optionally + For an array target, yields `array_path=...` and optionally `inner_array_path=...`; a trailing leaf path is rejected -- these mutations target an inner array directly, not a struct field on its - elements. For a `MapPath`, delegates to `_map_kwargs` (no leaf: a map - value has no inner array layer to address). + elements. For a map target, delegates to `_map_kwargs` (no leaf: a map + value has no inner array layer to address). A struct-nested `Direct` + target yields `element_path=...` (the pure-struct descent to the model). """ - if isinstance(check.target, MapPath): - return _map_kwargs(check.target, mutation_fn, allow_leaf=False) - if not isinstance(check.target, ArrayPath): - return [] - inner_struct_paths = check.target.iter_struct_paths - leaf_path = check.target.leaf - - if leaf_path: + target = check.target + if isinstance(target, Direct): + return _struct_nested_kwargs(target) + if isinstance(_first_iter_segment(target), MapSegment): + return _map_kwargs(target, mutation_fn, allow_leaf=False) + composite = _array_first_map_kwargs(target) + if composite is not None: + return composite + if target.leaf: raise ValueError( - f"{mutation_fn} does not accept struct_path (leaf_path={leaf_path!r})" + f"{mutation_fn} does not accept struct_path (leaf_path={target.leaf!r})" ) - kwargs = [f'array_path="{check.target.column_path}"'] - if inner_struct_paths: - if len(inner_struct_paths) > 1: + kwargs = [f'array_path="{target.outer_column}"'] + if target.iter_struct_paths: + if len(target.iter_struct_paths) > 1: raise ValueError( - f"multi-level inner struct paths {inner_struct_paths!r} not supported by " - f"{mutation_fn} (inner_array_path consumes one iteration)" + f"multi-level inner struct paths {target.iter_struct_paths!r} not " + f"supported by {mutation_fn} (inner_array_path consumes one iteration)" ) - if not inner_struct_paths[0]: + if not target.iter_struct_paths[0]: raise ValueError( f"empty inner struct path not supported by {mutation_fn} " - f"(target={check.target!r}); nested-iteration arrays without " + f"(target={target!r}); nested-iteration arrays without " f"intermediate struct fields cannot be addressed via inner_array_path" ) - kwargs.append(f'inner_array_path="{".".join(inner_struct_paths[0])}"') + kwargs.append(f'inner_array_path="{".".join(target.iter_struct_paths[0])}"') return kwargs diff --git a/packages/overture-schema-codegen/tests/test_pyspark_check_builder.py b/packages/overture-schema-codegen/tests/test_pyspark_check_builder.py index 88d8dc5da..c4c9f84ef 100644 --- a/packages/overture-schema-codegen/tests/test_pyspark_check_builder.py +++ b/packages/overture-schema-codegen/tests/test_pyspark_check_builder.py @@ -17,10 +17,7 @@ union_spec_for, ) from overture.schema.codegen.extraction.field import ( - ArrayOf, ConstraintSource, - FieldShape, - MapOf, Primitive, UnionRef, ) @@ -33,7 +30,6 @@ from overture.schema.codegen.pyspark._render_common import column_level_suffix from overture.schema.codegen.pyspark.check_builder import ( build_checks, - classify_map_projection, ) from overture.schema.codegen.pyspark.check_ir import ( Check, @@ -50,12 +46,12 @@ from overture.schema.common.scoping.lr import LinearlyReferencedRange from overture.schema.system.field_constraint.collection import UniqueItemsConstraint from overture.schema.system.field_path import ( - ArrayPath, ArraySegment, + Direct, FieldPath, - MapPath, + Iterated, MapProjection, - ScalarPath, + MapSegment, parse, ) from overture.schema.system.model_constraint import ( @@ -89,6 +85,23 @@ def _element_guard(check: Check) -> ElementGuard | None: return None +def _map_seg(target: FieldPath) -> MapSegment | None: + """Return the sole `MapSegment` of a map-reached target, or None. + + Captures the former map-path shape: an `Iterated` with exactly one + `MapSegment` and no `ArraySegment` (map reached struct-only). Tests use + it to filter map targets and read the projection now that the taxonomy + collapsed to `Direct` / `Iterated`. + """ + if not isinstance(target, Iterated): + return None + maps = [s for s in target.segments if isinstance(s, MapSegment)] + arrays = [s for s in target.segments if isinstance(s, ArraySegment)] + if len(maps) == 1 and not arrays: + return maps[0] + return None + + def _checks_for( model_cls: type[BaseModel], ) -> tuple[list[Check], list[ModelCheck]]: @@ -413,13 +426,13 @@ def test_array_min_length_is_scalar_shape(self, nodes: list[Check]) -> None: if any(d.function == "check_array_min_length" for d in n.descriptors) ] assert len(length_nodes) == 1 - assert isinstance(length_nodes[0].target, ScalarPath) + assert isinstance(length_nodes[0].target, Direct) def test_array_element_field_uses_bracket_notation( self, nodes: list[Check] ) -> None: paths = {n.target for n in nodes} - assert any(isinstance(p, ArrayPath) for p in paths) + assert any(isinstance(p, Iterated) for p in paths) def test_array_element_subfield_path(self, nodes: list[Check]) -> None: # ItemModel.value is required, so a check node for items[].value must exist @@ -690,7 +703,7 @@ def test_http_url_element_check_is_array_shape(self, nodes: list[Check]) -> None for n in nodes if any(d.function == "check_url_format" for d in n.descriptors) ] - assert isinstance(url_nodes[0].target, ArrayPath) + assert isinstance(url_nodes[0].target, Iterated) class _DeepInner(BaseModel): @@ -742,14 +755,14 @@ def test_list_subfield_element_checks_have_inner_levels( element_nodes = [n for n in nodes if n.target == _path("items[].tags[]")] assert len(element_nodes) >= 1 - def test_list_subfield_column_path_is_enclosing_array( + def test_list_subfield_outer_column_is_enclosing_array( self, nodes: list[Check] ) -> None: tag_nodes = [n for n in nodes if str(n.target).startswith("items[].tags")] for node in tag_nodes: - assert isinstance(node.target, ArrayPath) + assert isinstance(node.target, Iterated) # the outermost iterated column is `items`, not the inner `tags` list - assert node.target.array_chunks[0] == ((), "items", 1) + assert node.target.iter_frames[0] == ((), ArraySegment(name="items")) class _ArrayElementWithNewtype(BaseModel): @@ -834,6 +847,50 @@ def test_field_names_use_leaf_path(self, node: ModelCheck) -> None: assert set(node.descriptor.field_names) == {"fast.value", "slow.value"} +@require_any_of("foo", "bar") +class _StructNestedConstrained(BaseModel): + foo: int | None = None + bar: str | None = None + + +class _FeatureWithRequiredStruct(BaseModel): + details: _StructNestedConstrained + + +class _FeatureWithOptionalStruct(BaseModel): + details: _StructNestedConstrained | None = None + + +class TestStructNestedModelConstraint: + """A model constraint on a submodel reached through a plain struct field. + + The constraint anchors at the struct prefix (`Direct('details')`), not the + row root, so the renderer qualifies each field reference with the prefix. + A required struct carries no gate; an optional one gates on the struct + being non-null. Both cases -- formerly `NotImplementedError` at + `build_checks` -- now build a `ModelCheck` cleanly. + """ + + def test_required_struct_targets_prefix_without_gate(self) -> None: + _, model_nodes = _checks_for(_FeatureWithRequiredStruct) + nodes = _filter_nodes(model_nodes, "check_require_any_of") + assert len(nodes) == 1 + node = nodes[0] + assert node.target == _path("details") + assert node.gate is None + assert set(node.descriptor.field_names) == {"foo", "bar"} + assert node.read_columns == frozenset({"details"}) + + def test_optional_struct_gates_on_prefix(self) -> None: + _, model_nodes = _checks_for(_FeatureWithOptionalStruct) + nodes = _filter_nodes(model_nodes, "check_require_any_of") + assert len(nodes) == 1 + node = nodes[0] + assert node.target == _path("details") + assert node.gate == _path("details") + assert node.read_columns == frozenset({"details"}) + + class _SyntheticUnionFixtures: """Discriminated-union models exercising union check generation.""" @@ -985,7 +1042,7 @@ def test_forbid_if_nodes_are_top_level(self, model_nodes: list[ModelCheck]) -> N forbid_nodes = _filter_nodes(model_nodes, "check_forbid_if") assert len(forbid_nodes) == 2 for node in forbid_nodes: - assert node.target == ScalarPath() + assert node.target == Direct() class TestUnionMemberModelConstraints: @@ -1123,25 +1180,28 @@ class _VariantWithConstrainedModelRef(_SyntheticUnionFixtures.Base): class TestVariantSpecificDirectModelRefConstraint: - """Variant-specific non-list `ModelRef` with a constrained sub-model is unsupported. - - The direct-ref path routes through `_recurse_into_model` rather - than the array branch of `_walk_field_shape`, and pure struct - nesting can't anchor a real model constraint -- the dispatch - raises `NotImplementedError`. Distinct from the `list[Model]` - case in `TestVariantSpecificFieldDiscoveredModelConstraints`, - which is supported. + """Variant-specific non-list `ModelRef` with a constrained sub-model is supported. + + The direct-ref path routes through `_recurse_into_model` rather than the + array branch of `_walk_field_shape`. The constraint anchors at the struct + prefix (`Direct('speed')`) and the renderer qualifies field references + (`F.col("speed.max_speed")`). Because the variant field is optional + (`speed: _SpeedLimitElement | None`), the check carries a nullable gate on + the struct so an absent `speed` skips the constraint, and it inherits the + contributing arm (`"d"`) rather than being broadcast to every arm. """ - def test_direct_modelref_constraint_raises(self) -> None: - # Pure struct nesting can't anchor a real model constraint; today - # the only constraint kind that survives struct nesting raises. - with pytest.raises( - NotImplementedError, match="Model constraint on struct-nested" - ): - _union_model_nodes( - "DirectModelRefConstraint", _DirectModelRefConstraintUnion - ) + def test_direct_modelref_constraint_emits_struct_nested_check(self) -> None: + model_nodes = _union_model_nodes( + "DirectModelRefConstraint", _DirectModelRefConstraintUnion + ) + nodes = _filter_nodes(model_nodes, "check_require_any_of") + assert len(nodes) == 1 + node = nodes[0] + assert node.target == _path("speed") + assert node.gate == _path("speed") + assert node.arm == "d" + assert node.read_columns == frozenset({"speed"}) class _OuterWithStructNestedUnion(BaseModel): @@ -1154,7 +1214,7 @@ class TestStructNestedUnionWithConstraint: """Non-list `UnionRef` reaching a union with model checks is unsupported. `_recurse_into_union` mirrors `_recurse_into_model`'s guard: when - the prefix is struct-nested (no `ArrayPath` segment) and the union + the prefix is struct-nested (no `Iterated` segment) and the union would emit either union-level constraints or synthesized exclusivity checks (`check_forbid_if`/`check_require_if`), the dispatch raises because `_model_constraint_target` would collapse @@ -1213,14 +1273,14 @@ def test_struct_nested_union_variant_fields_raises( build_checks(discriminated_union_ref_spec) def test_row_root_union_with_variant_fields_succeeds(self) -> None: - """Row-root union (empty `ScalarPath`) must still build checks without raising.""" + """Row-root union (empty `Direct`) must still build checks without raising.""" field_checks, _ = _union_checks( "Synthetic", _SyntheticUnionFixtures.SyntheticUnion ) assert any(n.guards for n in field_checks) def test_array_reached_union_with_variant_fields_succeeds(self) -> None: - """Array-reached union (`ArrayPath` prefix) must still build checks without raising.""" + """Array-reached union (`Iterated` prefix) must still build checks without raising.""" field_checks, _ = _checks_for(_ListUnionContainer) assert any(n.guards for n in field_checks) @@ -1660,7 +1720,7 @@ def test_segment_vehicle_selector_field_checks( assert any("access_restrictions" in str(n.target) for n in dim_nodes) for node in dim_nodes: - assert isinstance(node.target, ArrayPath) + assert isinstance(node.target, Iterated) # vehicle[] is nested inside an outer array (speed_limits, etc.), # so the struct nav to `dimension` lands in the target's leaf. assert len(node.target.leaf) >= 1 @@ -1672,14 +1732,14 @@ def test_segment_vehicle_selector_exclusivity( vehicle_forbid = [ n for n in _filter_nodes(model_nodes, "check_forbid_if") - if "unit" in n.descriptor.field_names and isinstance(n.target, ArrayPath) + if "unit" in n.descriptor.field_names and isinstance(n.target, Iterated) ] assert len(vehicle_forbid) > 0 vehicle_require = [ n for n in _filter_nodes(model_nodes, "check_require_if") - if "unit" in n.descriptor.field_names and isinstance(n.target, ArrayPath) + if "unit" in n.descriptor.field_names and isinstance(n.target, Iterated) ] assert len(vehicle_require) > 0 @@ -1690,10 +1750,10 @@ def test_segment_vehicle_selector_exclusivity_has_inner_levels( vehicle_constraint_nodes = [ n for n in _filter_nodes(model_nodes, ("check_forbid_if", "check_require_if")) - if "unit" in n.descriptor.field_names and isinstance(n.target, ArrayPath) + if "unit" in n.descriptor.field_names and isinstance(n.target, Iterated) ] for node in vehicle_constraint_nodes: - assert isinstance(node.target, ArrayPath) + assert isinstance(node.target, Iterated) # The target reaches the inner vehicle[] via a second iteration: # one inner level navigating `when` to the `vehicle` array. iter_paths = node.target.iter_struct_paths @@ -1760,7 +1820,7 @@ def test_a_field_check_produced(self, a_nodes: list[Check]) -> None: assert len(a_nodes) >= 1 def test_a_field_is_array_shape(self, a_nodes: list[Check]) -> None: - assert isinstance(a_nodes[0].target, ArrayPath) + assert isinstance(a_nodes[0].target, Iterated) def test_a_field_target_is_items(self, a_nodes: list[Check]) -> None: assert a_nodes[0].target == _path("items[].a_field") @@ -1769,7 +1829,7 @@ def test_a_field_guard(self, a_nodes: list[Check]) -> None: assert a_nodes[0].guards == (ElementGuard(discriminator="kind", values=("a",)),) def test_a_nodes_have_array_shape(self, a_nodes: list[Check]) -> None: - assert all(isinstance(n.target, ArrayPath) for n in a_nodes) + assert all(isinstance(n.target, Iterated) for n in a_nodes) def test_b_field_check_produced(self, b_nodes: list[Check]) -> None: assert len(b_nodes) >= 1 @@ -1778,20 +1838,20 @@ def test_b_field_guard(self, b_nodes: list[Check]) -> None: assert b_nodes[0].guards == (ElementGuard(discriminator="kind", values=("b",)),) def test_b_nodes_have_array_shape(self, b_nodes: list[Check]) -> None: - assert all(isinstance(n.target, ArrayPath) for n in b_nodes) + assert all(isinstance(n.target, Iterated) for n in b_nodes) def test_forbid_nodes_produced(self, model_nodes: list[ModelCheck]) -> None: forbid_nodes = _filter_nodes(model_nodes, "check_forbid_if") assert len(forbid_nodes) > 0 - def test_forbid_nodes_have_array_column_path( + def test_forbid_nodes_have_array_target( self, model_nodes: list[ModelCheck] ) -> None: forbid_nodes = _filter_nodes(model_nodes, "check_forbid_if") for node in forbid_nodes: assert node.target == _path("items[]") - def test_require_if_model_nodes_have_array_column_path( + def test_require_if_model_nodes_have_array_target( self, model_nodes: list[ModelCheck] ) -> None: require_nodes = _filter_nodes(model_nodes, "check_require_if") @@ -1800,25 +1860,23 @@ def test_require_if_model_nodes_have_array_column_path( class TestTopLevelUnionColumnPath: - """Top-level union (not inside array) exclusivity nodes have column_path=None.""" + """Top-level union (not inside array) exclusivity nodes have an empty Direct target.""" @pytest.fixture(scope="class") @classmethod def model_nodes(cls) -> list[ModelCheck]: return _union_model_nodes("Synthetic", _SyntheticUnionFixtures.SyntheticUnion) - def test_forbid_if_column_path_is_none(self, model_nodes: list[ModelCheck]) -> None: + def test_forbid_if_target_is_row_root(self, model_nodes: list[ModelCheck]) -> None: forbid_nodes = _filter_nodes(model_nodes, "check_forbid_if") assert len(forbid_nodes) > 0 for node in forbid_nodes: - assert node.target == ScalarPath() + assert node.target == Direct() - def test_require_if_column_path_is_none( - self, model_nodes: list[ModelCheck] - ) -> None: + def test_require_if_target_is_row_root(self, model_nodes: list[ModelCheck]) -> None: require_nodes = _filter_nodes(model_nodes, "check_require_if") for node in require_nodes: - assert node.target == ScalarPath() + assert node.target == Direct() class _ListUnionContainer(BaseModel): @@ -1864,22 +1922,110 @@ def test_variant_field_uses_element_level_discriminator( class _NestedListUnionContainer(BaseModel): """Top-level `list[list[DiscriminatedUnion]]` with a constrained member. - A union nested under multiple list layers would need the union - target to record `list_depth` iterations, but the rebase in - `_recurse_into_union` records only one. No real schema exercises - this path; `build_checks` raises rather than emit a target that - silently drops iterations. + A union nested under multiple list layers reaches its members through + a named `ArraySegment` plus an anonymous one. The fold wraps each + variant-gated field check at the innermost element, where the + `ElementGuard`'s discriminator co-locates with the leaf accessor, so + both the field checks and the member's model constraint target the full + `nested[][]` geometry. """ nested: list[list[_SyntheticUnionFixtures.ConstrainedUnion]] class TestNestedListUnionModelConstraints: - """`list[list[Union]]` raises rather than emit a collapsed target.""" + """`list[list[Union]]` renders with targets encoding both array levels.""" + + @pytest.fixture() + def checks(self) -> tuple[list[Check], list[ModelCheck]]: + return _checks_for(_NestedListUnionContainer) + + def test_member_constraint_targets_both_array_levels( + self, checks: tuple[list[Check], list[ModelCheck]] + ) -> None: + # The constrained member's `require_any_of` reaches through both list + # layers; its target pins the full `nested[][]` geometry (arm-tagged to + # the member's discriminator value) rather than a collapsed single level. + _field_checks, model_checks = checks + any_of = _filter_nodes(model_checks, "check_require_any_of") + assert [str(n.target) for n in any_of] == ["nested[][]"] + assert any_of[0].arm == "c" + + def test_variant_field_is_element_gated_at_both_levels( + self, checks: tuple[list[Check], list[ModelCheck]] + ) -> None: + field_checks, _model_checks = checks + node = _node_for(field_checks, "nested[][].a_field", "check_enum") + guard = _element_guard(node) + assert guard is not None and guard.discriminator == "kind" + assert guard.values == ("a",) + + +class _VariantArmBase(BaseModel): + subtype: str + + +class _VariantArmAList(_VariantArmBase): + subtype: Literal["a"] + codes: list[Annotated[int, Field(ge=0)]] | None = None + - def test_build_checks_raises_not_implemented(self) -> None: - with pytest.raises(NotImplementedError, match="multiple list layers"): - _checks_for(_NestedListUnionContainer) +class _VariantArmBLabel(_VariantArmBase): + subtype: Literal["b"] + label: str | None = None + + +class _DeepUnionListVariant(BaseModel): + deep: list[ + list[ + Annotated[ + _VariantArmAList | _VariantArmBLabel, Field(discriminator="subtype") + ] + ] + ] + + +class TestUnionVariantFieldPastDiscriminator: + """A union variant field that iterates past its discriminator element raises. + + `list[list[Union{codes: list[int>=0]}]]` reaches the `codes` element bound + through a third array iteration, past the `subtype`-discriminated union + element. The `ElementGuard` renders at the innermost iteration variable -- + the `codes` element, where the discriminator does not live -- so the render + would silently gate on the wrong element. `build_checks` raises rather than + emit a mis-gated check. + """ + + def test_union_variant_iterates_past_element_raises(self) -> None: + with pytest.raises( + NotImplementedError, match="beyond the discriminator's element" + ): + _checks_for(_DeepUnionListVariant) + + +class _MapKeyModel(BaseModel): + kf: str + + +class _MapValModel(BaseModel): + vf: str + + +class _MapModelKeyAndValue(BaseModel): + m: dict[_MapKeyModel, _MapValModel] + + +class TestDictModelKeyAndValueGuard: + """`dict[Model, Model]` stays a single-terminal-return limit. + + Both projections reach a sub-model, but `_walk_field_shape` returns a single + `_ShapeTerminal`, so only one projection's sub-model can be descended. + `build_checks` raises rather than silently drop the other. + """ + + def test_dict_model_model_raises(self) -> None: + with pytest.raises(NotImplementedError, match="single terminal"): + _checks_for(_MapModelKeyAndValue) class _DeepInnerModel(BaseModel): @@ -1901,8 +2047,8 @@ def nodes(self) -> list[Check]: def test_subfield_target_encodes_both_array_levels( self, nodes: list[Check] ) -> None: - # A `list[list[Model]]` sub-field reaches `value` through a single - # ArraySegment with iter_count=2; the target pins the full geometry. + # A `list[list[Model]]` sub-field reaches `value` through a named + # ArraySegment plus an anonymous one; the target pins the full geometry. assert any(n.target == _path("items[][].value") for n in nodes) @@ -1922,9 +2068,9 @@ class _NestedScalarListModel(BaseModel): """list[list[scalar]] terminating directly in a constrained scalar. Exercises the one nested-array geometry the other tests miss: an - element-level check whose target's terminal ArraySegment carries - iter_count > 1 with no struct leaf after it (`grid[][]`, not - `grid[][].field`). + element-level check whose target's terminal is a named ArraySegment + followed by an anonymous one, with no struct leaf after it + (`grid[][]`, not `grid[][].field`). """ grid: list[list[Annotated[str, MinLen(1)]]] @@ -1933,15 +2079,16 @@ class _NestedScalarListModel(BaseModel): class TestNestedScalarListTarget: """Element-level check on list[list[scalar]] targets a bare `field[][]`.""" - def test_terminal_target_carries_iter_count_two(self) -> None: + def test_terminal_target_carries_anonymous_segment(self) -> None: nodes, _ = _checks_for(_NestedScalarListModel) node = _node_for(nodes, "grid[][]", "check_string_min_length") target = node.target - assert isinstance(target, ArrayPath) - last = target.segments[-1] - assert isinstance(last, ArraySegment) - assert last.name == "grid" - assert last.iter_count == 2 + assert isinstance(target, Iterated) + named, anon = target.segments[-2:] + assert isinstance(named, ArraySegment) + assert named.name == "grid" + assert isinstance(anon, ArraySegment) + assert anon.is_anonymous class TestPrimitiveBoundsFiltered: @@ -2082,12 +2229,12 @@ def _map_check(self, projection: MapProjection, function: str) -> Check: matches = [ c for c in field_checks - if isinstance(c.target, MapPath) - and c.target.projection is projection + if (ms := _map_seg(c.target)) is not None + and ms.projection is projection and any(d.function == function for d in c.descriptors) ] assert len(matches) >= 1, ( - f"no MapPath {projection} check with {function}; " + f"no map {projection} check with {function}; " f"targets={[str(c.target) for c in field_checks]}" ) return matches[0] @@ -2102,152 +2249,48 @@ def test_map_value_stripped_check_targets_names_value(self) -> None: def test_map_field_with_unconstrained_value_emits_no_value_check(self) -> None: # metadata: dict[str, int] -- neither key nor value carries a - # constraint, so no MapPath checks are produced for it. + # constraint, so no map projection checks are produced for it. field_checks, _ = _checks_for(FeatureWithDict) metadata_maps = [ c for c in field_checks - if isinstance(c.target, MapPath) and c.target.map_column == "metadata" + if _map_seg(c.target) is not None + and isinstance(c.target, Iterated) + and c.target.outer_column == "metadata" ] assert metadata_maps == [] -class _MapWithConstrainedListValueModel(BaseModel): - """`dict[K, list[constrained-scalar]]` -- a map value carrying an array layer. - - `terminal_scalar` unwraps the `ArrayOf` to the inner scalar, so the - naive scalar guard lets this through; the value scalar's constraint - has no `MapPath` + `ArraySegment` geometry to land on. - """ - - items: dict[str, list[Annotated[str, MinLen(1)]]] - - class _MapWithUnconstrainedListValueModel(BaseModel): """`dict[K, list[scalar]]` with no key/value constraint -- nothing to emit.""" items: dict[str, list[int]] -class _ListOfConstrainedMapModel(BaseModel): - """`list[dict[K, constrained-scalar]]` -- a map reached through an array.""" - - items: list[dict[str, Annotated[str, MinLen(1)]]] - - class _ListOfUnconstrainedMapModel(BaseModel): """`list[dict[K, scalar]]` with no key/value constraint -- nothing to emit.""" items: list[dict[str, str]] -class _PlainScalarMapModel(BaseModel): - items: dict[str, str] +class TestUnconstrainedRicherMaps: + """Richer map/array nestings with no key/value constraint emit no checks. - -class _ConstrainedScalarMapModel(BaseModel): - items: dict[str, Annotated[str, MinLen(1)]] - - -class TestClassifyMapProjection: - """`classify_map_projection` is the single arbiter of map-shape support. - - Every map-shape prohibition in `_map_projection_checks` routes through - this classifier rather than restating the rule inline. The classifier - names the representable shape (struct-prefix -> one MapSegment -> scalar - or model/union terminal, reached without array iteration, no array layer - in the projected shape) and the reason each unsupported shape is rejected. - """ - - def _scalar_shape(self, *, constrained: bool) -> FieldShape: - spec = spec_for_model( - _ConstrainedScalarMapModel if constrained else _PlainScalarMapModel - ) - assert isinstance(spec, RecordSpec) - shape = spec.fields[0].shape - assert isinstance(shape, MapOf) - return shape.value - - def test_scalar_terminal_reached_struct_only_is_representable(self) -> None: - verdict = classify_map_projection( - self._scalar_shape(constrained=True), _path("items{value}") - ) - assert verdict.representable - assert verdict.reason is None - - def test_map_reached_through_array_is_rejected(self) -> None: - # The classifier owns the path-structural rejection too: a map_path - # that is an ArrayPath cannot anchor a struct-prefixed MapPath. - verdict = classify_map_projection( - self._scalar_shape(constrained=True), _path("items[]") - ) - assert not verdict.representable - assert verdict.reason is not None - - def test_array_layer_in_projected_shape_is_rejected(self) -> None: - spec = spec_for_model(_MapWithConstrainedListValueModel) - assert isinstance(spec, RecordSpec) - shape = spec.fields[0].shape - assert isinstance(shape, MapOf) - verdict = classify_map_projection(shape.value, _path("items{value}")) - assert not verdict.representable - assert verdict.reason is not None - - def test_classifier_rejects_dict_of_list_value(self) -> None: - # dict[K, list[V]]: the projected value shape carries an array layer. - # The classifier rejects it, and `_checks_for` raises -- the model - # raises iff the classifier rejects a shape with something to validate. - spec = spec_for_model(_MapWithConstrainedListValueModel) - assert isinstance(spec, RecordSpec) - shape = spec.fields[0].shape - assert isinstance(shape, MapOf) - verdict = classify_map_projection(shape.value, _path("items{value}")) - assert not verdict.representable - assert verdict.has_value_to_validate - with pytest.raises(NotImplementedError): - _checks_for(_MapWithConstrainedListValueModel) - - def test_classifier_rejects_map_reached_through_array(self) -> None: - # list[dict[K, V]]: the map is reached through an array, so the - # map_path is an ArrayPath. The classifier rejects on the path alone. - spec = spec_for_model(_ListOfConstrainedMapModel) - assert isinstance(spec, RecordSpec) - outer = spec.fields[0].shape - assert isinstance(outer, ArrayOf) - inner_map = outer.element - assert isinstance(inner_map, MapOf) - verdict = classify_map_projection(inner_map.value, _path("items[]")) - assert not verdict.representable - assert verdict.has_value_to_validate - with pytest.raises(NotImplementedError): - _checks_for(_ListOfConstrainedMapModel) - - -class TestMapProjectionUnsupportedShapes: - """`_map_projection_checks` is bounded to a scalar terminal reached struct-only. - - Two shapes fall outside that bound -- a map value/key with an array - layer (`dict[K, list[V]]`), and a map reached through an array - (`list[dict[K, V]]`). For each, a key/value constraint raises to keep - the dropped check loud, and an unconstrained one yields no checks (a - `MapPath` cannot locate the value, but there is nothing to validate). + `dict[K, list[V]]` and `list[dict[K, V]]` are now representable -- the + former representability gate is gone -- but an unconstrained projection + has nothing to validate, so the walk yields no map-projection checks, + the same quiet treatment an unconstrained scalar map gets. The + constraint-carrying variants of these shapes are exercised by the + hand-built execution test, not here. """ - def test_constrained_list_value_raises(self) -> None: - with pytest.raises(NotImplementedError, match="map value"): - _checks_for(_MapWithConstrainedListValueModel) - def test_unconstrained_list_value_emits_no_projection_check(self) -> None: field_checks, _ = _checks_for(_MapWithUnconstrainedListValueModel) - assert not any(isinstance(c.target, MapPath) for c in field_checks) - - def test_constrained_map_in_array_raises(self) -> None: - with pytest.raises(NotImplementedError, match="map value"): - _checks_for(_ListOfConstrainedMapModel) + assert not any(_map_seg(c.target) is not None for c in field_checks) def test_unconstrained_map_in_array_emits_no_projection_check(self) -> None: field_checks, _ = _checks_for(_ListOfUnconstrainedMapModel) - assert not any(isinstance(c.target, MapPath) for c in field_checks) + assert not any(_map_seg(c.target) is not None for c in field_checks) class _InnerLabel(BaseModel): @@ -2257,7 +2300,7 @@ class _InnerLabel(BaseModel): class _MapOfModel(BaseModel): """A `dict[K, Model]` value model with a constrained scalar field. - The value model's `label` field is validated on a `MapPath` leaf + The value model's `label` field is validated on a map leaf target (`items{value}.label`), the map analogue of a `list[Model]` element. """ @@ -2284,8 +2327,9 @@ class TestMapValueModelDescent: """check_builder descends into a `dict[K, Model]` value model. A `ModelRef`/`UnionRef` map value is walked for its field and - model-level constraints on a `MapPath` target, the map analogue of a - `list[Model]` element reached through the `ModelRef` walker arm. + model-level constraints on a map-leaf `Iterated` target, the map + analogue of a `list[Model]` element reached through the `ModelRef` + walker arm. """ def test_value_field_constraint_targets_map_value_leaf(self) -> None: @@ -2293,7 +2337,7 @@ def test_value_field_constraint_targets_map_value_leaf(self) -> None: matches = [ c for c in field_checks - if isinstance(c.target, MapPath) + if _map_seg(c.target) is not None and str(c.target) == "items{value}.label" and any(d.function == "check_string_min_length" for d in c.descriptors) ] @@ -2304,7 +2348,7 @@ def test_value_required_field_emits_required_descriptor(self) -> None: leaf_checks = [ c for c in field_checks - if isinstance(c.target, MapPath) and str(c.target) == "items{value}.label" + if _map_seg(c.target) is not None and str(c.target) == "items{value}.label" ] assert leaf_checks functions = {d.function for c in leaf_checks for d in c.descriptors} @@ -2314,24 +2358,10 @@ def test_value_model_constraint_targets_map_value(self) -> None: _, model_checks = _checks_for(_ModelConstraintAsMapValue) matches = _filter_nodes(model_checks, "check_require_any_of", ("foo", "bar")) assert len(matches) == 1 - assert isinstance(matches[0].target, MapPath) + assert _map_seg(matches[0].target) is not None assert str(matches[0].target) == "subs{value}" -class _MapValueWithList(BaseModel): - tags: list[Annotated[str, MinLen(1)]] - - -class _ListInsideMapValueModel(BaseModel): - """A `dict[K, Model]` value model with a constrained list field. - - A list nested inside a map element has no representable `MapPath`, so - the descent raises rather than emitting an unanchored target. - """ - - items: dict[str, _MapValueWithList] - - class _UrlOrEmptyModel(BaseModel): """Required field typed `HttpUrl | Literal[""]` -- literal bypass scenario.""" @@ -2389,17 +2419,3 @@ def test_check_required_never_gets_allow_literals(self) -> None: assert desc.allow_literals == (), ( f"check_required at {check.target} carries unexpected allow_literals" ) - - -class TestMapValueModelDescentBoundary: - """Descent raises where a `MapPath` cannot represent the shape. - - A map value model is descended into for scalar fields and model - constraints; a container (list or map) nested inside it has no - `MapPath` geometry, so the walker raises rather than emitting an - unvalidated target. - """ - - def test_list_inside_map_value_model_raises(self) -> None: - with pytest.raises(NotImplementedError, match="list nested inside a map"): - _checks_for(_ListInsideMapValueModel) diff --git a/packages/overture-schema-codegen/tests/test_pyspark_pipeline.py b/packages/overture-schema-codegen/tests/test_pyspark_pipeline.py index 74ba280b2..c0b512fbf 100644 --- a/packages/overture-schema-codegen/tests/test_pyspark_pipeline.py +++ b/packages/overture-schema-codegen/tests/test_pyspark_pipeline.py @@ -21,7 +21,7 @@ generate_pyspark_modules, ) from overture.schema.codegen.spec_discovery import extract_model_spec -from overture.schema.system.field_path import ScalarPath +from overture.schema.system.field_path import Direct from overture.schema.system.geometric import GeometryType from pydantic import BaseModel @@ -347,7 +347,7 @@ def test_aggregates_across_descriptors(self) -> None: args=(GeometryType.POINT,), ), ), - target=ScalarPath(), + target=Direct(), ), Check( descriptors=( @@ -356,7 +356,7 @@ def test_aggregates_across_descriptors(self) -> None: args=(GeometryType.POLYGON, GeometryType.LINE_STRING), ), ), - target=ScalarPath(), + target=Direct(), ), ] assert _extract_geometry_types(checks) == ( diff --git a/packages/overture-schema-codegen/tests/test_pyspark_renderer.py b/packages/overture-schema-codegen/tests/test_pyspark_renderer.py index a7f2c9497..56ac887b0 100644 --- a/packages/overture-schema-codegen/tests/test_pyspark_renderer.py +++ b/packages/overture-schema-codegen/tests/test_pyspark_renderer.py @@ -48,7 +48,7 @@ ) from overture.schema.codegen.pyspark.schema_builder import SchemaField, build_schema from overture.schema.system.field_path import ( - ScalarPath, + Direct, parse, ) from overture.schema.system.geometric import ( @@ -65,7 +65,7 @@ ) from overture.schema.system.numeric import int32 from overture.schema.system.string import CountryCodeAlpha2 -from pydantic import BaseModel, HttpUrl +from pydantic import BaseModel, Field, HttpUrl from pydantic.fields import FieldInfo _path = parse @@ -168,7 +168,6 @@ def test_model_check_array_target_reads_only_container_column(self) -> None: ) assert check.read_columns == frozenset({"items"}) - # IMPORTANT 1 — descriptor gate on scalar vs array target def test_scalar_target_gate_column_included(self) -> None: # A descriptor gate on a scalar target renders as F.col("{gate}").isNotNull(), # a row-level read; the gate's top-level column must appear in read_columns. @@ -191,7 +190,6 @@ def test_array_target_gate_column_excluded(self) -> None: ) assert check.read_columns == frozenset({"items"}) - # IMPORTANT 2 — RequireIf condition field exclusion on array target def test_model_check_require_if_array_target_excludes_condition_field(self) -> None: # On an array target, the condition is el["cond"] (element-relative), not # F.col("cond"); only the outer array column is a row-level read. @@ -215,7 +213,8 @@ def test_model_check_forbid_if_array_target_excludes_condition_field(self) -> No ) assert check.read_columns == frozenset({"items"}) - # MINOR 3 — RadioGroup and MinFieldsSet share the RequireAnyOf match arm + # RadioGroup and MinFieldsSet share the RequireAnyOf match arm, so all + # three variants derive read_columns identically. @pytest.mark.parametrize( "descriptor", [ @@ -231,11 +230,10 @@ def test_model_check_row_root_field_names_in_read_columns( check = ModelCheck(descriptor=descriptor) assert check.read_columns == frozenset({"a", "b"}) - # MINOR 4 — ModelCheck on a MapPath target - def test_model_check_map_target_reads_only_map_column(self) -> None: - # A dict[K, Model] value-model constraint targets a MapPath; field references - # use the projected element variable (v["field"]), not F.col. Only the map - # column itself is a row-level read. + def test_model_check_map_target_reads_only_outer_column(self) -> None: + # A dict[K, Model] value-model constraint targets a map-projection + # `Iterated`; field references use the projected element variable + # (v["field"]), not F.col. Only the map column itself is a row-level read. check = ModelCheck( descriptor=RequireAnyOf(field_names=("label", "value")), target=_path("names.common{value}"), @@ -1164,8 +1162,8 @@ def test_arm_unique_check_shares_arm_suffix(self) -> None: }, labeled -class TestMapPathRendering: - """MapPath targets render to map_keys_check / map_values_check.""" +class TestMapProjectionRendering: + """Map-projection targets render to map_keys_check / map_values_check.""" def test_map_key_renders_map_keys_check(self) -> None: check = Check( @@ -1471,6 +1469,45 @@ def test_render_variant_expr_in_nested_array_element_disc(self) -> None: assert 'inner["kind"]' in source +class _TagItem(BaseModel): + tags: dict[str, Annotated[str, Field(min_length=3)]] + + +class _TagItemList(BaseModel): + items: list[_TagItem] + + +class _NestedIntMap(BaseModel): + subs: dict[str, dict[str, Annotated[int, Field(ge=0)]]] + + +class TestMapValueUnderContainerRendering: + """A map value checked under a further container folds a flattening helper + around `map_values_check`. + + The two mixed nestings -- a map value inside an array element and inside + another map value -- are the pairings no other renderer test pins; + `test_column_patterns` runs the same pairings in Spark. + """ + + def test_map_value_inside_array_element(self) -> None: + field_checks, _ = build_checks(spec_for_model(_TagItemList)) + assert any(str(c.target) == "items[].tags{value}" for c in field_checks) + source = _render(_TagItemList, "item_list") + assert "nested_array_check(" in source + assert "map_values_check(" in source + assert 'el["tags"]' in source + assert "check_string_min_length" in source + + def test_map_value_inside_map_value(self) -> None: + field_checks, _ = build_checks(spec_for_model(_NestedIntMap)) + assert any(str(c.target) == "subs{value}{value}" for c in field_checks) + source = _render(_NestedIntMap, "map_of_map") + assert "nested_map_values_check(" in source + assert "map_values_check(" in source + assert "check_bounds(" in source + + @require_any_of("a", "b") class _DoubleNestedConstrainedElement(BaseModel): a: str | None = None @@ -1650,7 +1687,7 @@ def test_gated_array_is_parseable(self, element_gated_check: Check) -> None: ast.parse(source) def test_column_level_gate_on_array_target_raises(self) -> None: - """A column-level gate on an ArrayPath target is not produced by check_builder.""" + """A column-level gate on an Iterated target is not produced by check_builder.""" check = Check( descriptors=( ExpressionDescriptor( @@ -1738,16 +1775,51 @@ def test_full_render_optional_sub_model_parseable(self) -> None: source = _render(_ArrayWithOptionalSubModel, "arr_optional_sub") ast.parse(source) - def test_gated_model_check_assertion_on_non_array_target(self) -> None: - """A gate paired with a non-ArrayPath target raises AssertionError.""" + def test_gated_model_check_assertion_on_row_root_target(self) -> None: + """A gate paired with an EMPTY (row-root) `Direct` target raises AssertionError. + + A struct-nested `Direct` target legitimately carries a gate (rendered as + `F.when(F.col("details").isNotNull(), ...)`); an empty `Direct` is the row + root, where a gate is meaningless and check_builder never emits one. + """ check = ModelCheck( descriptor=RequireAnyOf(field_names=("a", "b")), - target=ScalarPath(), + target=Direct(), gate=_path("items[].nested"), ) - with pytest.raises(AssertionError, match="gate.*non-ArrayPath"): + with pytest.raises(AssertionError, match="gate.*Direct target"): _render_model_node(check) + def test_gated_struct_nested_model_check_wraps_in_f_when(self) -> None: + """A struct-nested `Direct` target with a gate wraps in F.col(gate).isNotNull().""" + check = ModelCheck( + descriptor=RequireAnyOf(field_names=("foo", "bar")), + target=_path("details"), + gate=_path("details"), + ) + source = _render_model_node(check) + assert 'F.when(F.col("details").isNotNull()' in source + assert ( + 'check_require_any_of([F.col("details.foo"), F.col("details.bar")]' + in source + ) + ast.parse(source) + + def test_ungated_struct_nested_model_check_qualifies_field_refs(self) -> None: + """A required struct-nested `Direct` target qualifies field refs, no gate.""" + check = ModelCheck( + descriptor=RequireAnyOf(field_names=("foo", "bar")), + target=_path("details"), + gate=None, + ) + source = _render_model_node(check) + assert ( + 'check_require_any_of([F.col("details.foo"), F.col("details.bar")]' + in source + ) + assert "isNotNull" not in source + ast.parse(source) + class TestMapValueModelRendering: """Render `dict[K, Model]` value-model checks inside a map lambda. @@ -1804,8 +1876,8 @@ def test_full_module_parseable_with_map_value_constraint(self) -> None: ast.parse(source) assert "map_values_check(" in source - def test_model_constraint_func_name_prefixes_map_column(self) -> None: - # Mirrors the ArrayPath naming so distinct map columns yield distinct + def test_model_constraint_func_name_prefixes_outer_column(self) -> None: + # Mirrors the array-target naming so distinct map columns yield distinct # generated function names rather than colliding on `__`. check = self._model_check(MapValueConstraintModel) source = _render_model_node(check) diff --git a/packages/overture-schema-codegen/tests/test_pyspark_scaffold.py b/packages/overture-schema-codegen/tests/test_pyspark_scaffold.py index dbff1f009..84cdace4f 100644 --- a/packages/overture-schema-codegen/tests/test_pyspark_scaffold.py +++ b/packages/overture-schema-codegen/tests/test_pyspark_scaffold.py @@ -28,7 +28,7 @@ generate_scaffold, leaf_list_depth, ) -from overture.schema.system.field_path import ArrayPath, parse +from overture.schema.system.field_path import ArraySegment, Iterated, parse from pydantic import TypeAdapter _path = parse @@ -253,7 +253,9 @@ def test_multiple_element_guards_raises(self, segment_spec: ModelSpec) -> None: # Spans a union with a union-in-array (`Segment`'s `when.vehicle[]`), record # specs with `require_any_of` and optional nested-model arrays, a map field # (`Infrastructure.source_tags`), and `list[list[...]]` arrays -# (`Division.hierarchies[][]`, so iter_count>1 wrapping is covered). The +# (`Division.hierarchies[][]`, whose inner `[]` is an anonymous array segment -- +# the nested list, carrying no field name of its own -- so anonymous-segment +# wrapping is covered). The # conformance suite only asserts each scenario's own expected violation is # absent from its valid row -- whole-row validity of a scaffold is checked here, # so a model-specific scaffold defect can't hide behind it. @@ -371,13 +373,18 @@ def test_array_nested_model_constraint_builds_path( _, model_checks = build_checks(segment_spec) if not model_checks: pytest.skip("Segment has no model constraints") - # Find one with an array target. - nested = [c for c in model_checks if isinstance(c.target, ArrayPath)] + # Find one with an array target (array-first `Iterated`). + nested = [ + c + for c in model_checks + if isinstance(c.target, Iterated) + and isinstance(c.target.iter_frames[0][1], ArraySegment) + ] if not nested: pytest.skip("No nested model constraints found") check = nested[0] scaffold = generate_model_scaffold(check, segment_spec) assert isinstance(scaffold, dict) # The scaffold should contain the column root (top-level column name). - assert isinstance(check.target, ArrayPath) - assert check.target.array_chunks[0][1] in scaffold + assert isinstance(check.target, Iterated) + assert check.target.iter_frames[0][1].name in scaffold diff --git a/packages/overture-schema-codegen/tests/test_pyspark_test_renderer.py b/packages/overture-schema-codegen/tests/test_pyspark_test_renderer.py index 54a27291e..82da0d6de 100644 --- a/packages/overture-schema-codegen/tests/test_pyspark_test_renderer.py +++ b/packages/overture-schema-codegen/tests/test_pyspark_test_renderer.py @@ -3,11 +3,14 @@ import ast import re from enum import Enum +from typing import Annotated import pytest from overture.schema.codegen.extraction.field import ArrayOf, ModelRef, Primitive +from overture.schema.codegen.extraction.model_extraction import extract_model from overture.schema.codegen.extraction.specs import RecordSpec from overture.schema.codegen.pyspark._primitive_fill import PRIMITIVE_FILL_TABLE +from overture.schema.codegen.pyspark.check_builder import build_checks from overture.schema.codegen.pyspark.check_ir import ( Check, ColumnGuard, @@ -40,9 +43,14 @@ NoWhitespaceConstraint, StrippedConstraint, ) -from overture.schema.system.field_path import ArrayPath, ScalarPath, parse +from overture.schema.system.field_path import Direct, Iterated, parse from overture.schema.system.geometric.geom import GeometryType -from overture.schema.system.model_constraint import FieldEqCondition, Not +from overture.schema.system.model_constraint import ( + FieldEqCondition, + Not, + require_any_of, +) +from pydantic import BaseModel, Field _path = parse @@ -105,33 +113,33 @@ def _array( column: str, inner_struct_paths: tuple[tuple[str, ...], ...] = (), leaf_path: tuple[str, ...] = (), -) -> ArrayPath: - """Build an ArrayPath from a column name, inner struct paths, and a leaf path. +) -> Iterated: + """Build an `Iterated` from a column name, inner struct paths, and a leaf path. Each entry in `inner_struct_paths` is `(prefix_structs..., inner_array_name)`: the prefix names become struct segments and the last name becomes an inner ArraySegment. """ - column_path = _path(column) - if isinstance(column_path, ScalarPath): - prefix_structs = column_path.segments[:-1] - outer_name = column_path.segments[-1].name - prefix = ScalarPath(segments=prefix_structs) - path = prefix.append_array(outer_name, iter_count=1) + col_path = _path(column) + if isinstance(col_path, Direct): + prefix_structs = col_path.segments[:-1] + outer_name = col_path.segments[-1].name + prefix = Direct(segments=prefix_structs) + path = prefix.append_array(outer_name) else: - assert isinstance(column_path, ArrayPath) # never a MapPath here - path = column_path + assert isinstance(col_path, Iterated) # never a map here + path = col_path for sp in inner_struct_paths: for n in sp[:-1]: path = path.append_struct(n) - path = path.append_array(sp[-1], iter_count=1) + path = path.append_array(sp[-1]) for n in leaf_path: path = path.append_struct(n) return path -class TestMapPathScenarios: - """MapPath field checks emit mutate_map_key / mutate_map_value scenarios.""" +class TestMapProjectionScenarios: + """Map-projection field checks emit mutate_map_key / mutate_map_value scenarios.""" def test_map_key_emits_mutate_map_key(self) -> None: check = make_check( @@ -168,6 +176,105 @@ def test_map_mutation_helper_is_imported(self) -> None: assert source.count("mutate_map_value") >= 2 +class TestMapInArrayFieldChecks: + """A map reached after array iteration is not a map target. + + `_is_map_target` reads the FIRST iterating frame (map-first), matching + `check_builder`. An array-first map leaf (`items[].tags{value}`) is + array-first, so its mutation must descend the array via `set_at_path`'s + map grammar, not corrupt a top-level map with `mutate_map_value`. + """ + + def test_map_in_array_uses_set_at_path_not_map_helper(self) -> None: + check = make_check( + "check_string_min_length", + _path("items[].tags{value}"), + args=(3,), + ) + source = render_test_module("itemlist", [check], []) + ast.parse(source) + assert "set_at_path('items[].tags{value}', '')" in source + assert "mutate_map_value(row, 'items'" not in source + + def test_map_first_still_uses_map_helper(self) -> None: + """A genuine map-first field check keeps the in-place map mutation.""" + check = make_check( + "check_stripped", + _path("names.common{value}"), + constraint_type=StrippedConstraint, + ) + source = render_test_module("dictfeat", [check], []) + ast.parse(source) + assert "mutate_map_value(row, 'names.common'" in source + + +class TestMapOfContainerFieldChecks: + """Map-first field checks whose value is itself a container. + + `dict[K, list[...]]` (`subs{value}[]`) and `dict[K, dict[K2, ...]]` + (`subs{value}{value}`) reach the constrained scalar map-first, then + through anonymous trailing iteration. `mutate_map_key` / `mutate_map_value` + corrupt a map's single entry in place and cannot descend a container + value, so these route to `set_at_path` with the full path, which peels + each trailing container to reach the scalar. A struct leaf after a map + (`subs{value}.label`) has no such peel and stays loudly gated. + """ + + def test_map_of_list_trailing_iteration_uses_set_at_path(self) -> None: + check = make_check( + "check_string_min_length", + _path("subs{value}[]"), + args=(3,), + ) + source = render_test_module("mapoflist", [check], []) + ast.parse(source) + assert "set_at_path('subs{value}[]', '')" in source + assert "mutate_map_value(row, 'subs'" not in source + + def test_map_of_map_trailing_iteration_uses_set_at_path(self) -> None: + check = make_check( + "check_bounds", + _path("subs{value}{value}"), + kwargs=(("ge", 0),), + ) + source = render_test_module("mapofmap", [check], []) + ast.parse(source) + assert "set_at_path('subs{value}{value}', -1)" in source + assert "mutate_map_value(row, 'subs'" not in source + + def test_map_value_struct_leaf_still_raises(self) -> None: + """A struct leaf after a map projection has no in-place mutation.""" + check = make_check( + "check_string_min_length", + _path("subs{value}.label"), + args=(3,), + ) + with pytest.raises(NotImplementedError, match="subs"): + render_test_module("mapofmodel", [check], []) + + def test_map_of_list_field_check_uses_set_at_path_through_full_pipeline( + self, + ) -> None: + """`extract_model` + `check_builder` really produce `subs{value}[]`. + + Drives the full synthetic-model path (`extract_model` -> `build_checks` + -> `render_test_module`), proving the mutation routing fires on a target + `check_builder` actually emits, not just a hand-built one. + """ + + class _MapOfList(BaseModel): + subs: dict[str, list[Annotated[str, Field(min_length=3)]]] + + spec = extract_model(_MapOfList) + field_checks, model_checks = build_checks(spec) + assert any(str(fc.target) == "subs{value}[]" for fc in field_checks) + source = render_test_module( + "map_of_list", field_checks, model_checks, spec=spec + ) + ast.parse(source) + assert "set_at_path('subs{value}[]'" in source + + class TestRenderTestModuleParseable: def test_renders_valid_python_with_nodes(self) -> None: nodes = [make_check("check_required", _path("country"))] @@ -600,6 +707,24 @@ def test_min_fields_set_map_value_uses_map_path(self) -> None: ast.parse(source) assert 'map_path="subs"' in source + def test_min_fields_set_map_of_list_uses_element_path(self) -> None: + """`min_fields_set` also generalizes to the composite descent path. + + Mirrors `test_require_any_of_map_of_list_uses_element_path`: proves + the "min_fields_set element_path emission" claim, not just + require_any_of/require_if. + """ + model_nodes = [ + ModelCheck( + descriptor=MinFieldsSet(field_names=("foo", "bar"), count=1), + target=_path("subs{value}[]"), + ), + ] + source = render_test_module("test", [], model_nodes) + ast.parse(source) + assert 'element_path="subs{value}[]"' in source + assert 'map_path="subs"' not in source + def test_require_if_map_value_uses_map_path(self) -> None: model_nodes = [ ModelCheck( @@ -614,6 +739,103 @@ def test_require_if_map_value_uses_map_path(self) -> None: ast.parse(source) assert 'map_path="subs"' in source + def test_require_any_of_map_of_list_uses_element_path(self) -> None: + """A `dict[K, list[Model]]` target descends map-then-array. + + `subs{value}[]` reaches the constrained model through a map value that + is a list -- no scalar `map_path` addresses it, since the map value is + a list, not the model. The renderer emits the full composite descent + path, which the mutation walks: sole map value, then each list element. + """ + model_nodes = [ + ModelCheck( + descriptor=RequireAnyOf(field_names=("foo", "bar")), + target=_path("subs{value}[]"), + ), + ] + source = render_test_module("test", [], model_nodes) + ast.parse(source) + assert 'element_path="subs{value}[]"' in source + assert 'map_path="subs"' not in source + + def test_require_any_of_struct_nested_uses_element_path(self) -> None: + """A struct-nested `Direct('details')` target descends via element_path. + + A model constraint on a submodel reached through a plain struct field + emits the composite `element_path` -- the same kwarg map/array targets + use, now with only a struct segment -- rather than any container path. + """ + model_nodes = [ + ModelCheck( + descriptor=RequireAnyOf(field_names=("foo", "bar")), + target=_path("details"), + ), + ] + source = render_test_module("feature", [], model_nodes) + ast.parse(source) + assert 'element_path="details"' in source + + def test_require_any_of_array_of_map_uses_element_path(self) -> None: + """A `list[dict[K, Model]]` target descends array-then-map. + + `items[].configs{value}` reaches the constrained model through a map + value nested under array iteration. No scalar `array_path` expresses + the map boundary; the renderer emits the composite descent path. + """ + model_nodes = [ + ModelCheck( + descriptor=RequireAnyOf(field_names=("foo", "bar")), + target=_path("items[].configs{value}"), + ), + ] + source = render_test_module("test", [], model_nodes) + ast.parse(source) + assert 'element_path="items[].configs{value}"' in source + assert 'array_path="items"' not in source + + def test_require_if_map_of_list_uses_element_path(self) -> None: + """The `inner_array_path` call path also emits the composite descent.""" + model_nodes = [ + ModelCheck( + descriptor=RequireIf( + field_names=("admin_level",), + condition=FieldEqCondition("subtype", "country"), + ), + target=_path("subs{value}[]"), + ), + ] + source = render_test_module("test", [], model_nodes) + ast.parse(source) + assert 'element_path="subs{value}[]"' in source + + def test_require_any_of_map_of_model_list_emits_element_path_full_pipeline( + self, + ) -> None: + """`extract_model` + `check_builder` really produce a `subs{value}[]` model check. + + Mirrors `TestMapOfContainerFieldChecks`'s full-pipeline test on the + model side: proves the composite emission fires on a target + `check_builder` actually emits for `dict[K, list[Model]]`, not just a + hand-built one. + """ + + @require_any_of("foo", "bar") + class _ValueModel(BaseModel): + foo: str | None = None + bar: str | None = None + + class _MapOfModelList(BaseModel): + subs: dict[str, list[_ValueModel]] + + spec = extract_model(_MapOfModelList) + field_checks, model_checks = build_checks(spec) + assert any(str(mc.target) == "subs{value}[]" for mc in model_checks) + source = render_test_module( + "map_of_model_list", field_checks, model_checks, spec=spec + ) + ast.parse(source) + assert 'element_path="subs{value}[]"' in source + def test_radio_group_map_value_raises(self) -> None: """radio_group has no map-aware mutation; raise rather than emit a vacuous test.""" model_nodes = [ @@ -622,7 +844,24 @@ def test_radio_group_map_value_raises(self) -> None: target=_path("subs{value}"), ), ] - with pytest.raises(ValueError, match="map_path"): + with pytest.raises(ValueError, match="nested target"): + render_test_module("test", [], model_nodes) + + def test_radio_group_struct_nested_raises(self) -> None: + """radio_group on a struct-nested submodel raises: no element_path descent. + + The validation renders correctly (field refs qualify to `details.a`), + but `mutate_radio_group` takes no navigation kwarg, so a struct-nested + target would set top-level `a`/`b` -- a silent misroute. Raising keeps + the two aligned; no live schema declares radio_group on a nested model. + """ + model_nodes = [ + ModelCheck( + descriptor=RadioGroup(field_names=("a", "b")), + target=_path("details"), + ), + ] + with pytest.raises(ValueError, match="nested target"): render_test_module("test", [], model_nodes) def test_require_any_of_map_key_projection_raises(self) -> None: @@ -636,6 +875,22 @@ def test_require_any_of_map_key_projection_raises(self) -> None: with pytest.raises(ValueError, match="map key"): render_test_module("test", [], model_nodes) + def test_require_any_of_array_of_map_key_projection_raises(self) -> None: + """A map key nested under array iteration is likewise not model-targetable. + + The composite emission rejects a KEY frame at codegen time, matching + `_map_kwargs`'s map-first guard, rather than emitting a path the + mutation walker would only reject at runtime. + """ + model_nodes = [ + ModelCheck( + descriptor=RequireAnyOf(field_names=("foo", "bar")), + target=_path("items[].configs{key}"), + ), + ] + with pytest.raises(ValueError, match="map key"): + render_test_module("test", [], model_nodes) + def test_require_any_of_map_value_multi_segment_leaf_raises(self) -> None: """The mutation struct_path is a single segment; a deeper leaf has no support.""" model_nodes = [ @@ -698,14 +953,46 @@ def test_require_any_of_with_inner_levels_raises(self) -> None: render_test_module("test", [], model_nodes) def test_radio_group_with_array_path_raises(self) -> None: - """radio_group takes no array kwargs; nodes with column_path raise.""" + """radio_group takes no array kwargs; nodes with an iterated target raise.""" model_nodes = [ ModelCheck( descriptor=RadioGroup(field_names=("a", "b")), target=_array("outer"), ), ] - with pytest.raises(ValueError, match="array_path"): + with pytest.raises(ValueError, match="nested target"): + render_test_module("test", [], model_nodes) + + def test_require_any_true_iterated_raises(self) -> None: + """require_any_true has no map/array-aware mutation; iterated targets raise. + + Mirrors `test_radio_group_with_array_path_raises`: without this guard, + `_render_require_any_true_mutation_call` ignores `check.target` entirely + and silently renders a root-level `mutate_require_any_true(row, {})` + call for a nested constraint -- a silent misroute, not a loud failure. + """ + model_nodes = [ + ModelCheck( + descriptor=RequireAnyTrue( + conditions=(FieldEqCondition("a", True),), + ), + target=_array("outer"), + ), + ] + with pytest.raises(ValueError, match="nested target"): + render_test_module("test", [], model_nodes) + + def test_require_any_true_map_value_raises(self) -> None: + """A map-first iterated target for require_any_true likewise raises.""" + model_nodes = [ + ModelCheck( + descriptor=RequireAnyTrue( + conditions=(FieldEqCondition("a", True),), + ), + target=_path("subs{value}"), + ), + ] + with pytest.raises(ValueError, match="nested target"): render_test_module("test", [], model_nodes) def test_require_if_with_leaf_path_raises(self) -> None: diff --git a/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/column_patterns.py b/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/column_patterns.py index 18acbb8ef..9ee43fa11 100644 --- a/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/column_patterns.py +++ b/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/column_patterns.py @@ -81,16 +81,21 @@ def _map_projection_check( column: str | Column, projector: Callable[[Column], Column], check_fn: Callable[[Column], Column], + flatten: bool = False, ) -> Column: """Project a map column to an array, then null-guard and transform it. - *projector* is `F.map_keys` or `F.map_values`. The projection already - yields a Column, so this calls `_null_guarded_transform` directly -- - routing through `array_check` would re-resolve an already-resolved - Column. A null map column projects to null, which the guard yields - through as null. + *projector* is `F.map_keys` or `F.map_values`. The projection already yields + a resolved Column, so this calls `_null_guarded_transform` directly rather + than routing through `array_check` (which would re-resolve the column). + When *flatten* is True the projected-element checks each return an + `array` (the map holds further iteration, e.g. `dict[K, list]`), + so the `array>` is flattened before compaction -- the map + analogue of `nested_array_check`. """ - return _null_guarded_transform(projector(_resolve_column(column)), check_fn) + return _null_guarded_transform( + projector(_resolve_column(column)), check_fn, flatten=flatten + ) def map_keys_check( @@ -115,6 +120,29 @@ def map_values_check( return _map_projection_check(column, F.map_values, check_fn) +def nested_map_keys_check( + column: str | Column, check_fn: Callable[[Column], Column] +) -> Column: + """Validate a map's keys when each key-check returns an `array`. + + The flattening analogue of `map_keys_check`; use when a map key holds + further iteration (a nested container). A null map column yields null. + """ + return _map_projection_check(column, F.map_keys, check_fn, flatten=True) + + +def nested_map_values_check( + column: str | Column, check_fn: Callable[[Column], Column] +) -> Column: + """Validate a map's values when each value-check returns an `array`. + + The flattening analogue of `map_values_check`; use when a map value holds + further iteration (`dict[K, list]`, `dict[K, dict]`). A null map column + yields null. + """ + return _map_projection_check(column, F.map_values, check_fn, flatten=True) + + def check_struct_unique(column: str | Column) -> Column: """Check that an array has no duplicate items by whole-element comparison. diff --git a/packages/overture-schema-pyspark/tests/_support/helpers.py b/packages/overture-schema-pyspark/tests/_support/helpers.py index 3905f8cea..6c3c28dfe 100644 --- a/packages/overture-schema-pyspark/tests/_support/helpers.py +++ b/packages/overture-schema-pyspark/tests/_support/helpers.py @@ -9,7 +9,13 @@ from collections.abc import Callable from typing import Any -from overture.schema.system.field_path import ArraySegment, FieldPath, coerce +from overture.schema.system.field_path import ( + ArraySegment, + FieldPath, + MapProjection, + MapSegment, + coerce, +) def deep_merge(base: dict, scaffold: dict) -> dict: @@ -61,27 +67,35 @@ def _scaffold_array(target: dict, name: str, path: FieldPath | str) -> list: return child -def _descend_through_array( - segment: ArraySegment, target: dict, path: FieldPath | str -) -> list: - """Enter an array segment and walk through its `iter_count`. +def _require_non_empty_list(target: object, path: FieldPath | str) -> list: + """Assert *target* is already a non-empty list, returning it. - Scaffolds `[{}]` at the outer level when None; deeper levels - (`iter_count > 1`) must already be lists — scaffolding into - nested-list shapes isn't supported because no current schema - needs it. + An anonymous `ArraySegment` descends one more list level via element 0 + with no field name to scaffold against, so its parent element must + already be a list — deeper levels must already be lists, since + scaffolding into nested-list shapes isn't supported (no current schema + needs it). + """ + if not isinstance(target, list): + raise PathTraversalError( + f"Expected nested list in path '{path}', got {type(target).__name__}" + ) + if len(target) == 0: + raise PathTraversalError(f"Empty nested list in path '{path}'") + return target + + +def _array_slot(segment: ArraySegment, target: dict, path: FieldPath | str) -> list: + """Return the list *segment* indexes into, scaffolding when named. - Returns the innermost list. For terminal use, write to `[0]`; - for intermediate use, the next segment lives in `[0]`. + A named segment enters an array field, scaffolding `[{}]` when it's + None. An anonymous segment descends one more list level of the same + field (`list[list[...]]`), which has no field name to scaffold + against, so its parent element must already be a non-empty list. """ - container = _scaffold_array(target, segment.name, path) - for _ in range(segment.iter_count - 1): - if len(container) == 0 or not isinstance(container[0], list): - raise PathTraversalError( - f"Expected non-empty nested list at '{segment.name}' in path '{path}'" - ) - container = container[0] - return container + if segment.is_anonymous: + return _require_non_empty_list(target, path) + return _scaffold_array(target, segment.name, path) def set_at_path(path: FieldPath | str, value: object) -> Callable[[dict], dict]: @@ -94,6 +108,14 @@ def set_at_path(path: FieldPath | str, value: object) -> Callable[[dict], dict]: an intermediate array segment is scaffolded as `[{}]`. Empty arrays raise `PathTraversalError` when called — there is no element to mutate. + A trailing map marker (`"items[].tags{value}"`, `"tags{key}"`) corrupts + the single entry of the map it reaches: `{value}` replaces the entry's + value, `{key}` replaces its key, each preserving the other side. A + non-terminal `{value}` (`"subs{value}[]"`, `"subs{value}{value}"`) + descends the sole entry's value and keeps navigating, so a map value that + is itself a container is reachable. A non-terminal `{key}` raises — a map + key is an immutable scalar with nothing to descend into. + Parameters ---------- path @@ -110,8 +132,10 @@ def set_at_path(path: FieldPath | str, value: object) -> Callable[[dict], dict]: Raises ------ PathTraversalError - When the path is empty, or when an intermediate or final array - segment is empty (raised at call time, not at factory time). + When the path is empty, when an intermediate or final array segment + is empty, when a terminal map is missing or empty, or when a + non-terminal `{key}` projection appears (raised at call time, not at + factory time). """ segments = coerce(path).segments @@ -121,15 +145,74 @@ def mutator(row_dict: dict) -> dict: result = copy.deepcopy(row_dict) target: Any = result for segment in segments[:-1]: - if isinstance(segment, ArraySegment): - target = _descend_through_array(segment, target, path)[0] + if isinstance(segment, MapSegment): + target = _descend_map_projection(segment, target, path) + elif isinstance(segment, ArraySegment): + target = _array_slot(segment, target, path)[0] else: target = _scaffold_struct(target, segment.name) last = segments[-1] - if isinstance(last, ArraySegment): - _descend_through_array(last, target, path)[0] = value + if isinstance(last, MapSegment): + _set_map_projection(last, target, path, value) + elif isinstance(last, ArraySegment): + _array_slot(last, target, path)[0] = value else: target[last.name] = value return result return mutator + + +def _map_at(segment: MapSegment, parent: object, path: FieldPath | str) -> dict: + """Return the map *segment* projects, resolved from *parent*. + + A named segment reads the map at `parent[segment.name]`; an anonymous + segment (the parent element is itself a map, e.g. `subs{value}{value}`) + projects *parent* directly. Raises `PathTraversalError` when the map is + missing or empty — there is no entry to descend or corrupt. + """ + if segment.is_anonymous: + m = parent + else: + m = parent.get(segment.name) if isinstance(parent, dict) else None + if not isinstance(m, dict) or not m: + where = "" if segment.is_anonymous else f" at '{segment.name}'" + raise PathTraversalError(f"Missing or empty map{where} in path '{path}'") + return m + + +def _descend_map_projection( + segment: MapSegment, parent: object, path: FieldPath | str +) -> object: + """Descend a non-terminal `{value}` projection into the sole entry's value. + + Returns the first entry's value so navigation continues into a container + the map holds (`subs{value}[]`, `subs{value}{value}`). A `{key}` projection + raises: a map key is an immutable scalar with nothing to descend into. + """ + m = _map_at(segment, parent, path) + if segment.projection is not MapProjection.VALUE: + raise PathTraversalError( + f"Non-terminal map key projection {segment.name!r} in path '{path}'; " + f"a map key is a scalar and cannot be descended" + ) + return m[next(iter(m))] + + +def _set_map_projection( + segment: MapSegment, parent: object, path: FieldPath | str, value: object +) -> None: + """Corrupt the single entry of the map *segment* names inside *parent*. + + A VALUE projection replaces the first entry's value with *value* (keeping + its key); a KEY projection replaces the first entry's key with *value* + (keeping its value). One bad entry suffices — map checks are element-wise, + mirroring how `[]` mutates only element 0. Raises `PathTraversalError` + when the map is missing or empty, since there is no entry to corrupt. + """ + m = _map_at(segment, parent, path) + first_key = next(iter(m)) + if segment.projection is MapProjection.VALUE: + m[first_key] = value + else: + m[value] = m.pop(first_key) diff --git a/packages/overture-schema-pyspark/tests/_support/mutations.py b/packages/overture-schema-pyspark/tests/_support/mutations.py index 6695ce104..6841a08dc 100644 --- a/packages/overture-schema-pyspark/tests/_support/mutations.py +++ b/packages/overture-schema-pyspark/tests/_support/mutations.py @@ -13,10 +13,14 @@ from overture.schema.system.field_path import ( ArraySegment, + Direct, FieldPath, FieldSegment, - ScalarPath, + MapProjection, + MapSegment, + StructSegment, coerce, + terminal_run_start, ) from .helpers import PathTraversalError @@ -33,6 +37,7 @@ def mutate_require_any_of( array_path: FieldPath | str | None = None, struct_path: str | None = None, map_path: FieldPath | str | None = None, + element_path: FieldPath | str | None = None, ) -> dict: """Null every named field so `require_any_of` fires. @@ -47,6 +52,11 @@ def mutate_require_any_of( map_path Map column whose `dict[K, Model]` value carries the constraint. Mutually exclusive with `array_path`. + element_path + Full element-relative descent to the constrained model, walked + generically when the nesting mixes map and array boundaries + (`subs{value}[]`, `items[].configs{value}`) that no scalar path + expresses. Mutually exclusive with the scalar paths above. See `_null_all_named_fields` for the full nesting semantics. """ @@ -56,6 +66,7 @@ def mutate_require_any_of( array_path=array_path, struct_path=struct_path, map_path=map_path, + element_path=element_path, ) @@ -88,6 +99,7 @@ def mutate_min_fields_set( array_path: FieldPath | str | None = None, struct_path: str | None = None, map_path: FieldPath | str | None = None, + element_path: FieldPath | str | None = None, ) -> dict: """Null every named field so `min_fields_set(N)` fires (0 < N). @@ -97,9 +109,10 @@ def mutate_min_fields_set( `check_required` checks; the conformance test only asserts the expected violation is present, so the extra failures don't matter. - `array_path` / `struct_path` / `map_path` mirror `mutate_require_any_of` - for the case where the constrained model is reached through array or map - iteration (and optionally one intermediate struct field). + `array_path` / `struct_path` / `map_path` / `element_path` mirror + `mutate_require_any_of` for the case where the constrained model is + reached through array or map iteration (and optionally one intermediate + struct field, or a mixed map/array descent). """ return _null_all_named_fields( row_dict, @@ -107,6 +120,7 @@ def mutate_min_fields_set( array_path=array_path, struct_path=struct_path, map_path=map_path, + element_path=element_path, ) @@ -117,18 +131,28 @@ def _null_all_named_fields( array_path: FieldPath | str | None, struct_path: str | None, map_path: FieldPath | str | None = None, + element_path: FieldPath | str | None = None, ) -> dict: """Return a deep copy of *row_dict* with every named field set to None. - Without *array_path* or *map_path*, the fields live at the row root. - With *array_path*, the fields live inside elements of that array column; - with *map_path*, inside the value model of that map column. *struct_path* - names an optional single intermediate struct field between the array - element / map value and the target fields. A null array is replaced with - a single stub element so the violation has a row to fire on; a null map - is stubbed analogously. + Without *array_path*, *map_path*, or *element_path*, the fields live at the + row root. With *array_path*, the fields live inside elements of that array + column; with *map_path*, inside the value model of that map column. + *struct_path* names an optional single intermediate struct field between + the array element / map value and the target fields. *element_path* carries + a full mixed map/array descent (`_descend_to_targets`). A null array is + replaced with a single stub element so the violation has a row to fire on; + a null map is stubbed analogously. """ result = copy.deepcopy(row_dict) + if element_path is not None: + + def _null(target: dict) -> None: + for name in field_names: + _set_nested(target, name, None) + + _descend_to_targets(result, coerce(element_path).segments, _null) + return result if map_path is not None: target = _map_value_to_mutate(result, map_path) if struct_path: @@ -185,6 +209,102 @@ def _map_value_to_mutate(row: dict, map_path: FieldPath | str) -> dict: return stub +def _descend_to_targets( + target: Any, segments: tuple[FieldSegment, ...], fn: _Applicator +) -> None: + """Descend *segments* from *target*, applying *fn* at each reached model dict. + + Walks a full element-relative path that mixes container boundaries: + + - a `StructSegment` navigates one struct field, scaffolding `{}` when + missing or None; + - an `ArraySegment` iterates *every* element (a named array is scaffolded + as `[{}]` when absent; an anonymous one -- the parent element is itself + the list -- iterates the parent); + - a `MapSegment` value projection descends the sole entry's value + (stubbing one entry when a named map is absent). + + A `MapSegment` key projection raises: a model can't sit on the key side. + Applying to every array element (rather than element 0) mirrors the scalar + `array_path` helpers, so one invalid element per array is guaranteed. + """ + if not segments: + fn(target) + return + seg, rest = segments[0], segments[1:] + if isinstance(seg, StructSegment): + _descend_to_targets(_scaffold_struct_child(target, seg.name), rest, fn) + elif isinstance(seg, ArraySegment): + for element in _element_array(target, seg): + _descend_to_targets(element, rest, fn) + elif isinstance(seg, MapSegment): + _descend_to_targets(_element_map_value(target, seg, rest), rest, fn) + else: + raise PathTraversalError(f"unrecognized path segment {seg!r}") + + +def _element_array(parent: Any, seg: ArraySegment) -> list: + """Return the list *seg* enters, scaffolding `[{}]` for an absent named array. + + An anonymous segment treats *parent* as the list itself; the parent must + already be a non-empty list, since there is no field name to scaffold + against (mirroring `set_at_path`'s anonymous-array handling). + """ + if seg.is_anonymous: + if not isinstance(parent, list) or not parent: + raise PathTraversalError(f"expected non-empty list for anonymous {seg!r}") + return parent + arr = parent.get(seg.name) if isinstance(parent, dict) else None + if arr is None: + arr = [{}] + parent[seg.name] = arr + return arr + + +def _element_map_value( + parent: Any, seg: MapSegment, rest: tuple[FieldSegment, ...] +) -> Any: + """Return the sole value of the map *seg* projects, stubbing when absent. + + A named map absent from *parent* is stubbed with one entry so the descent + reaches a target; an anonymous map must already be present (the parent + element is itself the map). A key projection raises -- a model can't sit on + a map key. The stub is shaped by *rest*, the segments still to descend + (see `_stub_map_value`): a dict for a `dict[K, Model]` value, a list for a + `dict[K, list[...]]` value. + """ + if seg.projection is not MapProjection.VALUE: + raise PathTraversalError( + f"model constraint cannot target a map key projection ({seg!r})" + ) + m = ( + parent + if seg.is_anonymous + else (parent.get(seg.name) if isinstance(parent, dict) else None) + ) + if isinstance(m, dict) and m: + return next(iter(m.values())) + if seg.is_anonymous: + raise PathTraversalError(f"missing anonymous map for {seg!r}") + stub = _stub_map_value(rest) + parent[seg.name] = {_STUB_MAP_KEY: stub} + return stub + + +def _stub_map_value(rest: tuple[FieldSegment, ...]) -> Any: + """Build the initial value for a stubbed map entry, shaped to match *rest*. + + A leading anonymous `ArraySegment` in *rest* means the map's value type is + itself a list (`dict[K, list[X]]`); the stub is `[]`, recursing to + match a further leading anonymous run (`dict[K, list[list[X]]]`). Any + other leading segment -- a struct field, a named container, or no segment + at all -- means the map's value type is a model; the stub is `{}`. + """ + if rest and isinstance(rest[0], ArraySegment) and rest[0].is_anonymous: + return [_stub_map_value(rest[1:])] + return {} + + def mutate_require_if( row_dict: dict, field_names: list[FieldPath | str], @@ -195,6 +315,7 @@ def mutate_require_if( array_path: FieldPath | str | None = None, inner_array_path: FieldPath | str | None = None, map_path: FieldPath | str | None = None, + element_path: FieldPath | str | None = None, ) -> dict: """Set condition to trigger require_if, then null target fields.""" result = copy.deepcopy(row_dict) @@ -204,7 +325,9 @@ def _apply(target: dict) -> None: for name in field_names: _set_nested(target, name, None) - _apply_to_targets(result, _apply, array_path, inner_array_path, map_path) + _apply_to_targets( + result, _apply, array_path, inner_array_path, map_path, element_path + ) return result @@ -219,6 +342,7 @@ def mutate_forbid_if( array_path: FieldPath | str | None = None, inner_array_path: FieldPath | str | None = None, map_path: FieldPath | str | None = None, + element_path: FieldPath | str | None = None, ) -> dict: """Set condition to trigger forbid_if, ensure target fields are non-null. @@ -235,7 +359,9 @@ def _apply(target: dict) -> None: if _get_nested(target, name) is None: _set_nested(target, name, fills.get(name, _SENTINEL)) - _apply_to_targets(result, _apply, array_path, inner_array_path, map_path) + _apply_to_targets( + result, _apply, array_path, inner_array_path, map_path, element_path + ) return result @@ -244,28 +370,32 @@ def mutate_unique_items(row_dict: dict, path: FieldPath | str) -> dict: Supports bracket paths like `"restrictions[].when.mode"` -- enters element 0 at each `[]` segment, then duplicates the first element - of the final array. A terminal `[]` (e.g. `"hierarchies[]"`) - targets the inner array at element 0 of the named field -- the - walker descends one extra level per bracket on the terminal - segment and duplicates the first element of the array it lands on. + of the final array. A terminal run of brackets (e.g. `"hierarchies[]"`, + or `"grid[][]"` for `list[list[...]]`) targets the array reached by + descending one extra level per bracket in the run, past the named + field, and duplicates the first element of the array it lands on. """ result = copy.deepcopy(row_dict) segments = coerce(path).segments - parent: Any = _walk_strict(result, path, segments[:-1]) - last = segments[-1] + # The terminal run is the last segment if it's a struct, or the maximal + # trailing run of ArraySegments (one named, then anonymous) otherwise -- + # `hierarchies[][]` parses to two segments but is one bracket run. + run_start = terminal_run_start(segments) + run_len = len(segments) - run_start + + parent: Any = _walk_strict(result, path, segments[:run_start]) + last = segments[run_start] if not isinstance(parent, dict) or last.name not in parent: raise PathTraversalError(f"Missing key '{last.name}' in path '{path}'") - # When the terminal is an array segment, descend `iter_count` levels of - # `[0]`. Otherwise the terminal struct already references the list to - # mutate. The final `container[key]` must itself be a list. + # When the terminal is an array segment, descend the run's depth in + # levels of `[0]`. Otherwise the terminal struct already references the + # list to mutate. The final `container[key]` must itself be a list. container: Any key: int | str if isinstance(last, ArraySegment): - container, key = _descend_iter_count( - parent[last.name], last.iter_count, last.name, path - ) + container, key = _descend_array_run(parent[last.name], run_len, last.name, path) else: container = parent key = last.name @@ -321,40 +451,47 @@ def _walk_strict( """Walk *path* without scaffolding, raising on missing or null nodes. Raises `PathTraversalError` on missing or null struct intermediates, - and on empty arrays encountered at array segments (each `[]` in a - segment's `iter_count` descends one element, which requires a - non-empty list). When *segments* is provided it overrides the - segments derived from *path*; *path* still labels error messages. + and on empty arrays encountered at array segments (each named + `ArraySegment` descends one element via a key lookup; each following + anonymous `ArraySegment` descends one more element with no lookup, + since the parent element is already the next list). When *segments* + is provided it overrides the segments derived from *path*; *path* + still labels error messages. """ if segments is None: segments = coerce(path).segments for segment in segments: + if isinstance(segment, ArraySegment) and segment.is_anonymous: + _require_non_empty_array(target, "[]", path) + target = target[0] + continue if not isinstance(target, dict) or target.get(segment.name) is None: raise PathTraversalError( f"Missing or null key '{segment.name}' in path '{path}'" ) target = target[segment.name] if isinstance(segment, ArraySegment): - container, key = _descend_iter_count( - target, segment.iter_count, segment.name, path - ) - target = container[key] + _require_non_empty_array(target, segment.name, path) + target = target[0] return target -def _descend_iter_count( - arr: list, iter_count: int, name: str, path: FieldPath | str +def _descend_array_run( + arr: list, count: int, name: str, path: FieldPath | str ) -> tuple[Any, int]: - """Descend *iter_count* levels into nested lists via element 0. - - Each level requires a non-empty list; the error label for depth `d` - is *name* followed by `d` `[]` markers. Returns the final - `(container, key)` write site so callers can read (`container[key]`) - or replace (`container[key] = ...`) the innermost element. + """Descend *count* levels into nested lists via element 0. + + *count* is 1 for a plain named array terminal, or 1 plus the number of + trailing anonymous segments for a multi-bracket terminal + (`hierarchies[][]`). Each level requires a non-empty list; the error + label for depth `d` is *name* followed by `d` `[]` markers. Returns the + final `(container, key)` write site so callers can read + (`container[key]`) or replace (`container[key] = ...`) the innermost + element. """ container: Any = [arr] key = 0 - for depth in range(iter_count): + for depth in range(count): inner = container[key] _require_non_empty_array(inner, f"{name}{'[]' * depth}", path) container, key = inner, 0 @@ -388,6 +525,7 @@ def _apply_to_targets( array_path: FieldPath | str | None, inner_array_path: FieldPath | str | None, map_path: FieldPath | str | None = None, + element_path: FieldPath | str | None = None, ) -> None: """Apply a mutation function to target dicts at the appropriate nesting level. @@ -396,11 +534,15 @@ def _apply_to_targets( `array_path` and `inner_array_path`, iterates over outer elements, navigates the inner struct path to a nested array, then iterates those elements. With `map_path`, applies to the value model of that map column - (stubbing one entry when the map is absent). + (stubbing one entry when the map is absent). With `element_path`, walks a + full mixed map/array descent (`_descend_to_targets`). Creates stub array elements when the arrays are null so the mutation can populate them. """ + if element_path is not None: + _descend_to_targets(row, coerce(element_path).segments, fn) + return if map_path is not None: fn(_map_value_to_mutate(row, map_path)) return @@ -468,15 +610,15 @@ def _ensure_condition( _set_nested(d, condition_field, condition_value) -def _as_scalar_path(path: FieldPath | str) -> ScalarPath: - """Coerce *path* to a `ScalarPath`, rejecting array or map markers. +def _as_direct_path(path: FieldPath | str) -> Direct: + """Coerce *path* to a `Direct`, rejecting array or map markers. The dict-walking helpers operate only on struct fields; an array or map-projection marker indicates the caller wanted array-/map-aware navigation and picked the wrong helper. """ coerced = coerce(path) - if not isinstance(coerced, ScalarPath): + if not isinstance(coerced, Direct): raise ValueError(f"struct-only path expected, got {coerced!r} for {path!r}") return coerced @@ -490,7 +632,7 @@ def _set_nested( None. When an intermediate is None and *value* is also None, the path is already effectively null — returns without error. """ - segments = _as_scalar_path(path).segments + segments = _as_direct_path(path).segments target = d for segment in segments[:-1]: part = segment.name @@ -511,7 +653,7 @@ def _get_nested(d: dict, path: FieldPath | str) -> object: Returns None when any intermediate key is missing or not a dict. """ target: object = d - for segment in _as_scalar_path(path).segments: + for segment in _as_direct_path(path).segments: if not isinstance(target, dict) or segment.name not in target: return None target = target[segment.name] diff --git a/packages/overture-schema-pyspark/tests/expressions/test_column_patterns.py b/packages/overture-schema-pyspark/tests/expressions/test_column_patterns.py index 027598b64..a43be04a6 100644 --- a/packages/overture-schema-pyspark/tests/expressions/test_column_patterns.py +++ b/packages/overture-schema-pyspark/tests/expressions/test_column_patterns.py @@ -1,5 +1,20 @@ -"""Tests for column_patterns — structural PySpark composition helpers.""" - +"""Tests for column_patterns -- structural PySpark composition helpers. + +Each helper composition is exercised as a `_Case`: a uniquely-named input +column, the check built over it, and a predicate on the collected result. The +`results` fixture packs every case's input into one wide single-row DataFrame, +applies every check in one `select`, and collects once -- so the whole file +pays for a single `createDataFrame` + `collect` instead of one pair per test +(the same batch-once pattern the generated conformance harness uses). Cases +needing both a violating and a clean input carry two entries (`*_invalid` / +`*_valid`). +""" + +from collections.abc import Callable +from dataclasses import dataclass +from typing import Any + +import pytest from overture.schema.pyspark.expressions.column_patterns import ( array_check, check_struct_unique, @@ -8,393 +23,419 @@ map_keys_check, map_values_check, nested_array_check, + nested_map_keys_check, + nested_map_values_check, ) from overture.schema.pyspark.expressions.constraint_expressions import ( + check_bounds, check_require_any_of, check_string_min_length, ) -from pyspark.sql import Row, SparkSession +from pyspark.sql import Column, SparkSession from pyspark.sql import functions as F - -def test_error_msg_concatenates(spark: SparkSession) -> None: - df = spark.createDataFrame([Row(val="bad")]) - result = df.select(error_msg("field: got ", F.col("val")).alias("msg")).collect() - assert result[0]["msg"] == "field: got bad" - - -def test_error_msg_multiple_values(spark: SparkSession) -> None: - df = spark.createDataFrame([Row(a="x", b="y")]) - result = df.select( - error_msg("prefix ", F.col("a"), F.lit(" and "), F.col("b")).alias("msg") - ).collect() - assert result[0]["msg"] == "prefix x and y" - - -def test_error_msg_null_value_does_not_nullify_message(spark: SparkSession) -> None: - # A NULL interpolated value must not make the whole message NULL: F.concat - # would, and a NULL message is dropped by array_compact, silently swallowing - # the violation (e.g. an out-of-bounds linear-reference range [null, 1.5]). - # The null must render as a literal instead. - df = spark.createDataFrame([Row(val=None)], schema="val double") - result = df.select(error_msg("got ", F.col("val")).alias("msg")).collect() - assert result[0]["msg"] == "got null" - - -def test_array_check_null_column_returns_null(spark: SparkSession) -> None: - df = spark.createDataFrame( - [Row(items=None)], - schema="items array>", - ) - result = df.select( - array_check("items", lambda el: F.lit("err")).alias("errs") - ).collect() - assert result[0]["errs"] is None - - -def test_array_check_filters_nulls(spark: SparkSession) -> None: - df = spark.createDataFrame( - [Row(items=[Row(val="ok"), Row(val="bad")])], - schema="items array>", - ) - result = df.select( - array_check( - "items", - lambda el: F.when(el["val"] == "bad", F.lit("error")), - ).alias("errs") - ).collect() - assert result[0]["errs"] == ["error"] +# PySpark 3.4's collect() leaves its result socket for the GC to finalize; under +# -W error that ResourceWarning fails the batched `results` fixture. conftest's +# unraisablehook catches the finalizer path, but this fixture emits it +# synchronously -- and a filterwarnings mark is the only filter outranking the +# command-line -W error. +pytestmark = pytest.mark.filterwarnings("ignore::ResourceWarning") -def test_array_check_empty_when_all_valid(spark: SparkSession) -> None: - df = spark.createDataFrame( - [Row(items=[Row(val="ok")])], - schema="items array>", - ) - result = df.select( - array_check( - "items", - lambda el: F.when(el["val"] == "bad", F.lit("error")), - ).alias("errs") - ).collect() - assert result[0]["errs"] == [] - - -def test_struct_unique_no_duplicates(spark: SparkSession) -> None: - df = spark.createDataFrame( - [Row(items=[Row(id="a"), Row(id="b")])], - schema="items array>", - ) - result = df.select(check_struct_unique("items").alias("err")).collect() - assert result[0]["err"] is None - - -def test_struct_unique_with_duplicates(spark: SparkSession) -> None: - df = spark.createDataFrame( - [Row(items=[Row(id="a"), Row(id="a")])], - schema="items array>", - ) - result = df.select(check_struct_unique("items").alias("err")).collect() - assert result[0]["err"] is not None - assert "duplicate" in result[0]["err"] - - -def test_struct_unique_null_column(spark: SparkSession) -> None: - df = spark.createDataFrame( - [Row(items=None)], - schema="items array>", - ) - result = df.select(check_struct_unique("items").alias("err")).collect() - assert result[0]["err"] is None +@dataclass(frozen=True) +class _Case: + """One column_patterns assertion driven off the shared wide row. + `check(field)` builds the composition over the case's input column; + `expect(result)` is a predicate on that column's single collected value. + """ -def test_struct_unique_repeated_value_different_fields(spark: SparkSession) -> None: - """Structs with same value subfield but different other fields are unique.""" - df = spark.createDataFrame( + id: str + ddl: str + value: Any + check: Callable[[str], Column] + expect: Callable[[Any], bool] + + +_CASES: list[_Case] = [ + # --- error_msg: builds a string message, not an error array --------------- + _Case( + "em_concat", + "string", + "bad", + lambda f: error_msg("field: got ", F.col(f)), + lambda r: r == "field: got bad", + ), + _Case( + "em_multi", + "struct", + {"a": "x", "b": "y"}, + lambda f: error_msg("prefix ", F.col(f)["a"], F.lit(" and "), F.col(f)["b"]), + lambda r: r == "prefix x and y", + ), + # A NULL interpolated value must not nullify the whole message (F.concat + # would): a NULL message is dropped by array_compact, silently swallowing the + # violation (e.g. an out-of-bounds range [null, 1.5]). It renders literally. + _Case( + "em_null", + "double", + None, + lambda f: error_msg("got ", F.col(f)), + lambda r: r == "got null", + ), + # --- array_check ---------------------------------------------------------- + _Case( + "ac_null", + "array>", + None, + lambda f: array_check(f, lambda el: F.lit("err")), + lambda r: r is None, + ), + _Case( + "ac_filter", + "array>", + [{"val": "ok"}, {"val": "bad"}], + lambda f: array_check(f, lambda el: F.when(el["val"] == "bad", F.lit("error"))), + lambda r: r == ["error"], + ), + _Case( + "ac_empty", + "array>", + [{"val": "ok"}], + lambda f: array_check(f, lambda el: F.when(el["val"] == "bad", F.lit("error"))), + lambda r: r == [], + ), + # array_check / check_struct_unique also accept a Column, not just a name. + _Case( + "ac_col", + "array>", + [{"val": "ok"}, {"val": "bad"}], + lambda f: array_check( + F.col(f), lambda el: F.when(el["val"] == "bad", F.lit("error")) + ), + lambda r: r == ["error"], + ), + # --- check_struct_unique -------------------------------------------------- + _Case( + "su_nodup", + "array>", + [{"id": "a"}, {"id": "b"}], + lambda f: check_struct_unique(f), + lambda r: r is None, + ), + _Case( + "su_dup", + "array>", + [{"id": "a"}, {"id": "a"}], + lambda f: check_struct_unique(f), + lambda r: r is not None and "duplicate" in r, + ), + _Case( + "su_null", + "array>", + None, + lambda f: check_struct_unique(f), + lambda r: r is None, + ), + # Same value subfield but different other fields is not a duplicate. + _Case( + "su_repeat", + "array>", [ - Row( - items=[ - Row(value="a", pos=0.0), - Row(value="b", pos=0.5), - Row(value="a", pos=0.7), - ] + {"value": "a", "pos": 0.0}, + {"value": "b", "pos": 0.5}, + {"value": "a", "pos": 0.7}, + ], + lambda f: check_struct_unique(f), + lambda r: r is None, + ), + _Case( + "su_single", + "array>", + [{"id": "a"}], + lambda f: check_struct_unique(f), + lambda r: r is None, + ), + _Case( + "csu_col", + "array>", + [{"id": "a"}, {"id": "a"}], + lambda f: check_struct_unique(F.col(f)), + lambda r: r is not None and "duplicate" in r, + ), + _Case( + "csu_colnull", + "array>", + None, + lambda f: check_struct_unique(F.col(f)), + lambda r: r is None, + ), + # --- nested_array_check --------------------------------------------------- + _Case( + "na_flat", + "array>>", + [{"tags": ["good", "bad"]}, {"tags": ["worse"]}], + lambda f: coalesce_errors( + nested_array_check( + f, + lambda el: array_check( + el["tags"], + lambda tag: F.when(tag != "good", F.concat(F.lit("bad: "), tag)), + ), ) - ] - ) - result = df.select(check_struct_unique("items").alias("err")).collect() - assert result[0]["err"] is None - - -def test_struct_unique_single_element(spark: SparkSession) -> None: - df = spark.createDataFrame( - [Row(items=[Row(id="a")])], - schema="items array>", - ) - result = df.select(check_struct_unique("items").alias("err")).collect() - assert result[0]["err"] is None - - -def test_array_check_accepts_column(spark: SparkSession) -> None: - """array_check works when passed a Column instead of a string name.""" - df = spark.createDataFrame( - [Row(items=[Row(val="ok"), Row(val="bad")])], - schema="items array>", - ) - result = df.select( - array_check( - F.col("items"), - lambda el: F.when(el["val"] == "bad", F.lit("error")), - ).alias("errs") - ).collect() - assert result[0]["errs"] == ["error"] - - -def test_check_struct_unique_accepts_column(spark: SparkSession) -> None: - """check_struct_unique works when passed a Column instead of a string name.""" - df = spark.createDataFrame( - [Row(items=[Row(id="a"), Row(id="a")])], - schema="items array>", - ) - result = df.select(check_struct_unique(F.col("items")).alias("err")).collect() - assert result[0]["err"] is not None - assert "duplicate" in result[0]["err"] - - -def test_check_struct_unique_column_null(spark: SparkSession) -> None: - """check_struct_unique with Column input handles null.""" - df = spark.createDataFrame( - [Row(items=None)], schema="items array>" - ) - result = df.select(check_struct_unique(F.col("items")).alias("err")).collect() - assert result[0]["err"] is None - - -def test_nested_array_check_flattens(spark: SparkSession) -> None: - """Inner array_check per outer element produces flat error list.""" - schema = "items array>>" - df = spark.createDataFrame( - [ - Row( - items=[ - Row(tags=["good", "bad"]), - Row(tags=["worse"]), - ] + ), + lambda r: len(r) == 2 and all(isinstance(e, str) for e in r), + ), + _Case( + "na_null", + "array>>", + None, + lambda f: coalesce_errors( + nested_array_check( + f, + lambda el: array_check( + el["tags"], lambda tag: F.when(tag != "good", F.lit("bad")) + ), ) - ], - schema=schema, - ) - result_col = nested_array_check( - "items", - lambda el: array_check( - el["tags"], - lambda tag: F.when(tag != "good", F.concat(F.lit("bad: "), tag)), ), - ) - result = df.select(coalesce_errors(result_col).alias("errs")).collect() - errors = result[0]["errs"] - assert len(errors) == 2 - assert all(isinstance(e, str) for e in errors) - - -def test_nested_array_check_null_outer(spark: SparkSession) -> None: - schema = "items array>>" - df = spark.createDataFrame([Row(items=None)], schema=schema) - result_col = nested_array_check( - "items", - lambda el: array_check( - el["tags"], - lambda tag: F.when(tag != "good", F.lit("bad")), + lambda r: r == [], + ), + # A null inner array must not nullify sibling errors during flatten: + # F.flatten returns NULL whenever any sub-array is NULL, which would drop + # every sibling error unless inner nulls are guarded. + _Case( + "na_mixed", + "array>>", + [{"tags": ["good"]}, {"tags": None}, {"tags": ["bad"]}], + lambda f: coalesce_errors( + nested_array_check( + f, + lambda el: array_check( + el["tags"], + lambda tag: F.when(tag != "good", F.concat(F.lit("bad: "), tag)), + ), + ) ), - ) - result = df.select(coalesce_errors(result_col).alias("errs")).collect() - assert result[0]["errs"] == [] - - -def test_nested_array_check_mixed_null_inner_with_sibling_errors( - spark: SparkSession, -) -> None: - """A null inner array must not nullify sibling errors during flatten. - - `F.flatten` returns NULL whenever any sub-array is NULL. Without - guarding inner nulls, the outer transform produces NULL and every - sibling error is silently dropped. - """ - schema = "items array>>" - df = spark.createDataFrame( - [ - Row( - items=[ - Row(tags=["good"]), - Row(tags=None), - Row(tags=["bad"]), - ] + lambda r: r == ["bad: bad"], + ), + _Case( + "na_noerr", + "array>>", + [{"tags": ["good"]}], + lambda f: coalesce_errors( + nested_array_check( + f, + lambda el: array_check( + el["tags"], lambda tag: F.when(tag != "good", F.lit("bad")) + ), ) - ], - schema=schema, - ) - result_col = nested_array_check( - "items", - lambda el: array_check( - el["tags"], - lambda tag: F.when(tag != "good", F.concat(F.lit("bad: "), tag)), ), - ) - result = df.select(coalesce_errors(result_col).alias("errs")).collect() - assert result[0]["errs"] == ["bad: bad"] - - -def test_nested_array_check_no_errors(spark: SparkSession) -> None: - schema = "items array>>" - df = spark.createDataFrame( - [Row(items=[Row(tags=["good"])])], - schema=schema, - ) - result_col = nested_array_check( - "items", - lambda el: array_check( - el["tags"], - lambda tag: F.when(tag != "good", F.lit("bad")), + lambda r: r == [], + ), + # --- map_keys_check / map_values_check ------------------------------------ + _Case( + "mk_bad", + "map", + {"good": "v", "bad": "v"}, + lambda f: map_keys_check(f, lambda k: F.when(k == "bad", F.lit("bad key"))), + lambda r: r == ["bad key"], + ), + _Case( + "mv_bad", + "map", + {"a": "ok", "b": "bad"}, + lambda f: map_values_check(f, lambda v: F.when(v == "bad", F.lit("bad value"))), + lambda r: r == ["bad value"], + ), + # A field check on a dict[str, Model] value navigates into the value struct + # -- the exact composition the renderer emits for a map-of-model value field. + _Case( + "mv_struct", + "map>", + {"a": {"label": ""}}, + lambda f: map_values_check(f, lambda v: check_string_min_length(v["label"], 1)), + lambda r: r == ["minimum length 1, got 0"], + ), + _Case( + "mv_struct_ok", + "map>", + {"a": {"label": "ok"}}, + lambda f: map_values_check(f, lambda v: check_string_min_length(v["label"], 1)), + lambda r: r == [], + ), + # A model-level constraint on a dict[str, Model] value -- the composition the + # renderer emits for a map-of-model value-model constraint. + _Case( + "mv_model", + "map>", + {"a": {"foo": None, "bar": None}}, + lambda f: map_values_check( + f, lambda v: check_require_any_of([v["foo"], v["bar"]], ["foo", "bar"]) ), - ) - result = df.select(coalesce_errors(result_col).alias("errs")).collect() - assert result[0]["errs"] == [] - - -def test_map_keys_check_flags_bad_key(spark: SparkSession) -> None: - df = spark.createDataFrame( - [Row(tags={"good": "v", "bad": "v"})], - schema="tags map", - ) - result = df.select( - map_keys_check("tags", lambda k: F.when(k == "bad", F.lit("bad key"))).alias( - "errs" - ) - ).collect() - assert result[0]["errs"] == ["bad key"] - - -def test_map_values_check_flags_bad_value(spark: SparkSession) -> None: - df = spark.createDataFrame( - [Row(tags={"a": "ok", "b": "bad"})], - schema="tags map", - ) - result = df.select( - map_values_check( - "tags", lambda v: F.when(v == "bad", F.lit("bad value")) - ).alias("errs") - ).collect() - assert result[0]["errs"] == ["bad value"] - - -def test_map_values_check_descends_into_value_struct_field( - spark: SparkSession, -) -> None: - # A field check on a `dict[str, Model]` value navigates into the value - # struct: map_values_check over a struct-navigating lambda, the exact - # composition the renderer emits for a map-of-model value field. - df = spark.createDataFrame( - [Row(items={"a": Row(label="")})], - schema="items map>", - ) - result = df.select( - map_values_check( - "items", lambda v: check_string_min_length(v["label"], 1) - ).alias("errs") - ).collect() - assert result[0]["errs"] == ["minimum length 1, got 0"] - - -def test_map_values_check_passes_valid_value_struct_field( - spark: SparkSession, -) -> None: - df = spark.createDataFrame( - [Row(items={"a": Row(label="ok")})], - schema="items map>", - ) - result = df.select( - map_values_check( - "items", lambda v: check_string_min_length(v["label"], 1) - ).alias("errs") - ).collect() - assert result[0]["errs"] == [] - - -def test_map_values_check_enforces_value_model_constraint( - spark: SparkSession, -) -> None: - # A model-level constraint on a `dict[str, Model]` value: map_values_check - # wrapping check_require_any_of over the value struct's fields, the exact - # composition the renderer emits for a map-of-model value-model constraint. - df = spark.createDataFrame( - [Row(subs={"a": Row(foo=None, bar=None)})], - schema="subs map>", - ) - result = df.select( - map_values_check( - "subs", - lambda v: check_require_any_of([v["foo"], v["bar"]], ["foo", "bar"]), - ).alias("errs") - ).collect() - assert result[0]["errs"] == ["requires at least one of foo, bar"] - - -def test_map_values_check_passes_satisfied_value_model_constraint( - spark: SparkSession, -) -> None: - df = spark.createDataFrame( - [Row(subs={"a": Row(foo=1, bar=None)})], - schema="subs map>", - ) - result = df.select( - map_values_check( - "subs", - lambda v: check_require_any_of([v["foo"], v["bar"]], ["foo", "bar"]), - ).alias("errs") - ).collect() - assert result[0]["errs"] == [] - - -def test_map_keys_check_null_column_returns_null(spark: SparkSession) -> None: - df = spark.createDataFrame([Row(tags=None)], schema="tags map") - result = df.select( - map_keys_check("tags", lambda k: F.lit("err")).alias("errs") - ).collect() - assert result[0]["errs"] is None - - -def test_map_values_check_all_valid_empty(spark: SparkSession) -> None: - df = spark.createDataFrame( - [Row(tags={"a": "ok"})], schema="tags map" - ) - result = df.select( - map_values_check("tags", lambda v: F.when(v == "bad", F.lit("err"))).alias( - "errs" - ) - ).collect() - assert result[0]["errs"] == [] - - -def test_map_keys_check_accepts_column(spark: SparkSession) -> None: - df = spark.createDataFrame( - [Row(tags={"bad": "v"})], schema="tags map" - ) - result = df.select( - map_keys_check(F.col("tags"), lambda k: F.when(k == "bad", F.lit("err"))).alias( - "errs" - ) - ).collect() - assert result[0]["errs"] == ["err"] - - -def test_coalesce_errors_null_becomes_empty(spark: SparkSession) -> None: - df = spark.createDataFrame([Row(x=1)]) - result = df.select( - coalesce_errors(F.lit(None).cast("array")).alias("errs") - ).collect() - assert result[0]["errs"] == [] - - -def test_coalesce_errors_preserves_array(spark: SparkSession) -> None: - df = spark.createDataFrame([Row(x=1)]) - result = df.select(coalesce_errors(F.array(F.lit("err"))).alias("errs")).collect() - assert result[0]["errs"] == ["err"] + lambda r: r == ["requires at least one of foo, bar"], + ), + _Case( + "mv_model_ok", + "map>", + {"a": {"foo": 1, "bar": None}}, + lambda f: map_values_check( + f, lambda v: check_require_any_of([v["foo"], v["bar"]], ["foo", "bar"]) + ), + lambda r: r == [], + ), + _Case( + "mk_null", + "map", + None, + lambda f: map_keys_check(f, lambda k: F.lit("err")), + lambda r: r is None, + ), + _Case( + "mv_valid", + "map", + {"a": "ok"}, + lambda f: map_values_check(f, lambda v: F.when(v == "bad", F.lit("err"))), + lambda r: r == [], + ), + _Case( + "mk_col", + "map", + {"bad": "v"}, + lambda f: map_keys_check(F.col(f), lambda k: F.when(k == "bad", F.lit("err"))), + lambda r: r == ["err"], + ), + # --- coalesce_errors (input-independent literals) ------------------------- + _Case( + "coal_null", + "int", + 1, + lambda f: coalesce_errors(F.lit(None).cast("array")), + lambda r: r == [], + ), + _Case( + "coal_array", + "int", + 1, + lambda f: coalesce_errors(F.array(F.lit("err"))), + lambda r: r == ["err"], + ), + # --- nested_map_{values,keys}_check flatten an inner array -------- + _Case( + "nmv_flat_invalid", + "map>", + {"a": [1, 0], "b": [2]}, + lambda f: nested_map_values_check( + f, + lambda v: array_check( + v, lambda e: F.when(e == 0, F.lit("zero not allowed")) + ), + ), + lambda r: r == ["zero not allowed"], + ), + _Case( + "nmv_flat_valid", + "map>", + {"a": [1], "b": [3]}, + lambda f: nested_map_values_check( + f, + lambda v: array_check( + v, lambda e: F.when(e == 0, F.lit("zero not allowed")) + ), + ), + lambda r: r == [], + ), + _Case( + "nmk_flat_invalid", + "map", + {"ax": 1, "b": 2}, + lambda f: nested_map_keys_check( + f, + lambda k: array_check( + F.split(k, ""), lambda ch: F.when(ch == "x", F.lit("x not allowed")) + ), + ), + lambda r: r == ["x not allowed"], + ), + _Case( + "nmk_flat_valid", + "map", + {"cd": 1}, + lambda f: nested_map_keys_check( + f, + lambda k: array_check( + F.split(k, ""), lambda ch: F.when(ch == "x", F.lit("x not allowed")) + ), + ), + lambda r: r == [], + ), + # --- flattening helper wrapped around map_values_check -------------------- + # `items[].tags{value}` and `subs{value}{value}`: the pairings the generated + # nested_array_check / nested_map_values_check fold around map_values_check. + _Case( + "nawmv_invalid", + "array>>", + [{"tags": {"k": "ab"}}], + lambda f: nested_array_check( + f, + lambda el: map_values_check( + el["tags"], lambda v: check_string_min_length(v, 3) + ), + ), + lambda r: r != [], + ), + _Case( + "nawmv_valid", + "array>>", + [{"tags": {"k": "abc"}}, {"tags": {"k2": "wxyz"}}], + lambda f: nested_array_check( + f, + lambda el: map_values_check( + el["tags"], lambda v: check_string_min_length(v, 3) + ), + ), + lambda r: r == [], + ), + _Case( + "nmvwmv_invalid", + "map>", + {"k": {"a": -2}}, + lambda f: nested_map_values_check( + f, + lambda v: map_values_check( + v, lambda w: check_bounds(w, ge=0, check_nan=False) + ), + ), + lambda r: r != [], + ), + _Case( + "nmvwmv_valid", + "map>", + {"k": {"a": 0, "b": 1}}, + lambda f: nested_map_values_check( + f, + lambda v: map_values_check( + v, lambda w: check_bounds(w, ge=0, check_nan=False) + ), + ), + lambda r: r == [], + ), +] + + +@pytest.fixture(scope="module") +def results(spark: SparkSession) -> Any: + """Pack every case's input into one row, apply every check, collect once.""" + # A DDL schema string, not StructType.fromDDL -- fromDDL landed in PySpark + # 3.5, and createDataFrame parses the string itself on the >=3.4 floor. + schema = ", ".join(f"`{c.id}` {c.ddl}" for c in _CASES) + row = {c.id: c.value for c in _CASES} + # dict rows are read by field name against the explicit schema, a form the + # createDataFrame stubs don't model (they want tuple/Row for RowLike). + df = spark.createDataFrame([row], schema=schema, verifySchema=False) # type: ignore[call-overload] + return df.select(*[c.check(c.id).alias(c.id) for c in _CASES]).collect()[0] + + +@pytest.mark.parametrize("case", _CASES, ids=lambda c: c.id) +def test_column_pattern(case: _Case, results: Any) -> None: + value = results[case.id] + assert case.expect(value), f"{case.id}: got {value!r}" diff --git a/packages/overture-schema-pyspark/tests/expressions/test_constraint_expressions.py b/packages/overture-schema-pyspark/tests/expressions/test_constraint_expressions.py index d62936c30..26a29663b 100644 --- a/packages/overture-schema-pyspark/tests/expressions/test_constraint_expressions.py +++ b/packages/overture-schema-pyspark/tests/expressions/test_constraint_expressions.py @@ -1,6 +1,23 @@ -"""Tests for constraint_expressions — constraint type to Column translation.""" +"""Tests for constraint_expressions -- constraint type to Column translation. + +Each constraint is exercised as a `_Case`: a uniquely-named input column, the +check built over it, and a predicate on the collected result. The `results` +fixture packs every case's input into one wide single-row DataFrame, applies +every check in one `select`, and collects once -- so the whole file pays for a +single `createDataFrame` + `collect` instead of one pair per test (the same +batch-once pattern the generated conformance harness uses). Cases needing both +a violating and a clean input (or a valid/invalid pair of geometries) carry +separate entries. + +Column DDL types are load-bearing: `double` vs `int` changes NaN behavior, and +multi-field checks (require_if / forbid_if / radio_group / min_fields_set) pack +their inputs into a struct column so the check can reference each field. +""" import struct +from collections.abc import Callable +from dataclasses import dataclass +from typing import Any import pytest from overture.schema.pyspark.expressions.constraint_expressions import ( @@ -32,1561 +49,1473 @@ except_literals, ) from overture.schema.system.geometric import GeometryType -from pyspark.sql import Row, SparkSession +from pyspark.sql import Column, SparkSession from pyspark.sql import functions as F -from pyspark.sql.types import DoubleType, StringType, StructField, StructType from shapely.geometry import LineString, MultiPolygon, Point, Polygon +# PySpark 3.4's collect() leaves its result socket for the GC to finalize; under +# -W error that ResourceWarning fails the batched `results` fixture. conftest's +# unraisablehook catches the finalizer path, but this fixture emits it +# synchronously -- and a filterwarnings mark is the only filter outranking the +# command-line -W error. +pytestmark = pytest.mark.filterwarnings("ignore::ResourceWarning") -def _except_literals_error(spark: SparkSession, value: str | None) -> str | None: - """Run `except_literals` over `check_url_format` for one string value.""" - df = spark.createDataFrame( - [Row(val=value)], schema=StructType([StructField("val", StringType(), True)]) - ) - col = F.col("val") - expr = except_literals(col, check_url_format(col), [""]) - # Spark Row field access is untyped (Any); the column holds an error string. - return df.select(expr.alias("err")).collect()[0]["err"] # type: ignore[no-any-return] - - -def test_except_literals_suppresses_allowed_literal(spark: SparkSession) -> None: - # "" is an allowed literal alternative -> the url_format error is suppressed. - assert _except_literals_error(spark, "") is None - - -def test_except_literals_passes_through_real_violation(spark: SparkSession) -> None: - # A non-literal invalid value still surfaces the inner check's error. - assert _except_literals_error(spark, "not a url") is not None - - -def test_except_literals_passes_through_valid_value(spark: SparkSession) -> None: - assert _except_literals_error(spark, "https://example.com/x") is None - - -def test_except_literals_null_is_not_an_error(spark: SparkSession) -> None: - assert _except_literals_error(spark, None) is None - - -def test_check_multiple_of_integral_float_passes(spark: SparkSession) -> None: - df = spark.createDataFrame([Row(val=2.0)], schema="val double") - result = df.select(check_multiple_of(F.col("val"), 1).alias("err")).collect() - assert result[0]["err"] is None - - -def test_check_multiple_of_negative_integral_float_passes(spark: SparkSession) -> None: - df = spark.createDataFrame([Row(val=-3.0)], schema="val double") - result = df.select(check_multiple_of(F.col("val"), 1).alias("err")).collect() - assert result[0]["err"] is None - - -def test_check_multiple_of_fractional_float_violation(spark: SparkSession) -> None: - df = spark.createDataFrame([Row(val=2.5)], schema="val double") - result = df.select(check_multiple_of(F.col("val"), 1).alias("err")).collect() - assert result[0]["err"] is not None - assert "multiple of" in result[0]["err"] - - -def test_check_multiple_of_null_passthrough(spark: SparkSession) -> None: - df = spark.createDataFrame([Row(val=None)], schema="val double") - result = df.select(check_multiple_of(F.col("val"), 1).alias("err")).collect() - assert result[0]["err"] is None - - -def test_check_multiple_of_large_integral_double_passes(spark: SparkSession) -> None: - # Integral doubles beyond 2^63 are whole numbers Pydantic accepts - # ((1e30).is_integer() is True). A floor-based check would saturate the - # LongType cast and wrongly fire; the remainder check passes them. - df = spark.createDataFrame([Row(val=1e30)], schema="val double") - result = df.select(check_multiple_of(F.col("val"), 1).alias("err")).collect() - assert result[0]["err"] is None - - -def test_check_multiple_of_nan_violation(spark: SparkSession) -> None: - df = spark.createDataFrame([Row(val=float("nan"))], schema="val double") - result = df.select(check_multiple_of(F.col("val"), 1).alias("err")).collect() - assert result[0]["err"] is not None - assert "multiple of" in result[0]["err"] - - -def test_check_multiple_of_positive_infinity_violation(spark: SparkSession) -> None: - df = spark.createDataFrame([Row(val=float("inf"))], schema="val double") - result = df.select(check_multiple_of(F.col("val"), 1).alias("err")).collect() - assert result[0]["err"] is not None - assert "multiple of" in result[0]["err"] - - -def test_check_multiple_of_negative_infinity_violation(spark: SparkSession) -> None: - df = spark.createDataFrame([Row(val=float("-inf"))], schema="val double") - result = df.select(check_multiple_of(F.col("val"), 1).alias("err")).collect() - assert result[0]["err"] is not None - assert "multiple of" in result[0]["err"] - - -def test_check_multiple_of_non_unit_divisor(spark: SparkSession) -> None: - # Divisor need not be 1: 1.5 is a multiple of 0.5, 1.75 is not. - df = spark.createDataFrame([Row(val=1.5), Row(val=1.75)], schema="val double") - result = df.select(check_multiple_of(F.col("val"), 0.5).alias("err")).collect() - assert result[0]["err"] is None - assert result[1]["err"] is not None - - -def test_check_bounds_ge_le_valid(spark: SparkSession) -> None: - df = spark.createDataFrame([Row(val=5)]) - result = df.select(check_bounds(F.col("val"), ge=1, le=10).alias("err")).collect() - assert result[0]["err"] is None - - -def test_check_bounds_ge_violation(spark: SparkSession) -> None: - df = spark.createDataFrame([Row(val=0)]) - result = df.select(check_bounds(F.col("val"), ge=1).alias("err")).collect() - assert result[0]["err"] is not None - assert ">= 1" in result[0]["err"] - - -def test_check_bounds_gt_violation(spark: SparkSession) -> None: - df = spark.createDataFrame([Row(val=0)]) - result = df.select(check_bounds(F.col("val"), gt=0).alias("err")).collect() - assert result[0]["err"] is not None - assert "> 0" in result[0]["err"] - - -def test_check_bounds_le_violation(spark: SparkSession) -> None: - df = spark.createDataFrame([Row(val=100)]) - result = df.select(check_bounds(F.col("val"), le=50).alias("err")).collect() - assert result[0]["err"] is not None - - -def test_check_bounds_null_passthrough(spark: SparkSession) -> None: - df = spark.createDataFrame([Row(val=None)], schema="val int") - result = df.select(check_bounds(F.col("val"), ge=1).alias("err")).collect() - assert result[0]["err"] is None - - -def test_check_bounds_nan_lower_bound_violation(spark: SparkSession) -> None: - """NaN satisfies no Pydantic bound, but Spark sorts NaN above all values, - so a lower bound (NaN < v) never fires. check_bounds must reject it.""" - df = spark.createDataFrame([Row(val=float("nan"))], schema="val double") - result = df.select(check_bounds(F.col("val"), ge=0).alias("err")).collect() - assert result[0]["err"] is not None - assert "NaN" in result[0]["err"] - - -def test_check_bounds_nan_gt_violation(spark: SparkSession) -> None: - """Same lower-bound leak as ge, via the strict-greater comparison.""" - df = spark.createDataFrame([Row(val=float("nan"))], schema="val double") - result = df.select(check_bounds(F.col("val"), gt=0).alias("err")).collect() - assert result[0]["err"] is not None - assert "NaN" in result[0]["err"] - - -def test_check_bounds_nan_upper_bound_violation(spark: SparkSession) -> None: - """An upper bound already rejects NaN in Spark (NaN > v is true); the - explicit NaN check keeps that behavior.""" - df = spark.createDataFrame([Row(val=float("nan"))], schema="val double") - result = df.select(check_bounds(F.col("val"), le=1).alias("err")).collect() - assert result[0]["err"] is not None - - -def test_check_bounds_nan_no_bounds_passes(spark: SparkSession) -> None: - """With no bounds there is nothing to violate; NaN passes, matching - Pydantic's allow_inf_nan default for unconstrained floats.""" - df = spark.createDataFrame([Row(val=float("nan"))], schema="val double") - result = df.select(check_bounds(F.col("val")).alias("err")).collect() - assert result[0]["err"] is None - - -def test_check_bounds_valid_float_passes(spark: SparkSession) -> None: - """A finite in-range float is unaffected by the NaN guard.""" - df = spark.createDataFrame([Row(val=0.5)], schema="val double") - result = df.select(check_bounds(F.col("val"), ge=0, le=1).alias("err")).collect() - assert result[0]["err"] is None - - -def test_check_bounds_nan_guard_off_passes_nan(spark: SparkSession) -> None: - """With check_nan=False the NaN guard is absent; NaN slips past a lower bound.""" - df = spark.createDataFrame([Row(val=float("nan"))], schema="val double") - result = df.select( - check_bounds(F.col("val"), ge=0, check_nan=False).alias("err") - ).collect() - assert result[0]["err"] is None - - -def test_check_bounds_nan_guard_on_rejects_nan(spark: SparkSession) -> None: - """With check_nan=True (default) NaN is rejected even with a lower bound.""" - df = spark.createDataFrame([Row(val=float("nan"))], schema="val double") - result = df.select( - check_bounds(F.col("val"), ge=0, check_nan=True).alias("err") - ).collect() - assert result[0]["err"] is not None - assert "NaN" in result[0]["err"] - - -def test_check_bounds_integer_column_rejects_violation(spark: SparkSession) -> None: - """check_nan=False is safe for integer columns; bound violations still fire.""" - df = spark.createDataFrame([Row(val=0)], schema="val int") - result = df.select( - check_bounds(F.col("val"), ge=1, check_nan=False).alias("err") - ).collect() - assert result[0]["err"] is not None - - -def test_check_bounds_integer_column_accepts_valid(spark: SparkSession) -> None: - """check_nan=False on an integer column: in-bound values pass.""" - df = spark.createDataFrame([Row(val=5)], schema="val int") - result = df.select( - check_bounds(F.col("val"), ge=1, le=10, check_nan=False).alias("err") - ).collect() - assert result[0]["err"] is None - - -def test_check_enum_valid(spark: SparkSession) -> None: - df = spark.createDataFrame([Row(val="road")]) - result = df.select( - check_enum(F.col("val"), ["road", "rail", "water"]).alias("err") - ).collect() - assert result[0]["err"] is None - - -def test_check_enum_invalid(spark: SparkSession) -> None: - df = spark.createDataFrame([Row(val="sky")]) - result = df.select( - check_enum(F.col("val"), ["road", "rail", "water"]).alias("err") - ).collect() - assert result[0]["err"] is not None - assert "sky" in result[0]["err"] - - -class TestCheckPattern: - def test_valid(self, spark: SparkSession) -> None: - df = spark.createDataFrame([("AB",)], ["v"]) - result = df.select( - check_pattern(F.col("v"), r"^[A-Z]{2}$", label="test pattern").alias("e") - ) - assert result.collect()[0]["e"] is None - - def test_invalid(self, spark: SparkSession) -> None: - df = spark.createDataFrame([("abc",)], ["v"]) - result = df.select( - check_pattern(F.col("v"), r"^[A-Z]{2}$", label="test pattern").alias("e") - ) - err = result.collect()[0]["e"] - assert "invalid test pattern" in err - assert "abc" in err - - def test_null_passes(self, spark: SparkSession) -> None: - df = spark.createDataFrame([(None,)], schema="v string") - result = df.select( - check_pattern(F.col("v"), r"^[A-Z]{2}$", label="test pattern").alias("e") - ) - assert result.collect()[0]["e"] is None - - -class TestCheckMinLength: - def test_at_limit(self, spark: SparkSession) -> None: - df = spark.createDataFrame( - [Row(items=["a", "b"])], schema="items array" - ) - result = df.select( - check_array_min_length(F.col("items"), 2).alias("err") - ).collect() - assert result[0]["err"] is None - - def test_below_limit(self, spark: SparkSession) -> None: - df = spark.createDataFrame([Row(items=["a"])], schema="items array") - result = df.select( - check_array_min_length(F.col("items"), 2).alias("err") - ).collect() - assert result[0]["err"] is not None - assert "minimum length 2" in result[0]["err"] - - def test_null_passthrough(self, spark: SparkSession) -> None: - df = spark.createDataFrame([Row(items=None)], schema="items array") - result = df.select( - check_array_min_length(F.col("items"), 2).alias("err") - ).collect() - assert result[0]["err"] is None - - -class TestCheckMaxLength: - def test_within_limit(self, spark: SparkSession) -> None: - df = spark.createDataFrame( - [Row(items=["a", "b"])], schema="items array" - ) - result = df.select( - check_array_max_length(F.col("items"), 3).alias("err") - ).collect() - assert result[0]["err"] is None - - def test_at_limit(self, spark: SparkSession) -> None: - df = spark.createDataFrame( - [Row(items=["a", "b"])], schema="items array" - ) - result = df.select( - check_array_max_length(F.col("items"), 2).alias("err") - ).collect() - assert result[0]["err"] is None - - def test_exceeds_limit(self, spark: SparkSession) -> None: - df = spark.createDataFrame( - [Row(items=["a", "b", "c"])], schema="items array" - ) - result = df.select( - check_array_max_length(F.col("items"), 2).alias("err") - ).collect() - assert result[0]["err"] is not None - assert "maximum length 2" in result[0]["err"] - - def test_null_passthrough(self, spark: SparkSession) -> None: - df = spark.createDataFrame([Row(items=None)], schema="items array") - result = df.select( - check_array_max_length(F.col("items"), 2).alias("err") - ).collect() - assert result[0]["err"] is None - - -def test_check_require_any_of_satisfied(spark: SparkSession) -> None: - df = spark.createDataFrame([Row(a=1, b=None)], schema="a int, b int") - result = df.select( - check_require_any_of([F.col("a"), F.col("b")], ["a", "b"]).alias("err") - ).collect() - assert result[0]["err"] is None - - -def test_check_require_any_of_all_null(spark: SparkSession) -> None: - df = spark.createDataFrame([Row(a=None, b=None)], schema="a int, b int") - result = df.select( - check_require_any_of([F.col("a"), F.col("b")], ["a", "b"]).alias("err") - ).collect() - assert result[0]["err"] is not None - assert "a" in result[0]["err"] - assert "b" in result[0]["err"] - - -class TestCheckRequireAnyTrue: - _NAMES = ["is_land", "is_territorial"] - - def _conds(self) -> list: - return [ - F.col("is_land") == F.lit(True), - F.col("is_territorial") == F.lit(True), - ] - - def test_one_condition_true(self, spark: SparkSession) -> None: - """At least one condition true -> no error.""" - df = spark.createDataFrame( - [Row(is_land=True, is_territorial=False)], - schema="is_land boolean, is_territorial boolean", - ) - result = df.select( - check_require_any_true(self._conds(), self._NAMES).alias("err") - ).collect() - assert result[0]["err"] is None - - def test_all_conditions_false(self, spark: SparkSession) -> None: - """No condition true -> error naming the fields.""" - df = spark.createDataFrame( - [Row(is_land=False, is_territorial=False)], - schema="is_land boolean, is_territorial boolean", - ) - result = df.select( - check_require_any_true(self._conds(), self._NAMES).alias("err") - ).collect() - assert result[0]["err"] is not None - assert "is_land" in result[0]["err"] - assert "is_territorial" in result[0]["err"] - - def test_all_conditions_null_is_violation(self, spark: SparkSession) -> None: - """Null fields -> conditions not true -> error. - - Mirrors Python's `None == True` -> `False`: a null column value - does not satisfy the condition, so an all-null row violates. - """ - df = spark.createDataFrame( - [Row(is_land=None, is_territorial=None)], - schema="is_land boolean, is_territorial boolean", - ) - result = df.select( - check_require_any_true(self._conds(), self._NAMES).alias("err") - ).collect() - assert result[0]["err"] is not None - - -class TestCheckRequireIf: - def test_required_present(self, spark: SparkSession) -> None: - """Target is present when condition is true -> no error.""" - df = spark.createDataFrame( - [("road", "primary")], schema="subtype string, road_class string" - ) - result = df.select( - check_require_if( - F.col("road_class"), - F.col("subtype").isin(["road", "rail"]), - "subtype in [road, rail]", - ).alias("err") - ).collect() - assert result[0]["err"] is None - - def test_required_absent(self, spark: SparkSession) -> None: - """Target is null when condition is true -> error.""" - df = spark.createDataFrame( - [("road", None)], schema="subtype string, road_class string" - ) - result = df.select( - check_require_if( - F.col("road_class"), - F.col("subtype").isin(["road", "rail"]), - "subtype in [road, rail]", - ).alias("err") - ).collect() - assert result[0]["err"] is not None - assert "required" in result[0]["err"] - - def test_condition_false_skips(self, spark: SparkSession) -> None: - """Target is null but condition is false -> no error.""" - df = spark.createDataFrame( - [("water", None)], schema="subtype string, road_class string" - ) - result = df.select( - check_require_if( - F.col("road_class"), - F.col("subtype").isin(["road", "rail"]), - "subtype in [road, rail]", - ).alias("err") - ).collect() - assert result[0]["err"] is None - - def test_with_value_cols(self, spark: SparkSession) -> None: - """Error message includes actual discriminator value.""" - df = spark.createDataFrame( - [("road", None)], schema="subtype string, road_class string" - ) - result = df.select( - check_require_if( - F.col("road_class"), - F.col("subtype").isin(["road", "rail"]), - "subtype in [road, rail]", - F.col("subtype"), - ).alias("err") - ).collect() - assert "road" in result[0]["err"] - - -class TestCheckForbidIf: - def test_forbidden_absent(self, spark: SparkSession) -> None: - """Target is null when condition is true -> no error.""" - df = spark.createDataFrame( - [Row(subtype="country", parent=None)], - schema="subtype string, parent string", - ) - result = df.select( - check_forbid_if( - F.col("parent"), - F.col("subtype") == "country", - "subtype = country", - ).alias("err") - ).collect() - assert result[0]["err"] is None - - def test_forbidden_present(self, spark: SparkSession) -> None: - """Target is present when condition is true -> error.""" - df = spark.createDataFrame([Row(subtype="country", parent="abc")]) - result = df.select( - check_forbid_if( - F.col("parent"), - F.col("subtype") == "country", - "subtype = country", - ).alias("err") - ).collect() - assert result[0]["err"] is not None - assert "forbidden" in result[0]["err"] - - def test_condition_false_skips(self, spark: SparkSession) -> None: - """Target is present but condition is false -> no error.""" - df = spark.createDataFrame([Row(subtype="region", parent="abc")]) - result = df.select( - check_forbid_if( - F.col("parent"), - F.col("subtype") == "country", - "subtype = country", - ).alias("err") - ).collect() - assert result[0]["err"] is None - - def test_with_value_cols(self, spark: SparkSession) -> None: - """Error message includes actual discriminator value.""" - df = spark.createDataFrame([Row(subtype="country", parent="abc")]) - result = df.select( - check_forbid_if( - F.col("parent"), - F.col("subtype") == "country", - "subtype = country", - F.col("subtype"), - ).alias("err") - ).collect() - assert "country" in result[0]["err"] - - -class TestCheckStringMinLength: - def test_valid_length(self, spark: SparkSession) -> None: - df = spark.createDataFrame([Row(val="abc")]) - result = df.select( - check_string_min_length(F.col("val"), 1).alias("err") - ).collect() - assert result[0]["err"] is None - - def test_empty_string_violation(self, spark: SparkSession) -> None: - df = spark.createDataFrame([Row(val="")]) - result = df.select( - check_string_min_length(F.col("val"), 1).alias("err") - ).collect() - assert result[0]["err"] is not None - assert "minimum length" in result[0]["err"] - - def test_null_passthrough(self, spark: SparkSession) -> None: - df = spark.createDataFrame([Row(val=None)], schema="val string") - result = df.select( - check_string_min_length(F.col("val"), 1).alias("err") - ).collect() - assert result[0]["err"] is None - - def test_exact_min_length(self, spark: SparkSession) -> None: - df = spark.createDataFrame([Row(val="ab")]) - result = df.select( - check_string_min_length(F.col("val"), 2).alias("err") - ).collect() - assert result[0]["err"] is None - - def test_below_min_length(self, spark: SparkSession) -> None: - df = spark.createDataFrame([Row(val="a")]) - result = df.select( - check_string_min_length(F.col("val"), 2).alias("err") - ).collect() - assert result[0]["err"] is not None - - -class TestCheckStringMaxLength: - def test_valid_length(self, spark: SparkSession) -> None: - df = spark.createDataFrame([Row(val="abc")]) - result = df.select( - check_string_max_length(F.col("val"), 5).alias("err") - ).collect() - assert result[0]["err"] is None - - def test_above_max_length(self, spark: SparkSession) -> None: - df = spark.createDataFrame([Row(val="abcdef")]) - result = df.select( - check_string_max_length(F.col("val"), 5).alias("err") - ).collect() - assert result[0]["err"] is not None - assert "maximum length" in result[0]["err"] - - def test_null_passthrough(self, spark: SparkSession) -> None: - df = spark.createDataFrame([Row(val=None)], schema="val string") - result = df.select( - check_string_max_length(F.col("val"), 5).alias("err") - ).collect() - assert result[0]["err"] is None - - def test_exact_max_length(self, spark: SparkSession) -> None: - df = spark.createDataFrame([Row(val="abcde")]) - result = df.select( - check_string_max_length(F.col("val"), 5).alias("err") - ).collect() - assert result[0]["err"] is None - - -class TestCheckRadioGroup: - def test_exactly_one_true(self, spark: SparkSession) -> None: - df = spark.createDataFrame([Row(is_land=True, is_territorial=False)]) - result = df.select( - check_radio_group( - [F.col("is_land"), F.col("is_territorial")], - ["is_land", "is_territorial"], - ).alias("err") - ).collect() - assert result[0]["err"] is None - - def test_none_true(self, spark: SparkSession) -> None: - df = spark.createDataFrame([Row(is_land=False, is_territorial=False)]) - result = df.select( - check_radio_group( - [F.col("is_land"), F.col("is_territorial")], - ["is_land", "is_territorial"], - ).alias("err") - ).collect() - assert result[0]["err"] is not None - assert "exactly one" in result[0]["err"] - assert "0" in result[0]["err"] - - def test_both_true(self, spark: SparkSession) -> None: - df = spark.createDataFrame([Row(is_land=True, is_territorial=True)]) - result = df.select( - check_radio_group( - [F.col("is_land"), F.col("is_territorial")], - ["is_land", "is_territorial"], - ).alias("err") - ).collect() - assert result[0]["err"] is not None - assert "2" in result[0]["err"] - - def test_null_treated_as_false(self, spark: SparkSession) -> None: - """Null booleans count as not-true (0 toward the count).""" - df = spark.createDataFrame( - [Row(is_land=True, is_territorial=None)], - schema="is_land boolean, is_territorial boolean", - ) - result = df.select( - check_radio_group( - [F.col("is_land"), F.col("is_territorial")], - ["is_land", "is_territorial"], - ).alias("err") - ).collect() - assert result[0]["err"] is None - - -class TestCheckGeometryType: - def test_point_matches(self, spark: SparkSession) -> None: - wkb_bytes = Point(0, 0).wkb - df = spark.createDataFrame( - [Row(geometry=bytearray(wkb_bytes))], schema="geometry binary" - ) - result = df.select( - check_geometry_type(F.col("geometry"), GeometryType.POINT).alias("err") - ).collect() - assert result[0]["err"] is None - - def test_point_rejects_linestring(self, spark: SparkSession) -> None: - wkb_bytes = LineString([(0, 0), (1, 1)]).wkb - df = spark.createDataFrame( - [Row(geometry=bytearray(wkb_bytes))], schema="geometry binary" - ) - result = df.select( - check_geometry_type(F.col("geometry"), GeometryType.POINT).alias("err") - ).collect() - assert result[0]["err"] is not None - assert "Point" in result[0]["err"] - - def test_multiple_allowed_types(self, spark: SparkSession) -> None: - wkb_polygon = Polygon([(0, 0), (1, 0), (1, 1), (0, 0)]).wkb - wkb_multi = MultiPolygon([Polygon([(0, 0), (1, 0), (1, 1), (0, 0)])]).wkb - df = spark.createDataFrame( - [ - Row(geometry=bytearray(wkb_polygon)), - Row(geometry=bytearray(wkb_multi)), - ], - schema="geometry binary", - ) - result = df.select( - check_geometry_type( - F.col("geometry"), - GeometryType.POLYGON, - GeometryType.MULTI_POLYGON, - ).alias("err") - ).collect() - assert all(r["err"] is None for r in result) - - def test_multiple_allowed_rejects_wrong_type(self, spark: SparkSession) -> None: - wkb_point = Point(0, 0).wkb - df = spark.createDataFrame( - [Row(geometry=bytearray(wkb_point))], schema="geometry binary" - ) - result = df.select( - check_geometry_type( - F.col("geometry"), - GeometryType.POLYGON, - GeometryType.MULTI_POLYGON, - ).alias("err") - ).collect() - assert result[0]["err"] is not None - - def test_null_passthrough(self, spark: SparkSession) -> None: - df = spark.createDataFrame([Row(geometry=None)], schema="geometry binary") - result = df.select( - check_geometry_type(F.col("geometry"), GeometryType.POINT).alias("err") - ).collect() - assert result[0]["err"] is None - - def test_big_endian_wkb(self, spark: SparkSession) -> None: - """Verify BE byte order handling. - - Shapely writes LE by default. Construct BE WKB for a Point - manually: byte_order=0x00, type=0x00000001, x=0.0, y=0.0. - """ - be_point = struct.pack(">bIdd", 0, 1, 0.0, 0.0) - df = spark.createDataFrame( - [Row(geometry=bytearray(be_point))], schema="geometry binary" - ) - result = df.select( - check_geometry_type(F.col("geometry"), GeometryType.POINT).alias("err") - ).collect() - assert result[0]["err"] is None - - def test_iso_wkb_z_point_accepted(self, spark: SparkSession) -> None: - """ISO WKB encodes Z by offsetting the type (PointZ=1001), shifting - the low byte to 0xE9. GeoParquet mandates ISO WKB, so 3D geometries - reach the check this way and must still validate by base type.""" - iso_point_z = struct.pack(" None: - iso_point_z_be = struct.pack(">bIddd", 0, 1001, 0.0, 0.0, 5.0) - df = spark.createDataFrame( - [Row(geometry=bytearray(iso_point_z_be))], schema="geometry binary" - ) - result = df.select( - check_geometry_type(F.col("geometry"), GeometryType.POINT).alias("err") - ).collect() - assert result[0]["err"] is None - - def test_ewkb_z_point_accepted(self, spark: SparkSession) -> None: - """EWKB encodes Z as a high flag bit (0x80000001), leaving the low - byte at 0x01 -- shapely's `.wkb` default. Must keep validating.""" - ewkb_point_z = struct.pack(" None: - """shapely's native 3D WKB output validates as POINT.""" - df = spark.createDataFrame( - [Row(geometry=bytearray(Point(0, 0, 5).wkb))], schema="geometry binary" - ) - result = df.select( - check_geometry_type(F.col("geometry"), GeometryType.POINT).alias("err") - ).collect() - assert result[0]["err"] is None - - def test_iso_wkb_z_wrong_type_rejected(self, spark: SparkSession) -> None: - """A 3D LineString (ISO 1002) is still rejected when POINT is expected - -- normalization strips the dimension offset, not the base type.""" - iso_linestring_z = struct.pack(" None: - """A non-null WKB blob too short to contain a type word is flagged as a violation.""" - truncated = bytearray(b"\x01") - df = spark.createDataFrame([Row(geometry=truncated)], schema="geometry binary") - result = df.select( - check_geometry_type(F.col("geometry"), GeometryType.POINT).alias("err") - ).collect() - assert result[0]["err"] is not None - - @pytest.mark.parametrize("nbytes", [1, 2, 3, 4]) - def test_partial_header_wkb_flagged(self, spark: SparkSession, nbytes: int) -> None: - """A blob with a partial WKB header is flagged, even when conv() yields a non-null type. - - A little-endian order flag followed by a partial type word (2-4 bytes) - parses to a non-null but bogus base type -- e.g. `b"\\x01\\x01"` reads as - type 1, the Point code, and would silently validate as a Point. Only a - 0-1 byte blob makes conv() return NULL, so a length gate (not a null - check) is what closes the truncation hole. - """ - partial = bytearray(b"\x01" * nbytes) - df = spark.createDataFrame([Row(geometry=partial)], schema="geometry binary") - result = df.select( - check_geometry_type(F.col("geometry"), GeometryType.POINT).alias("err") - ).collect() - assert result[0]["err"] is not None - - -class TestCheckStripped: - def test_clean_string(self, spark: SparkSession) -> None: - df = spark.createDataFrame([Row(val="hello world")]) - result = df.select(check_stripped(F.col("val")).alias("err")).collect() - assert result[0]["err"] is None - - def test_single_char(self, spark: SparkSession) -> None: - df = spark.createDataFrame([Row(val="x")]) - result = df.select(check_stripped(F.col("val")).alias("err")).collect() - assert result[0]["err"] is None - - def test_leading_space(self, spark: SparkSession) -> None: - df = spark.createDataFrame([Row(val=" hello")]) - result = df.select(check_stripped(F.col("val")).alias("err")).collect() - assert result[0]["err"] is not None - assert "whitespace" in result[0]["err"] - - def test_trailing_space(self, spark: SparkSession) -> None: - df = spark.createDataFrame([Row(val="hello ")]) - result = df.select(check_stripped(F.col("val")).alias("err")).collect() - assert result[0]["err"] is not None - assert "whitespace" in result[0]["err"] - - def test_leading_tab(self, spark: SparkSession) -> None: - """Tab is Unicode whitespace -- must be caught (not just ASCII space).""" - df = spark.createDataFrame([Row(val="\thello")]) - result = df.select(check_stripped(F.col("val")).alias("err")).collect() - assert result[0]["err"] is not None - - def test_trailing_newline(self, spark: SparkSession) -> None: - """Trailing newline requires \\z anchor -- $ matches before it in Java regex.""" - df = spark.createDataFrame([Row(val="hello\n")]) - result = df.select(check_stripped(F.col("val")).alias("err")).collect() - assert result[0]["err"] is not None - - def test_null_passthrough(self, spark: SparkSession) -> None: - df = spark.createDataFrame([Row(val=None)], schema="val string") - result = df.select(check_stripped(F.col("val")).alias("err")).collect() - assert result[0]["err"] is None - - def test_empty_string(self, spark: SparkSession) -> None: - """Empty string has no leading/trailing whitespace -- passes.""" - df = spark.createDataFrame([Row(val="")]) - result = df.select(check_stripped(F.col("val")).alias("err")).collect() - assert result[0]["err"] is None - - def test_trailing_unit_separator(self, spark: SparkSession) -> None: - """U+001F (unit separator) -- Python strips it, Java \\S with (?U) does not.""" - df = spark.createDataFrame([Row(val="Main St \x1f")]) - result = df.select(check_stripped(F.col("val")).alias("err")).collect() - assert result[0]["err"] is not None - - def test_leading_file_separator(self, spark: SparkSession) -> None: - """U+001C (file separator) -- C0 control char Python treats as whitespace.""" - df = spark.createDataFrame([Row(val="\x1chello")]) - result = df.select(check_stripped(F.col("val")).alias("err")).collect() - assert result[0]["err"] is not None - - def test_trailing_soh(self, spark: SparkSession) -> None: - """U+0001 (SOH) -- C0 control char that even Python's strip() misses.""" - df = spark.createDataFrame([Row(val="hello\x01")]) - result = df.select(check_stripped(F.col("val")).alias("err")).collect() - assert result[0]["err"] is not None - - def test_trailing_del(self, spark: SparkSession) -> None: - """U+007F (DEL) -- control char outside C0 range.""" - df = spark.createDataFrame([Row(val="hello\x7f")]) - result = df.select(check_stripped(F.col("val")).alias("err")).collect() - assert result[0]["err"] is not None - - def test_trailing_c1_control(self, spark: SparkSession) -> None: - """U+009F (APC) -- C1 control char.""" - df = spark.createDataFrame([Row(val="hello\x9f")]) - result = df.select(check_stripped(F.col("val")).alias("err")).collect() - assert result[0]["err"] is not None - - def test_control_char_in_middle_passes(self, spark: SparkSession) -> None: - """Control chars in the middle of a string are not a stripped concern.""" - df = spark.createDataFrame([Row(val="hel\x1flo")]) - result = df.select(check_stripped(F.col("val")).alias("err")).collect() - assert result[0]["err"] is None - - -class TestCheckJsonPointer: - def test_valid_pointer(self, spark: SparkSession) -> None: - df = spark.createDataFrame([Row(val="/properties/name")]) - result = df.select(check_json_pointer(F.col("val")).alias("err")).collect() - assert result[0]["err"] is None - - def test_root_pointer(self, spark: SparkSession) -> None: - df = spark.createDataFrame([Row(val="/")]) - result = df.select(check_json_pointer(F.col("val")).alias("err")).collect() - assert result[0]["err"] is None - - def test_empty_string_valid(self, spark: SparkSession) -> None: - """Empty string is valid per RFC 6901 (references whole document).""" - df = spark.createDataFrame([Row(val="")]) - result = df.select(check_json_pointer(F.col("val")).alias("err")).collect() - assert result[0]["err"] is None - - def test_missing_leading_slash(self, spark: SparkSession) -> None: - df = spark.createDataFrame([Row(val="properties/name")]) - result = df.select(check_json_pointer(F.col("val")).alias("err")).collect() - assert result[0]["err"] is not None - assert "JSON pointer" in result[0]["err"] - assert "properties/name" in result[0]["err"] - - def test_null_passthrough(self, spark: SparkSession) -> None: - df = spark.createDataFrame([Row(val=None)], schema="val string") - result = df.select(check_json_pointer(F.col("val")).alias("err")).collect() - assert result[0]["err"] is None - - -class TestCheckLinearRangeLength: - def test_valid_length(self, spark: SparkSession) -> None: - df = spark.createDataFrame( - [Row(between=[0.0, 1.0])], schema="between array" - ) - result = df.select( - check_linear_range_length(F.col("between")).alias("err") - ).collect() - assert result[0]["err"] is None - - def test_wrong_length_one(self, spark: SparkSession) -> None: - df = spark.createDataFrame([Row(between=[0.5])], schema="between array") - result = df.select( - check_linear_range_length(F.col("between")).alias("err") - ).collect() - assert result[0]["err"] is not None - assert "2 elements" in result[0]["err"] - - def test_wrong_length_three(self, spark: SparkSession) -> None: - df = spark.createDataFrame( - [Row(between=[0.0, 0.5, 1.0])], schema="between array" - ) - result = df.select( - check_linear_range_length(F.col("between")).alias("err") - ).collect() - assert result[0]["err"] is not None - assert "2 elements" in result[0]["err"] - - def test_empty_array(self, spark: SparkSession) -> None: - df = spark.createDataFrame([Row(between=[])], schema="between array") - result = df.select( - check_linear_range_length(F.col("between")).alias("err") - ).collect() - assert result[0]["err"] is not None - assert "2 elements" in result[0]["err"] - - def test_null_passthrough(self, spark: SparkSession) -> None: - df = spark.createDataFrame([Row(between=None)], schema="between array") - result = df.select( - check_linear_range_length(F.col("between")).alias("err") - ).collect() - assert result[0]["err"] is None - - -class TestCheckLinearRangeBounds: - def test_valid_bounds(self, spark: SparkSession) -> None: - df = spark.createDataFrame( - [Row(between=[0.2, 0.8])], schema="between array" - ) - result = df.select( - check_linear_range_bounds(F.col("between")).alias("err") - ).collect() - assert result[0]["err"] is None - - def test_value_below_zero(self, spark: SparkSession) -> None: - df = spark.createDataFrame( - [Row(between=[-0.1, 0.5])], schema="between array" - ) - result = df.select( - check_linear_range_bounds(F.col("between")).alias("err") - ).collect() - assert result[0]["err"] is not None - assert "[0.0, 1.0]" in result[0]["err"] - - def test_value_above_one(self, spark: SparkSession) -> None: - df = spark.createDataFrame( - [Row(between=[0.0, 1.1])], schema="between array" - ) - result = df.select( - check_linear_range_bounds(F.col("between")).alias("err") - ).collect() - assert result[0]["err"] is not None - assert "[0.0, 1.0]" in result[0]["err"] - - def test_wrong_length_passthrough(self, spark: SparkSession) -> None: - """Wrong-length arrays are not this function's concern.""" - df = spark.createDataFrame([Row(between=[0.5])], schema="between array") - result = df.select( - check_linear_range_bounds(F.col("between")).alias("err") - ).collect() - assert result[0]["err"] is None - - def test_null_passthrough(self, spark: SparkSession) -> None: - df = spark.createDataFrame([Row(between=None)], schema="between array") - result = df.select( - check_linear_range_bounds(F.col("between")).alias("err") - ).collect() - assert result[0]["err"] is None - - -class TestCheckLinearRangeOrder: - def test_valid_order(self, spark: SparkSession) -> None: - df = spark.createDataFrame( - [Row(between=[0.2, 0.8])], schema="between array" - ) - result = df.select( - check_linear_range_order(F.col("between")).alias("err") - ).collect() - assert result[0]["err"] is None - - def test_start_equals_end(self, spark: SparkSession) -> None: - df = spark.createDataFrame( - [Row(between=[0.5, 0.5])], schema="between array" - ) - result = df.select( - check_linear_range_order(F.col("between")).alias("err") - ).collect() - assert result[0]["err"] is not None - assert "start must be < end" in result[0]["err"] - - def test_start_after_end(self, spark: SparkSession) -> None: - df = spark.createDataFrame( - [Row(between=[0.8, 0.2])], schema="between array" - ) - result = df.select( - check_linear_range_order(F.col("between")).alias("err") - ).collect() - assert result[0]["err"] is not None - assert "start must be < end" in result[0]["err"] - - def test_wrong_length_passthrough(self, spark: SparkSession) -> None: - """Wrong-length arrays are not this function's concern.""" - df = spark.createDataFrame([Row(between=[0.5])], schema="between array") - result = df.select( - check_linear_range_order(F.col("between")).alias("err") - ).collect() - assert result[0]["err"] is None - - def test_null_passthrough(self, spark: SparkSession) -> None: - df = spark.createDataFrame([Row(between=None)], schema="between array") - result = df.select( - check_linear_range_order(F.col("between")).alias("err") - ).collect() - assert result[0]["err"] is None - - -def test_check_required_null_is_error(spark: SparkSession) -> None: - df = spark.createDataFrame([Row(val=None)], schema="val string") - result = df.select(check_required(F.col("val")).alias("err")).collect() - assert result[0]["err"] is not None - assert "missing" in result[0]["err"] - - -def test_check_required_non_null_passes(spark: SparkSession) -> None: - df = spark.createDataFrame([Row(val="hello")]) - result = df.select(check_required(F.col("val")).alias("err")).collect() - assert result[0]["err"] is None - - -def test_check_required_composes_with_enum(spark: SparkSession) -> None: - """check_required + check_enum via F.coalesce catches both null and invalid.""" - df = spark.createDataFrame([Row(val=None)], schema="val string") - expr = F.coalesce( - check_required(F.col("val")), - check_enum(F.col("val"), ["a", "b"]), - ) - result = df.select(expr.alias("err")).collect() - assert result[0]["err"] is not None - assert "missing" in result[0]["err"] - - +# Pattern/label pairs shared between a case's check and its expect predicate. _COUNTRY_CODE_PATTERN = r"^[A-Z]{2}\z" _COUNTRY_CODE_LABEL = "ISO 3166-1 alpha-2 country code" - - -class TestCheckCountryCodeViaPattern: - """Country code validation through check_pattern with label.""" - - def test_valid(self, spark: SparkSession) -> None: - df = spark.createDataFrame([("US",)], ["v"]) - result = df.select( - check_pattern( - F.col("v"), _COUNTRY_CODE_PATTERN, label=_COUNTRY_CODE_LABEL - ).alias("e") - ) - assert result.collect()[0]["e"] is None - - def test_lowercase_invalid(self, spark: SparkSession) -> None: - df = spark.createDataFrame([("us",)], ["v"]) - result = df.select( - check_pattern( - F.col("v"), _COUNTRY_CODE_PATTERN, label=_COUNTRY_CODE_LABEL - ).alias("e") - ) - err = result.collect()[0]["e"] - assert f"invalid {_COUNTRY_CODE_LABEL}" in err - assert "us" in err - - def test_three_chars_invalid(self, spark: SparkSession) -> None: - df = spark.createDataFrame([("USA",)], ["v"]) - result = df.select( - check_pattern( - F.col("v"), _COUNTRY_CODE_PATTERN, label=_COUNTRY_CODE_LABEL - ).alias("e") - ) - assert result.collect()[0]["e"] is not None - - def test_null_passes(self, spark: SparkSession) -> None: - df = spark.createDataFrame([(None,)], schema="v string") - result = df.select( - check_pattern( - F.col("v"), _COUNTRY_CODE_PATTERN, label=_COUNTRY_CODE_LABEL - ).alias("e") - ) - assert result.collect()[0]["e"] is None - - _REGION_CODE_PATTERN = r"^[A-Z]{2}-[A-Z0-9]{1,3}\z" _REGION_CODE_LABEL = "ISO 3166-2 subdivision code" - - -class TestCheckRegionCodeViaPattern: - """Region code validation through check_pattern with label.""" - - def test_valid(self, spark: SparkSession) -> None: - df = spark.createDataFrame([("US-NY",)], ["v"]) - result = df.select( - check_pattern( - F.col("v"), _REGION_CODE_PATTERN, label=_REGION_CODE_LABEL - ).alias("e") - ) - assert result.collect()[0]["e"] is None - - def test_valid_numeric(self, spark: SparkSession) -> None: - df = spark.createDataFrame([("CN-11",)], ["v"]) - result = df.select( - check_pattern( - F.col("v"), _REGION_CODE_PATTERN, label=_REGION_CODE_LABEL - ).alias("e") - ) - assert result.collect()[0]["e"] is None - - def test_no_dash_invalid(self, spark: SparkSession) -> None: - df = spark.createDataFrame([("USNY",)], ["v"]) - result = df.select( - check_pattern( - F.col("v"), _REGION_CODE_PATTERN, label=_REGION_CODE_LABEL - ).alias("e") - ) - err = result.collect()[0]["e"] - assert f"invalid {_REGION_CODE_LABEL}" in err - assert "USNY" in err - - def test_null_passes(self, spark: SparkSession) -> None: - df = spark.createDataFrame([(None,)], schema="v string") - result = df.select( - check_pattern( - F.col("v"), _REGION_CODE_PATTERN, label=_REGION_CODE_LABEL - ).alias("e") - ) - assert result.collect()[0]["e"] is None - - _SNAKE_CASE_PATTERN = r"^[a-z0-9]+(_[a-z0-9]+)*\z" _SNAKE_CASE_LABEL = "Category in snake_case format" - - -class TestCheckSnakeCaseViaPattern: - """Snake_case validation through check_pattern with label.""" - - def test_valid(self, spark: SparkSession) -> None: - df = spark.createDataFrame([("hello_world",)], ["v"]) - result = df.select( - check_pattern( - F.col("v"), _SNAKE_CASE_PATTERN, label=_SNAKE_CASE_LABEL - ).alias("e") - ) - assert result.collect()[0]["e"] is None - - def test_single_word(self, spark: SparkSession) -> None: - df = spark.createDataFrame([("hello",)], ["v"]) - result = df.select( - check_pattern( - F.col("v"), _SNAKE_CASE_PATTERN, label=_SNAKE_CASE_LABEL - ).alias("e") - ) - assert result.collect()[0]["e"] is None - - def test_with_numbers(self, spark: SparkSession) -> None: - df = spark.createDataFrame([("hello_123",)], ["v"]) - result = df.select( - check_pattern( - F.col("v"), _SNAKE_CASE_PATTERN, label=_SNAKE_CASE_LABEL - ).alias("e") - ) - assert result.collect()[0]["e"] is None - - def test_uppercase_invalid(self, spark: SparkSession) -> None: - df = spark.createDataFrame([("Hello_World",)], ["v"]) - result = df.select( - check_pattern( - F.col("v"), _SNAKE_CASE_PATTERN, label=_SNAKE_CASE_LABEL - ).alias("e") - ) - err = result.collect()[0]["e"] - assert f"invalid {_SNAKE_CASE_LABEL}" in err - - def test_spaces_invalid(self, spark: SparkSession) -> None: - df = spark.createDataFrame([("hello world",)], ["v"]) - result = df.select( - check_pattern( - F.col("v"), _SNAKE_CASE_PATTERN, label=_SNAKE_CASE_LABEL - ).alias("e") - ) - assert result.collect()[0]["e"] is not None - - def test_null_passes(self, spark: SparkSession) -> None: - df = spark.createDataFrame([(None,)], schema="v string") - result = df.select( - check_pattern( - F.col("v"), _SNAKE_CASE_PATTERN, label=_SNAKE_CASE_LABEL - ).alias("e") - ) - assert result.collect()[0]["e"] is None - - -def test_check_url_format_http_valid(spark: SparkSession) -> None: - df = spark.createDataFrame([Row(val="http://example.com")]) - result = df.select(check_url_format(F.col("val")).alias("err")).collect() - assert result[0]["err"] is None - - -def test_check_url_format_https_valid(spark: SparkSession) -> None: - df = spark.createDataFrame([Row(val="https://example.com/path?q=1")]) - result = df.select(check_url_format(F.col("val")).alias("err")).collect() - assert result[0]["err"] is None - - -def test_check_url_format_uppercase_scheme_valid(spark: SparkSession) -> None: - """Pydantic HttpUrl lowercases the scheme, so HTTP:// is accepted.""" - df = spark.createDataFrame([Row(val="HTTP://example.com")]) - result = df.select(check_url_format(F.col("val")).alias("err")).collect() - assert result[0]["err"] is None - - -def test_check_url_format_mixed_case_scheme_valid(spark: SparkSession) -> None: - df = spark.createDataFrame([Row(val="HtTpS://example.com/path")]) - result = df.select(check_url_format(F.col("val")).alias("err")).collect() - assert result[0]["err"] is None - - -def test_check_url_format_no_scheme_invalid(spark: SparkSession) -> None: - df = spark.createDataFrame([Row(val="example.com")]) - result = df.select(check_url_format(F.col("val")).alias("err")).collect() - assert result[0]["err"] is not None - - -def test_check_url_format_ftp_scheme_invalid(spark: SparkSession) -> None: - df = spark.createDataFrame([Row(val="ftp://example.com")]) - result = df.select(check_url_format(F.col("val")).alias("err")).collect() - assert result[0]["err"] is not None - - -def test_check_url_format_null_passes(spark: SparkSession) -> None: - df = spark.createDataFrame([Row(val=None)], schema="val string") - result = df.select(check_url_format(F.col("val")).alias("err")).collect() - assert result[0]["err"] is None - - -def test_check_url_length_exceeds_2083_chars_invalid(spark: SparkSession) -> None: - long_url = "https://example.com/" + "a" * 2064 # 2084 chars - df = spark.createDataFrame([Row(val=long_url)]) - result = df.select(check_url_length(F.col("val")).alias("err")).collect() - assert result[0]["err"] is not None - - -def test_check_url_length_exactly_2083_chars_valid(spark: SparkSession) -> None: - url = "https://example.com/" + "a" * 2063 # 2083 chars - df = spark.createDataFrame([Row(val=url)]) - result = df.select(check_url_length(F.col("val")).alias("err")).collect() - assert result[0]["err"] is None - - -def test_check_url_length_null_passes(spark: SparkSession) -> None: - df = spark.createDataFrame([Row(val=None)], schema="val string") - result = df.select(check_url_length(F.col("val")).alias("err")).collect() - assert result[0]["err"] is None - - -def test_check_email_valid(spark: SparkSession) -> None: - df = spark.createDataFrame([Row(val="user@example.com")]) - result = df.select(check_email(F.col("val")).alias("err")).collect() - assert result[0]["err"] is None - - -def test_check_email_no_at_invalid(spark: SparkSession) -> None: - df = spark.createDataFrame([Row(val="userexample.com")]) - result = df.select(check_email(F.col("val")).alias("err")).collect() - assert result[0]["err"] is not None - - -def test_check_email_no_domain_invalid(spark: SparkSession) -> None: - df = spark.createDataFrame([Row(val="user@")]) - result = df.select(check_email(F.col("val")).alias("err")).collect() - assert result[0]["err"] is not None - - -def test_check_email_spaces_invalid(spark: SparkSession) -> None: - df = spark.createDataFrame([Row(val="user @example.com")]) - result = df.select(check_email(F.col("val")).alias("err")).collect() - assert result[0]["err"] is not None - - -def test_check_email_null_passes(spark: SparkSession) -> None: - df = spark.createDataFrame([Row(val=None)], schema="val string") - result = df.select(check_email(F.col("val")).alias("err")).collect() - assert result[0]["err"] is None - - -def test_check_email_trailing_period_invalid(spark: SparkSession) -> None: - df = spark.createDataFrame([Row(val="user@example.com.")]) - result = df.select(check_email(F.col("val")).alias("err")).collect() - assert result[0]["err"] is not None - - -def test_check_email_leading_period_invalid(spark: SparkSession) -> None: - df = spark.createDataFrame([Row(val=".user@example.com")]) - result = df.select(check_email(F.col("val")).alias("err")).collect() - assert result[0]["err"] is not None - - -def test_check_email_period_before_at_invalid(spark: SparkSession) -> None: - df = spark.createDataFrame([Row(val="user.@example.com")]) - result = df.select(check_email(F.col("val")).alias("err")).collect() - assert result[0]["err"] is not None - - -def test_check_email_period_after_at_invalid(spark: SparkSession) -> None: - df = spark.createDataFrame([Row(val="user@.example.com")]) - result = df.select(check_email(F.col("val")).alias("err")).collect() - assert result[0]["err"] is not None - - -def test_check_email_double_period_domain_invalid(spark: SparkSession) -> None: - df = spark.createDataFrame([Row(val="user@example..com")]) - result = df.select(check_email(F.col("val")).alias("err")).collect() - assert result[0]["err"] is not None - - -def test_check_email_dotted_local_valid(spark: SparkSession) -> None: - df = spark.createDataFrame([Row(val="user.name@example.com")]) - result = df.select(check_email(F.col("val")).alias("err")).collect() - assert result[0]["err"] is None - - -def test_check_email_subdomain_valid(spark: SparkSession) -> None: - df = spark.createDataFrame([Row(val="user@mail.example.co.uk")]) - result = df.select(check_email(F.col("val")).alias("err")).collect() - assert result[0]["err"] is None - - _PHONE_PATTERN = r"^\+\d{1,3}[\s\-\(\)0-9]+\z" _PHONE_LABEL = "International phone number (+ followed by country code and number)" - - -class TestCheckPhoneViaPattern: - """Phone number validation through check_pattern with label.""" - - def test_valid_us(self, spark: SparkSession) -> None: - df = spark.createDataFrame([("+1 555-555-5555",)], ["v"]) - result = df.select( - check_pattern(F.col("v"), _PHONE_PATTERN, label=_PHONE_LABEL).alias("e") - ) - assert result.collect()[0]["e"] is None - - def test_valid_international(self, spark: SparkSession) -> None: - df = spark.createDataFrame([("+44 20 7946 0958",)], ["v"]) - result = df.select( - check_pattern(F.col("v"), _PHONE_PATTERN, label=_PHONE_LABEL).alias("e") - ) - assert result.collect()[0]["e"] is None - - def test_no_plus_invalid(self, spark: SparkSession) -> None: - df = spark.createDataFrame([("555-555-5555",)], ["v"]) - result = df.select( - check_pattern(F.col("v"), _PHONE_PATTERN, label=_PHONE_LABEL).alias("e") - ) - err = result.collect()[0]["e"] - assert f"invalid {_PHONE_LABEL}" in err - - def test_letters_invalid(self, spark: SparkSession) -> None: - df = spark.createDataFrame([("+1 abc-defg",)], ["v"]) - result = df.select( - check_pattern(F.col("v"), _PHONE_PATTERN, label=_PHONE_LABEL).alias("e") - ) - assert result.collect()[0]["e"] is not None - - def test_null_passes(self, spark: SparkSession) -> None: - df = spark.createDataFrame([(None,)], schema="v string") - result = df.select( - check_pattern(F.col("v"), _PHONE_PATTERN, label=_PHONE_LABEL).alias("e") - ) - assert result.collect()[0]["e"] is None - - _WIKIDATA_PATTERN = r"^Q\d+\z" _WIKIDATA_LABEL = "Wikidata identifier (Q followed by digits)" -class TestCheckWikidataIdViaPattern: - """Wikidata ID validation through check_pattern with label.""" +@dataclass(frozen=True) +class _Case: + """One constraint_expressions assertion driven off the shared wide row. - def test_valid(self, spark: SparkSession) -> None: - df = spark.createDataFrame([("Q42",)], ["v"]) - result = df.select( - check_pattern(F.col("v"), _WIKIDATA_PATTERN, label=_WIKIDATA_LABEL).alias( - "e" - ) - ) - assert result.collect()[0]["e"] is None + `check(field)` builds the constraint check over the case's input column; + `expect(result)` is a predicate on that column's single collected value. + """ - def test_large_number(self, spark: SparkSession) -> None: - df = spark.createDataFrame([("Q123456789",)], ["v"]) - result = df.select( - check_pattern(F.col("v"), _WIKIDATA_PATTERN, label=_WIKIDATA_LABEL).alias( - "e" - ) - ) - assert result.collect()[0]["e"] is None + id: str + ddl: str + value: Any + check: Callable[[str], Column] + expect: Callable[[Any], bool] - def test_lowercase_q_invalid(self, spark: SparkSession) -> None: - df = spark.createDataFrame([("q42",)], ["v"]) - result = df.select( - check_pattern(F.col("v"), _WIKIDATA_PATTERN, label=_WIKIDATA_LABEL).alias( - "e" - ) - ) - err = result.collect()[0]["e"] - assert f"invalid {_WIKIDATA_LABEL}" in err - def test_no_digits_invalid(self, spark: SparkSession) -> None: - df = spark.createDataFrame([("Q",)], ["v"]) - result = df.select( - check_pattern(F.col("v"), _WIKIDATA_PATTERN, label=_WIKIDATA_LABEL).alias( - "e" - ) - ) - assert result.collect()[0]["e"] is not None - - def test_p_prefix_invalid(self, spark: SparkSession) -> None: - df = spark.createDataFrame([("P42",)], ["v"]) - result = df.select( - check_pattern(F.col("v"), _WIKIDATA_PATTERN, label=_WIKIDATA_LABEL).alias( - "e" - ) - ) - assert result.collect()[0]["e"] is not None - - def test_null_passes(self, spark: SparkSession) -> None: - df = spark.createDataFrame([(None,)], schema="v string") - result = df.select( - check_pattern(F.col("v"), _WIKIDATA_PATTERN, label=_WIKIDATA_LABEL).alias( - "e" - ) - ) - assert result.collect()[0]["e"] is None - - -class TestCheckMinFieldsSet: - def test_meets_threshold(self, spark: SparkSession) -> None: - """Count at threshold -> no error.""" - df = spark.createDataFrame( - [Row(a=1, b=2, c=None)], schema="a int, b int, c int" - ) - result = df.select( - check_min_fields_set( - [F.col("a"), F.col("b"), F.col("c")], - ["a", "b", "c"], - 2, - ).alias("err") - ).collect() - assert result[0]["err"] is None - - def test_exceeds_threshold(self, spark: SparkSession) -> None: - """Count above threshold -> no error.""" - df = spark.createDataFrame([Row(a=1, b=2, c=3)], schema="a int, b int, c int") - result = df.select( - check_min_fields_set( - [F.col("a"), F.col("b"), F.col("c")], - ["a", "b", "c"], - 2, - ).alias("err") - ).collect() - assert result[0]["err"] is None - - def test_below_threshold(self, spark: SparkSession) -> None: - """Count below threshold -> error with field names and actual count.""" - df = spark.createDataFrame( - [Row(a=1, b=None, c=None)], schema="a int, b int, c int" - ) - result = df.select( - check_min_fields_set( - [F.col("a"), F.col("b"), F.col("c")], - ["a", "b", "c"], - 2, - ).alias("err") - ).collect() - err = result[0]["err"] - assert err is not None - assert "at least 2" in err - assert "a, b, c" in err - assert "1" in err - - def test_all_null_below_threshold(self, spark: SparkSession) -> None: - """All null -> error showing 0 non-null.""" - df = spark.createDataFrame([Row(a=None, b=None)], schema="a int, b int") - result = df.select( - check_min_fields_set( - [F.col("a"), F.col("b")], - ["a", "b"], - 1, - ).alias("err") - ).collect() - err = result[0]["err"] - assert err is not None - assert "0" in err - - def test_error_message_format(self, spark: SparkSession) -> None: - """Error message matches expected format exactly.""" - df = spark.createDataFrame([Row(x=None, y=None)], schema="x int, y int") - result = df.select( - check_min_fields_set( - [F.col("x"), F.col("y")], - ["x", "y"], - 1, - ).alias("err") - ).collect() - err = result[0]["err"] - assert err == "at least 1 of x, y required, got 0 non-null" - - -_BBOX_SCHEMA = StructType( - [ - StructField( - "bbox", - StructType( - [ - StructField("xmin", DoubleType(), True), - StructField("xmax", DoubleType(), True), - StructField("ymin", DoubleType(), True), - StructField("ymax", DoubleType(), True), - ] - ), - True, +_CASES: list[_Case] = [ + # --- except_literals: suppress an allowed literal, pass real violations ---- + # "" is an allowed literal alternative -> the url_format error is suppressed. + _Case( + "el_suppress", + "string", + "", + lambda f: except_literals(F.col(f), check_url_format(F.col(f)), [""]), + lambda r: r is None, + ), + # A non-literal invalid value still surfaces the inner check's error. + _Case( + "el_violation", + "string", + "not a url", + lambda f: except_literals(F.col(f), check_url_format(F.col(f)), [""]), + lambda r: r is not None, + ), + _Case( + "el_valid", + "string", + "https://example.com/x", + lambda f: except_literals(F.col(f), check_url_format(F.col(f)), [""]), + lambda r: r is None, + ), + _Case( + "el_null", + "string", + None, + lambda f: except_literals(F.col(f), check_url_format(F.col(f)), [""]), + lambda r: r is None, + ), + # --- check_multiple_of ---------------------------------------------------- + _Case( + "multiple_of_integral_float_passes", + "double", + 2.0, + lambda f: check_multiple_of(F.col(f), 1), + lambda r: r is None, + ), + _Case( + "multiple_of_negative_integral_float_passes", + "double", + -3.0, + lambda f: check_multiple_of(F.col(f), 1), + lambda r: r is None, + ), + _Case( + "multiple_of_fractional_float_violation", + "double", + 2.5, + lambda f: check_multiple_of(F.col(f), 1), + lambda r: r is not None and "multiple of" in r, + ), + _Case( + "multiple_of_null_passthrough", + "double", + None, + lambda f: check_multiple_of(F.col(f), 1), + lambda r: r is None, + ), + # Integral doubles beyond 2^63 are whole numbers Pydantic accepts + # ((1e30).is_integer() is True). A floor-based check would saturate the + # LongType cast and wrongly fire; the remainder check passes them. + _Case( + "multiple_of_large_integral_double_passes", + "double", + 1e30, + lambda f: check_multiple_of(F.col(f), 1), + lambda r: r is None, + ), + _Case( + "multiple_of_nan_violation", + "double", + float("nan"), + lambda f: check_multiple_of(F.col(f), 1), + lambda r: r is not None and "multiple of" in r, + ), + _Case( + "multiple_of_positive_infinity_violation", + "double", + float("inf"), + lambda f: check_multiple_of(F.col(f), 1), + lambda r: r is not None and "multiple of" in r, + ), + _Case( + "multiple_of_negative_infinity_violation", + "double", + float("-inf"), + lambda f: check_multiple_of(F.col(f), 1), + lambda r: r is not None and "multiple of" in r, + ), + # Divisor need not be 1: 1.5 is a multiple of 0.5, 1.75 is not. + _Case( + "multiple_of_non_unit_divisor_valid", + "double", + 1.5, + lambda f: check_multiple_of(F.col(f), 0.5), + lambda r: r is None, + ), + _Case( + "multiple_of_non_unit_divisor_violation", + "double", + 1.75, + lambda f: check_multiple_of(F.col(f), 0.5), + lambda r: r is not None, + ), + # --- check_bounds --------------------------------------------------------- + _Case( + "bounds_ge_le_valid", + "int", + 5, + lambda f: check_bounds(F.col(f), ge=1, le=10), + lambda r: r is None, + ), + _Case( + "bounds_ge_violation", + "int", + 0, + lambda f: check_bounds(F.col(f), ge=1), + lambda r: r is not None and ">= 1" in r, + ), + _Case( + "bounds_gt_violation", + "int", + 0, + lambda f: check_bounds(F.col(f), gt=0), + lambda r: r is not None and "> 0" in r, + ), + _Case( + "bounds_le_violation", + "int", + 100, + lambda f: check_bounds(F.col(f), le=50), + lambda r: r is not None, + ), + _Case( + "bounds_null_passthrough", + "int", + None, + lambda f: check_bounds(F.col(f), ge=1), + lambda r: r is None, + ), + # NaN satisfies no Pydantic bound, but Spark sorts NaN above all values, so a + # lower bound (NaN < v) never fires. check_bounds must reject it explicitly. + _Case( + "bounds_nan_ge_violation", + "double", + float("nan"), + lambda f: check_bounds(F.col(f), ge=0), + lambda r: r is not None and "NaN" in r, + ), + # Same lower-bound leak, via the strict-greater comparison. + _Case( + "bounds_nan_gt_violation", + "double", + float("nan"), + lambda f: check_bounds(F.col(f), gt=0), + lambda r: r is not None and "NaN" in r, + ), + # An upper bound already rejects NaN in Spark (NaN > v is true); the explicit + # NaN check keeps that behavior. + _Case( + "bounds_nan_le_violation", + "double", + float("nan"), + lambda f: check_bounds(F.col(f), le=1), + lambda r: r is not None, + ), + # With no bounds there is nothing to violate; NaN passes, matching Pydantic's + # allow_inf_nan default for unconstrained floats. + _Case( + "bounds_nan_no_bounds_passes", + "double", + float("nan"), + lambda f: check_bounds(F.col(f)), + lambda r: r is None, + ), + # A finite in-range float is unaffected by the NaN guard. + _Case( + "bounds_valid_float_passes", + "double", + 0.5, + lambda f: check_bounds(F.col(f), ge=0, le=1), + lambda r: r is None, + ), + # With check_nan=False the NaN guard is absent; NaN slips past a lower bound. + _Case( + "bounds_nan_guard_off_passes", + "double", + float("nan"), + lambda f: check_bounds(F.col(f), ge=0, check_nan=False), + lambda r: r is None, + ), + # With check_nan=True (default) NaN is rejected even with a lower bound. + _Case( + "bounds_nan_guard_on_rejects", + "double", + float("nan"), + lambda f: check_bounds(F.col(f), ge=0, check_nan=True), + lambda r: r is not None and "NaN" in r, + ), + # check_nan=False is safe for integer columns; bound violations still fire. + _Case( + "bounds_int_rejects_violation", + "int", + 0, + lambda f: check_bounds(F.col(f), ge=1, check_nan=False), + lambda r: r is not None, + ), + _Case( + "bounds_int_accepts_valid", + "int", + 5, + lambda f: check_bounds(F.col(f), ge=1, le=10, check_nan=False), + lambda r: r is None, + ), + # --- check_enum ----------------------------------------------------------- + _Case( + "enum_valid", + "string", + "road", + lambda f: check_enum(F.col(f), ["road", "rail", "water"]), + lambda r: r is None, + ), + _Case( + "enum_invalid", + "string", + "sky", + lambda f: check_enum(F.col(f), ["road", "rail", "water"]), + lambda r: r is not None and "sky" in r, + ), + # --- check_pattern (generic) ---------------------------------------------- + _Case( + "pat_valid", + "string", + "AB", + lambda f: check_pattern(F.col(f), r"^[A-Z]{2}$", label="test pattern"), + lambda r: r is None, + ), + _Case( + "pat_invalid", + "string", + "abc", + lambda f: check_pattern(F.col(f), r"^[A-Z]{2}$", label="test pattern"), + lambda r: r is not None and "invalid test pattern" in r and "abc" in r, + ), + _Case( + "pat_null_passes", + "string", + None, + lambda f: check_pattern(F.col(f), r"^[A-Z]{2}$", label="test pattern"), + lambda r: r is None, + ), + # --- check_array_min_length ----------------------------------------------- + _Case( + "amin_at_limit", + "array", + ["a", "b"], + lambda f: check_array_min_length(F.col(f), 2), + lambda r: r is None, + ), + _Case( + "amin_below_limit", + "array", + ["a"], + lambda f: check_array_min_length(F.col(f), 2), + lambda r: r is not None and "minimum length 2" in r, + ), + _Case( + "amin_null_passthrough", + "array", + None, + lambda f: check_array_min_length(F.col(f), 2), + lambda r: r is None, + ), + # --- check_array_max_length ----------------------------------------------- + _Case( + "amax_within_limit", + "array", + ["a", "b"], + lambda f: check_array_max_length(F.col(f), 3), + lambda r: r is None, + ), + _Case( + "amax_at_limit", + "array", + ["a", "b"], + lambda f: check_array_max_length(F.col(f), 2), + lambda r: r is None, + ), + _Case( + "amax_exceeds_limit", + "array", + ["a", "b", "c"], + lambda f: check_array_max_length(F.col(f), 2), + lambda r: r is not None and "maximum length 2" in r, + ), + _Case( + "amax_null_passthrough", + "array", + None, + lambda f: check_array_max_length(F.col(f), 2), + lambda r: r is None, + ), + # --- check_require_any_of (multi-field -> struct column) ------------------ + _Case( + "rao_satisfied", + "struct", + {"a": 1, "b": None}, + lambda f: check_require_any_of([F.col(f)["a"], F.col(f)["b"]], ["a", "b"]), + lambda r: r is None, + ), + _Case( + "rao_all_null", + "struct", + {"a": None, "b": None}, + lambda f: check_require_any_of([F.col(f)["a"], F.col(f)["b"]], ["a", "b"]), + lambda r: r is not None and "a" in r and "b" in r, + ), + # --- check_require_any_true (multi-field -> struct column) ---------------- + # At least one condition true -> no error. + _Case( + "rat_one_true", + "struct", + {"is_land": True, "is_territorial": False}, + lambda f: check_require_any_true( + [ + F.col(f)["is_land"] == F.lit(True), + F.col(f)["is_territorial"] == F.lit(True), + ], + ["is_land", "is_territorial"], ), - ] -) - - -def test_check_bbox_completeness_valid(spark: SparkSession) -> None: - df = spark.createDataFrame( - [Row(bbox=Row(xmin=0.0, xmax=1.0, ymin=0.0, ymax=1.0))], - schema=_BBOX_SCHEMA, - ) - result = df.select(check_bbox_completeness(F.col("bbox")).alias("err")).collect() - assert result[0]["err"] is None - - -def test_check_bbox_completeness_null_bbox_passes(spark: SparkSession) -> None: - df = spark.createDataFrame([Row(bbox=None)], schema=_BBOX_SCHEMA) - result = df.select(check_bbox_completeness(F.col("bbox")).alias("err")).collect() - assert result[0]["err"] is None - - -def test_check_bbox_completeness_null_subfield_fails(spark: SparkSession) -> None: - df = spark.createDataFrame( - [Row(bbox=Row(xmin=None, xmax=1.0, ymin=0.0, ymax=1.0))], - schema=_BBOX_SCHEMA, - ) - result = df.select(check_bbox_completeness(F.col("bbox")).alias("err")).collect() - assert result[0]["err"] is not None + lambda r: r is None, + ), + # No condition true -> error naming the fields. + _Case( + "rat_all_false", + "struct", + {"is_land": False, "is_territorial": False}, + lambda f: check_require_any_true( + [ + F.col(f)["is_land"] == F.lit(True), + F.col(f)["is_territorial"] == F.lit(True), + ], + ["is_land", "is_territorial"], + ), + lambda r: r is not None and "is_land" in r and "is_territorial" in r, + ), + # Null fields mirror Python's `None == True` -> False: an all-null row violates. + _Case( + "rat_all_null", + "struct", + {"is_land": None, "is_territorial": None}, + lambda f: check_require_any_true( + [ + F.col(f)["is_land"] == F.lit(True), + F.col(f)["is_territorial"] == F.lit(True), + ], + ["is_land", "is_territorial"], + ), + lambda r: r is not None, + ), + # --- check_require_if (multi-field -> struct column) ---------------------- + # Target present when condition true -> no error. + _Case( + "rif_present", + "struct", + {"subtype": "road", "road_class": "primary"}, + lambda f: check_require_if( + F.col(f)["road_class"], + F.col(f)["subtype"].isin(["road", "rail"]), + "subtype in [road, rail]", + ), + lambda r: r is None, + ), + # Target null when condition true -> error. + _Case( + "rif_absent", + "struct", + {"subtype": "road", "road_class": None}, + lambda f: check_require_if( + F.col(f)["road_class"], + F.col(f)["subtype"].isin(["road", "rail"]), + "subtype in [road, rail]", + ), + lambda r: r is not None and "required" in r, + ), + # Target null but condition false -> no error. + _Case( + "rif_condition_false", + "struct", + {"subtype": "water", "road_class": None}, + lambda f: check_require_if( + F.col(f)["road_class"], + F.col(f)["subtype"].isin(["road", "rail"]), + "subtype in [road, rail]", + ), + lambda r: r is None, + ), + # Error message includes the actual discriminator value. + _Case( + "rif_value_cols", + "struct", + {"subtype": "road", "road_class": None}, + lambda f: check_require_if( + F.col(f)["road_class"], + F.col(f)["subtype"].isin(["road", "rail"]), + "subtype in [road, rail]", + F.col(f)["subtype"], + ), + lambda r: r is not None and "road" in r, + ), + # --- check_forbid_if (multi-field -> struct column) ----------------------- + # Target null when condition true -> no error. + _Case( + "fif_absent", + "struct", + {"subtype": "country", "parent": None}, + lambda f: check_forbid_if( + F.col(f)["parent"], + F.col(f)["subtype"] == "country", + "subtype = country", + ), + lambda r: r is None, + ), + # Target present when condition true -> error. + _Case( + "fif_present", + "struct", + {"subtype": "country", "parent": "abc"}, + lambda f: check_forbid_if( + F.col(f)["parent"], + F.col(f)["subtype"] == "country", + "subtype = country", + ), + lambda r: r is not None and "forbidden" in r, + ), + # Target present but condition false -> no error. + _Case( + "fif_condition_false", + "struct", + {"subtype": "region", "parent": "abc"}, + lambda f: check_forbid_if( + F.col(f)["parent"], + F.col(f)["subtype"] == "country", + "subtype = country", + ), + lambda r: r is None, + ), + # Error message includes the actual discriminator value. + _Case( + "fif_value_cols", + "struct", + {"subtype": "country", "parent": "abc"}, + lambda f: check_forbid_if( + F.col(f)["parent"], + F.col(f)["subtype"] == "country", + "subtype = country", + F.col(f)["subtype"], + ), + lambda r: r is not None and "country" in r, + ), + # --- check_string_min_length ---------------------------------------------- + _Case( + "smin_valid", + "string", + "abc", + lambda f: check_string_min_length(F.col(f), 1), + lambda r: r is None, + ), + _Case( + "smin_empty_violation", + "string", + "", + lambda f: check_string_min_length(F.col(f), 1), + lambda r: r is not None and "minimum length" in r, + ), + _Case( + "smin_null_passthrough", + "string", + None, + lambda f: check_string_min_length(F.col(f), 1), + lambda r: r is None, + ), + _Case( + "smin_exact", + "string", + "ab", + lambda f: check_string_min_length(F.col(f), 2), + lambda r: r is None, + ), + _Case( + "smin_below", + "string", + "a", + lambda f: check_string_min_length(F.col(f), 2), + lambda r: r is not None, + ), + # --- check_string_max_length ---------------------------------------------- + _Case( + "smax_valid", + "string", + "abc", + lambda f: check_string_max_length(F.col(f), 5), + lambda r: r is None, + ), + _Case( + "smax_above", + "string", + "abcdef", + lambda f: check_string_max_length(F.col(f), 5), + lambda r: r is not None and "maximum length" in r, + ), + _Case( + "smax_null_passthrough", + "string", + None, + lambda f: check_string_max_length(F.col(f), 5), + lambda r: r is None, + ), + _Case( + "smax_exact", + "string", + "abcde", + lambda f: check_string_max_length(F.col(f), 5), + lambda r: r is None, + ), + # --- check_radio_group (multi-field -> struct column) --------------------- + _Case( + "rg_exactly_one", + "struct", + {"is_land": True, "is_territorial": False}, + lambda f: check_radio_group( + [F.col(f)["is_land"], F.col(f)["is_territorial"]], + ["is_land", "is_territorial"], + ), + lambda r: r is None, + ), + _Case( + "rg_none_true", + "struct", + {"is_land": False, "is_territorial": False}, + lambda f: check_radio_group( + [F.col(f)["is_land"], F.col(f)["is_territorial"]], + ["is_land", "is_territorial"], + ), + lambda r: r is not None and "exactly one" in r and "0" in r, + ), + _Case( + "rg_both_true", + "struct", + {"is_land": True, "is_territorial": True}, + lambda f: check_radio_group( + [F.col(f)["is_land"], F.col(f)["is_territorial"]], + ["is_land", "is_territorial"], + ), + lambda r: r is not None and "2" in r, + ), + # Null booleans count as not-true (0 toward the count). + _Case( + "rg_null_as_false", + "struct", + {"is_land": True, "is_territorial": None}, + lambda f: check_radio_group( + [F.col(f)["is_land"], F.col(f)["is_territorial"]], + ["is_land", "is_territorial"], + ), + lambda r: r is None, + ), + # --- check_geometry_type (WKB in a binary column) ------------------------- + _Case( + "geom_point_matches", + "binary", + bytearray(Point(0, 0).wkb), + lambda f: check_geometry_type(F.col(f), GeometryType.POINT), + lambda r: r is None, + ), + _Case( + "geom_point_rejects_line", + "binary", + bytearray(LineString([(0, 0), (1, 1)]).wkb), + lambda f: check_geometry_type(F.col(f), GeometryType.POINT), + lambda r: r is not None and "Point" in r, + ), + # Multiple allowed types: a polygon and a multipolygon both pass (one row each). + _Case( + "geom_multi_polygon_ok", + "binary", + bytearray(Polygon([(0, 0), (1, 0), (1, 1), (0, 0)]).wkb), + lambda f: check_geometry_type( + F.col(f), GeometryType.POLYGON, GeometryType.MULTI_POLYGON + ), + lambda r: r is None, + ), + _Case( + "geom_multi_multipolygon_ok", + "binary", + bytearray(MultiPolygon([Polygon([(0, 0), (1, 0), (1, 1), (0, 0)])]).wkb), + lambda f: check_geometry_type( + F.col(f), GeometryType.POLYGON, GeometryType.MULTI_POLYGON + ), + lambda r: r is None, + ), + _Case( + "geom_multi_rejects_point", + "binary", + bytearray(Point(0, 0).wkb), + lambda f: check_geometry_type( + F.col(f), GeometryType.POLYGON, GeometryType.MULTI_POLYGON + ), + lambda r: r is not None, + ), + _Case( + "geom_null_passthrough", + "binary", + None, + lambda f: check_geometry_type(F.col(f), GeometryType.POINT), + lambda r: r is None, + ), + # BE byte order: byte_order=0x00, type=0x00000001, x=0.0, y=0.0. + _Case( + "geom_big_endian", + "binary", + bytearray(struct.pack(">bIdd", 0, 1, 0.0, 0.0)), + lambda f: check_geometry_type(F.col(f), GeometryType.POINT), + lambda r: r is None, + ), + # ISO WKB encodes Z by offsetting the type (PointZ=1001); must validate by base type. + _Case( + "geom_iso_z_point", + "binary", + bytearray(struct.pack("bIddd", 0, 1001, 0.0, 0.0, 5.0)), + lambda f: check_geometry_type(F.col(f), GeometryType.POINT), + lambda r: r is None, + ), + # EWKB encodes Z as a high flag bit (0x80000001), leaving the low byte at 0x01. + _Case( + "geom_ewkb_z_point", + "binary", + bytearray(struct.pack("", + [0.0, 1.0], + lambda f: check_linear_range_length(F.col(f)), + lambda r: r is None, + ), + _Case( + "lrl_one", + "array", + [0.5], + lambda f: check_linear_range_length(F.col(f)), + lambda r: r is not None and "2 elements" in r, + ), + _Case( + "lrl_three", + "array", + [0.0, 0.5, 1.0], + lambda f: check_linear_range_length(F.col(f)), + lambda r: r is not None and "2 elements" in r, + ), + _Case( + "lrl_empty", + "array", + [], + lambda f: check_linear_range_length(F.col(f)), + lambda r: r is not None and "2 elements" in r, + ), + _Case( + "lrl_null", + "array", + None, + lambda f: check_linear_range_length(F.col(f)), + lambda r: r is None, + ), + # --- check_linear_range_bounds -------------------------------------------- + _Case( + "lrb_valid", + "array", + [0.2, 0.8], + lambda f: check_linear_range_bounds(F.col(f)), + lambda r: r is None, + ), + _Case( + "lrb_below_zero", + "array", + [-0.1, 0.5], + lambda f: check_linear_range_bounds(F.col(f)), + lambda r: r is not None and "[0.0, 1.0]" in r, + ), + _Case( + "lrb_above_one", + "array", + [0.0, 1.1], + lambda f: check_linear_range_bounds(F.col(f)), + lambda r: r is not None and "[0.0, 1.0]" in r, + ), + # Wrong-length arrays are not this function's concern. + _Case( + "lrb_wrong_length_passthrough", + "array", + [0.5], + lambda f: check_linear_range_bounds(F.col(f)), + lambda r: r is None, + ), + _Case( + "lrb_null", + "array", + None, + lambda f: check_linear_range_bounds(F.col(f)), + lambda r: r is None, + ), + # --- check_linear_range_order --------------------------------------------- + _Case( + "lro_valid", + "array", + [0.2, 0.8], + lambda f: check_linear_range_order(F.col(f)), + lambda r: r is None, + ), + _Case( + "lro_equal", + "array", + [0.5, 0.5], + lambda f: check_linear_range_order(F.col(f)), + lambda r: r is not None and "start must be < end" in r, + ), + _Case( + "lro_after", + "array", + [0.8, 0.2], + lambda f: check_linear_range_order(F.col(f)), + lambda r: r is not None and "start must be < end" in r, + ), + # Wrong-length arrays are not this function's concern. + _Case( + "lro_wrong_length_passthrough", + "array", + [0.5], + lambda f: check_linear_range_order(F.col(f)), + lambda r: r is None, + ), + _Case( + "lro_null", + "array", + None, + lambda f: check_linear_range_order(F.col(f)), + lambda r: r is None, + ), + # --- check_required ------------------------------------------------------- + _Case( + "req_null_is_error", + "string", + None, + lambda f: check_required(F.col(f)), + lambda r: r is not None and "missing" in r, + ), + _Case( + "req_non_null_passes", + "string", + "hello", + lambda f: check_required(F.col(f)), + lambda r: r is None, + ), + # check_required + check_enum via F.coalesce catches both null and invalid. + _Case( + "req_composes_with_enum", + "string", + None, + lambda f: F.coalesce( + check_required(F.col(f)), check_enum(F.col(f), ["a", "b"]) + ), + lambda r: r is not None and "missing" in r, + ), + # --- country code via check_pattern --------------------------------------- + _Case( + "cc_valid", + "string", + "US", + lambda f: check_pattern( + F.col(f), _COUNTRY_CODE_PATTERN, label=_COUNTRY_CODE_LABEL + ), + lambda r: r is None, + ), + _Case( + "cc_lowercase_invalid", + "string", + "us", + lambda f: check_pattern( + F.col(f), _COUNTRY_CODE_PATTERN, label=_COUNTRY_CODE_LABEL + ), + lambda r: r is not None and f"invalid {_COUNTRY_CODE_LABEL}" in r and "us" in r, + ), + _Case( + "cc_three_chars_invalid", + "string", + "USA", + lambda f: check_pattern( + F.col(f), _COUNTRY_CODE_PATTERN, label=_COUNTRY_CODE_LABEL + ), + lambda r: r is not None, + ), + _Case( + "cc_null_passes", + "string", + None, + lambda f: check_pattern( + F.col(f), _COUNTRY_CODE_PATTERN, label=_COUNTRY_CODE_LABEL + ), + lambda r: r is None, + ), + # --- region code via check_pattern ---------------------------------------- + _Case( + "rc_valid", + "string", + "US-NY", + lambda f: check_pattern( + F.col(f), _REGION_CODE_PATTERN, label=_REGION_CODE_LABEL + ), + lambda r: r is None, + ), + _Case( + "rc_valid_numeric", + "string", + "CN-11", + lambda f: check_pattern( + F.col(f), _REGION_CODE_PATTERN, label=_REGION_CODE_LABEL + ), + lambda r: r is None, + ), + _Case( + "rc_no_dash_invalid", + "string", + "USNY", + lambda f: check_pattern( + F.col(f), _REGION_CODE_PATTERN, label=_REGION_CODE_LABEL + ), + lambda r: ( + r is not None and f"invalid {_REGION_CODE_LABEL}" in r and "USNY" in r + ), + ), + _Case( + "rc_null_passes", + "string", + None, + lambda f: check_pattern( + F.col(f), _REGION_CODE_PATTERN, label=_REGION_CODE_LABEL + ), + lambda r: r is None, + ), + # --- snake_case via check_pattern ----------------------------------------- + _Case( + "sc_valid", + "string", + "hello_world", + lambda f: check_pattern(F.col(f), _SNAKE_CASE_PATTERN, label=_SNAKE_CASE_LABEL), + lambda r: r is None, + ), + _Case( + "sc_single_word", + "string", + "hello", + lambda f: check_pattern(F.col(f), _SNAKE_CASE_PATTERN, label=_SNAKE_CASE_LABEL), + lambda r: r is None, + ), + _Case( + "sc_with_numbers", + "string", + "hello_123", + lambda f: check_pattern(F.col(f), _SNAKE_CASE_PATTERN, label=_SNAKE_CASE_LABEL), + lambda r: r is None, + ), + _Case( + "sc_uppercase_invalid", + "string", + "Hello_World", + lambda f: check_pattern(F.col(f), _SNAKE_CASE_PATTERN, label=_SNAKE_CASE_LABEL), + lambda r: r is not None and f"invalid {_SNAKE_CASE_LABEL}" in r, + ), + _Case( + "sc_spaces_invalid", + "string", + "hello world", + lambda f: check_pattern(F.col(f), _SNAKE_CASE_PATTERN, label=_SNAKE_CASE_LABEL), + lambda r: r is not None, + ), + _Case( + "sc_null_passes", + "string", + None, + lambda f: check_pattern(F.col(f), _SNAKE_CASE_PATTERN, label=_SNAKE_CASE_LABEL), + lambda r: r is None, + ), + # --- check_url_format ----------------------------------------------------- + _Case( + "url_http_valid", + "string", + "http://example.com", + lambda f: check_url_format(F.col(f)), + lambda r: r is None, + ), + _Case( + "url_https_valid", + "string", + "https://example.com/path?q=1", + lambda f: check_url_format(F.col(f)), + lambda r: r is None, + ), + # Pydantic HttpUrl lowercases the scheme, so HTTP:// is accepted. + _Case( + "url_uppercase_scheme", + "string", + "HTTP://example.com", + lambda f: check_url_format(F.col(f)), + lambda r: r is None, + ), + _Case( + "url_mixed_case_scheme", + "string", + "HtTpS://example.com/path", + lambda f: check_url_format(F.col(f)), + lambda r: r is None, + ), + _Case( + "url_no_scheme_invalid", + "string", + "example.com", + lambda f: check_url_format(F.col(f)), + lambda r: r is not None, + ), + _Case( + "url_ftp_scheme_invalid", + "string", + "ftp://example.com", + lambda f: check_url_format(F.col(f)), + lambda r: r is not None, + ), + _Case( + "url_null_passes", + "string", + None, + lambda f: check_url_format(F.col(f)), + lambda r: r is None, + ), + # --- check_url_length ----------------------------------------------------- + _Case( + "urllen_exceeds", + "string", + "https://example.com/" + "a" * 2064, # 2084 chars + lambda f: check_url_length(F.col(f)), + lambda r: r is not None, + ), + _Case( + "urllen_exactly_2083", + "string", + "https://example.com/" + "a" * 2063, # 2083 chars + lambda f: check_url_length(F.col(f)), + lambda r: r is None, + ), + _Case( + "urllen_null_passes", + "string", + None, + lambda f: check_url_length(F.col(f)), + lambda r: r is None, + ), + # --- check_email ---------------------------------------------------------- + _Case( + "email_valid", + "string", + "user@example.com", + lambda f: check_email(F.col(f)), + lambda r: r is None, + ), + _Case( + "email_no_at", + "string", + "userexample.com", + lambda f: check_email(F.col(f)), + lambda r: r is not None, + ), + _Case( + "email_no_domain", + "string", + "user@", + lambda f: check_email(F.col(f)), + lambda r: r is not None, + ), + _Case( + "email_spaces", + "string", + "user @example.com", + lambda f: check_email(F.col(f)), + lambda r: r is not None, + ), + _Case( + "email_null", + "string", + None, + lambda f: check_email(F.col(f)), + lambda r: r is None, + ), + _Case( + "email_trailing_period", + "string", + "user@example.com.", + lambda f: check_email(F.col(f)), + lambda r: r is not None, + ), + _Case( + "email_leading_period", + "string", + ".user@example.com", + lambda f: check_email(F.col(f)), + lambda r: r is not None, + ), + _Case( + "email_period_before_at", + "string", + "user.@example.com", + lambda f: check_email(F.col(f)), + lambda r: r is not None, + ), + _Case( + "email_period_after_at", + "string", + "user@.example.com", + lambda f: check_email(F.col(f)), + lambda r: r is not None, + ), + _Case( + "email_double_period_domain", + "string", + "user@example..com", + lambda f: check_email(F.col(f)), + lambda r: r is not None, + ), + _Case( + "email_dotted_local_valid", + "string", + "user.name@example.com", + lambda f: check_email(F.col(f)), + lambda r: r is None, + ), + _Case( + "email_subdomain_valid", + "string", + "user@mail.example.co.uk", + lambda f: check_email(F.col(f)), + lambda r: r is None, + ), + # --- phone via check_pattern ---------------------------------------------- + _Case( + "phone_valid_us", + "string", + "+1 555-555-5555", + lambda f: check_pattern(F.col(f), _PHONE_PATTERN, label=_PHONE_LABEL), + lambda r: r is None, + ), + _Case( + "phone_valid_international", + "string", + "+44 20 7946 0958", + lambda f: check_pattern(F.col(f), _PHONE_PATTERN, label=_PHONE_LABEL), + lambda r: r is None, + ), + _Case( + "phone_no_plus_invalid", + "string", + "555-555-5555", + lambda f: check_pattern(F.col(f), _PHONE_PATTERN, label=_PHONE_LABEL), + lambda r: r is not None and f"invalid {_PHONE_LABEL}" in r, + ), + _Case( + "phone_letters_invalid", + "string", + "+1 abc-defg", + lambda f: check_pattern(F.col(f), _PHONE_PATTERN, label=_PHONE_LABEL), + lambda r: r is not None, + ), + _Case( + "phone_null_passes", + "string", + None, + lambda f: check_pattern(F.col(f), _PHONE_PATTERN, label=_PHONE_LABEL), + lambda r: r is None, + ), + # --- wikidata id via check_pattern ---------------------------------------- + _Case( + "wd_valid", + "string", + "Q42", + lambda f: check_pattern(F.col(f), _WIKIDATA_PATTERN, label=_WIKIDATA_LABEL), + lambda r: r is None, + ), + _Case( + "wd_large_number", + "string", + "Q123456789", + lambda f: check_pattern(F.col(f), _WIKIDATA_PATTERN, label=_WIKIDATA_LABEL), + lambda r: r is None, + ), + _Case( + "wd_lowercase_q_invalid", + "string", + "q42", + lambda f: check_pattern(F.col(f), _WIKIDATA_PATTERN, label=_WIKIDATA_LABEL), + lambda r: r is not None and f"invalid {_WIKIDATA_LABEL}" in r, + ), + _Case( + "wd_no_digits_invalid", + "string", + "Q", + lambda f: check_pattern(F.col(f), _WIKIDATA_PATTERN, label=_WIKIDATA_LABEL), + lambda r: r is not None, + ), + _Case( + "wd_p_prefix_invalid", + "string", + "P42", + lambda f: check_pattern(F.col(f), _WIKIDATA_PATTERN, label=_WIKIDATA_LABEL), + lambda r: r is not None, + ), + _Case( + "wd_null_passes", + "string", + None, + lambda f: check_pattern(F.col(f), _WIKIDATA_PATTERN, label=_WIKIDATA_LABEL), + lambda r: r is None, + ), + # --- check_min_fields_set (multi-field -> struct column) ------------------ + # Count at threshold -> no error. + _Case( + "mfs_meets_threshold", + "struct", + {"a": 1, "b": 2, "c": None}, + lambda f: check_min_fields_set( + [F.col(f)["a"], F.col(f)["b"], F.col(f)["c"]], ["a", "b", "c"], 2 + ), + lambda r: r is None, + ), + # Count above threshold -> no error. + _Case( + "mfs_exceeds_threshold", + "struct", + {"a": 1, "b": 2, "c": 3}, + lambda f: check_min_fields_set( + [F.col(f)["a"], F.col(f)["b"], F.col(f)["c"]], ["a", "b", "c"], 2 + ), + lambda r: r is None, + ), + # Count below threshold -> error with field names and actual count. + _Case( + "mfs_below_threshold", + "struct", + {"a": 1, "b": None, "c": None}, + lambda f: check_min_fields_set( + [F.col(f)["a"], F.col(f)["b"], F.col(f)["c"]], ["a", "b", "c"], 2 + ), + lambda r: r is not None and "at least 2" in r and "a, b, c" in r and "1" in r, + ), + # All null -> error showing 0 non-null. + _Case( + "mfs_all_null", + "struct", + {"a": None, "b": None}, + lambda f: check_min_fields_set([F.col(f)["a"], F.col(f)["b"]], ["a", "b"], 1), + lambda r: r is not None and "0" in r, + ), + # Error message matches the expected format exactly. + _Case( + "mfs_message_format", + "struct", + {"x": None, "y": None}, + lambda f: check_min_fields_set([F.col(f)["x"], F.col(f)["y"]], ["x", "y"], 1), + lambda r: r == "at least 1 of x, y required, got 0 non-null", + ), + # --- check_bbox_completeness (bbox struct column) ------------------------- + _Case( + "bbox_valid", + "struct", + {"xmin": 0.0, "xmax": 1.0, "ymin": 0.0, "ymax": 1.0}, + lambda f: check_bbox_completeness(F.col(f)), + lambda r: r is None, + ), + _Case( + "bbox_null_passes", + "struct", + None, + lambda f: check_bbox_completeness(F.col(f)), + lambda r: r is None, + ), + _Case( + "bbox_null_subfield_fails", + "struct", + {"xmin": None, "xmax": 1.0, "ymin": 0.0, "ymax": 1.0}, + lambda f: check_bbox_completeness(F.col(f)), + lambda r: r is not None, + ), +] + + +@pytest.fixture(scope="module") +def results(spark: SparkSession) -> Any: + """Pack every case's input into one row, apply every check, collect once.""" + # A DDL schema string, not StructType.fromDDL -- fromDDL landed in PySpark + # 3.5, and createDataFrame parses the string itself on the >=3.4 floor. + schema = ", ".join(f"`{c.id}` {c.ddl}" for c in _CASES) + row = {c.id: c.value for c in _CASES} + # dict rows are read by field name against the explicit schema, a form the + # createDataFrame stubs don't model (they want tuple/Row for RowLike). + df = spark.createDataFrame([row], schema=schema, verifySchema=False) # type: ignore[call-overload] + return df.select(*[c.check(c.id).alias(c.id) for c in _CASES]).collect()[0] + + +@pytest.mark.parametrize("case", _CASES, ids=lambda c: c.id) +def test_constraint_expression(case: _Case, results: Any) -> None: + value = results[case.id] + assert case.expect(value), f"{case.id}: got {value!r}" diff --git a/packages/overture-schema-pyspark/tests/test_helpers.py b/packages/overture-schema-pyspark/tests/test_helpers.py index b202ce3d1..ad2937745 100644 --- a/packages/overture-schema-pyspark/tests/test_helpers.py +++ b/packages/overture-schema-pyspark/tests/test_helpers.py @@ -112,6 +112,98 @@ def test_chained_calls_through_deep_null_path(self) -> None: assert with_value["outer"][0]["inner"][0]["value"] is None +class TestSetAtPathMapProjection: + """A trailing `{value}` / `{key}` marker mutates the map's single entry. + + An array-first map leaf (`items[].tags{value}`) descends the array to + element 0, then corrupts the inner map in place -- preserving the other + side of the entry so the check under test is the only violation. + """ + + def test_map_value_in_array(self) -> None: + row = {"items": [{"tags": {"k": "abc"}}]} + result = set_at_path("items[].tags{value}", "XY")(row) + assert result["items"][0]["tags"] == {"k": "XY"} + + def test_map_value_does_not_mutate_original(self) -> None: + row = {"items": [{"tags": {"k": "abc"}}]} + set_at_path("items[].tags{value}", "XY")(row) + assert row["items"][0]["tags"] == {"k": "abc"} + + def test_map_key_in_array_preserves_value(self) -> None: + row = {"items": [{"tags": {"k": "abc"}}]} + result = set_at_path("items[].tags{key}", "BAD")(row) + assert result["items"][0]["tags"] == {"BAD": "abc"} + + def test_map_value_top_level(self) -> None: + row = {"tags": {"k": "abc"}} + result = set_at_path("tags{value}", "XY")(row) + assert result["tags"] == {"k": "XY"} + + def test_raises_on_missing_map(self) -> None: + row: dict[str, Any] = {"items": [{"tags": None}]} + with pytest.raises(PathTraversalError, match="tags"): + set_at_path("items[].tags{value}", "XY")(row) + + +class TestSetAtPathNonTerminalProjection: + """A non-terminal `{value}` descends the sole entry's value and continues. + + `subs{value}[]` (dict[str, list[X]]) projects to the sole map value, then + indexes element 0 of that list; `subs{value}{value}` (dict[str, dict[str, + X]]) projects twice, reaching the inner map's sole value. A non-terminal + `{key}` can't be descended -- a key is an immutable scalar -- so it raises. + """ + + def test_map_value_then_array(self) -> None: + row = {"subs": {"k": ["abc"]}} + result = set_at_path("subs{value}[]", "")(row) + assert result["subs"]["k"] == [""] + + def test_map_value_then_map_value(self) -> None: + row = {"subs": {"k": {"j": 7}}} + result = set_at_path("subs{value}{value}", -1)(row) + assert result["subs"]["k"] == {"j": -1} + + def test_map_value_then_array_does_not_mutate_original(self) -> None: + row = {"subs": {"k": ["abc"]}} + set_at_path("subs{value}[]", "")(row) + assert row["subs"]["k"] == ["abc"] + + def test_map_value_then_map_value_does_not_mutate_original(self) -> None: + row = {"subs": {"k": {"j": 7}}} + set_at_path("subs{value}{value}", -1)(row) + assert row["subs"]["k"] == {"j": 7} + + def test_map_value_preserves_sibling_map_entries(self) -> None: + row = {"subs": {"k": ["abc"], "other": ["keep"]}} + result = set_at_path("subs{value}[]", "")(row) + assert result["subs"]["other"] == ["keep"] + + def test_non_terminal_key_projection_raises(self) -> None: + row = {"subs": {"k": ["abc"]}} + with pytest.raises(PathTraversalError, match="key"): + set_at_path("subs{key}[]", "")(row) + + def test_missing_map_at_non_terminal_raises(self) -> None: + row: dict[str, Any] = {"subs": None} + with pytest.raises(PathTraversalError, match="subs"): + set_at_path("subs{value}[]", "")(row) + + def test_map_value_then_two_array_levels(self) -> None: + """`subs{value}[][]` (dict[str, list[list[X]]]) descends map, then two + anonymous array levels, each indexing element 0. + + Backs the report's "generalizes for free" claim: no code path is + specific to a single trailing array level, since `_array_slot` + handles an anonymous segment identically regardless of how many + precede it. + """ + row = {"subs": {"k": [["abc", "def"]]}} + result = set_at_path("subs{value}[][]", "")(row) + assert result["subs"]["k"] == [["", "def"]] + + class TestDeepMerge: def test_flat_merge(self) -> None: base = {"a": 1, "b": 2} diff --git a/packages/overture-schema-pyspark/tests/test_mutations.py b/packages/overture-schema-pyspark/tests/test_mutations.py index 707ab0f41..9a3430b0c 100644 --- a/packages/overture-schema-pyspark/tests/test_mutations.py +++ b/packages/overture-schema-pyspark/tests/test_mutations.py @@ -273,7 +273,7 @@ def test_struct_path_descends_into_map_value(self) -> None: ) assert result["subs"]["en"]["inner"] == {"foo": None, "bar": None} - def test_nested_map_column_path(self) -> None: + def test_dotted_map_path(self) -> None: row = {"outer": {"subs": {"en": {"foo": 1, "bar": 2}}}} result = mutate_require_any_of(row, ["foo", "bar"], map_path="outer.subs") assert result["outer"]["subs"]["en"] == {"foo": None, "bar": None} @@ -316,6 +316,150 @@ def test_forbid_if_sets_condition_and_ensures_non_null_in_map_value(self) -> Non assert value["admin_level"] is not None +class TestMutateCompositeElementPath: + """`element_path` threads a model mutation through mixed map/array nesting. + + Neither a scalar `array_path` nor `map_path` expresses a container-after- + container descent. `element_path` carries the full descent to the target + model, walked generically: array segments iterate every element, map + `{value}` segments take the sole entry's value, struct segments navigate a + field. It composes in either order -- map-then-array (`subs{value}[]`, + dict[K, list[Model]]) and array-then-map (`items[].configs{value}`, + list[dict[K, Model]]) -- reaching the model where fields are nulled. With + only struct segments (`details`) it descends a plain struct-nested model. + """ + + def test_require_any_of_plain_struct(self) -> None: + """A struct-only `element_path` nulls a struct-nested model's fields. + + A model constraint on a submodel reached through a plain struct field + (`details`, no array or map) emits `element_path="details"`; the descent + navigates the struct and nulls each field in place. + """ + row = {"details": {"foo": 1, "bar": "x"}} + result = mutate_require_any_of(row, ["foo", "bar"], element_path="details") + assert result["details"] == {"foo": None, "bar": None} + + def test_require_any_of_map_then_array(self) -> None: + row = {"subs": {"k": [{"foo": 1, "bar": 2}]}} + result = mutate_require_any_of( + row, ["foo", "bar"], element_path="subs{value}[]" + ) + assert result["subs"]["k"] == [{"foo": None, "bar": None}] + + def test_require_any_of_map_then_array_preserves_map_key(self) -> None: + row = {"subs": {"k": [{"foo": 1, "bar": 2}]}} + result = mutate_require_any_of( + row, ["foo", "bar"], element_path="subs{value}[]" + ) + assert list(result["subs"]) == ["k"] + + def test_require_any_of_array_then_map(self) -> None: + row = {"items": [{"configs": {"k": {"foo": 1, "bar": 2}}}]} + result = mutate_require_any_of( + row, ["foo", "bar"], element_path="items[].configs{value}" + ) + assert result["items"][0]["configs"]["k"] == {"foo": None, "bar": None} + + def test_require_any_of_array_then_map_nulls_every_element(self) -> None: + row = { + "items": [ + {"configs": {"k": {"foo": 1, "bar": 2}}}, + {"configs": {"j": {"foo": 3, "bar": 4}}}, + ] + } + result = mutate_require_any_of( + row, ["foo", "bar"], element_path="items[].configs{value}" + ) + assert result["items"][0]["configs"]["k"] == {"foo": None, "bar": None} + assert result["items"][1]["configs"]["j"] == {"foo": None, "bar": None} + + def test_require_any_of_composite_does_not_mutate_original(self) -> None: + row = {"subs": {"k": [{"foo": 1, "bar": 2}]}} + mutate_require_any_of(row, ["foo", "bar"], element_path="subs{value}[]") + assert row["subs"]["k"] == [{"foo": 1, "bar": 2}] + + def test_min_fields_set_composite_descent(self) -> None: + row = {"subs": {"k": [{"a": 1, "b": 2}]}} + result = mutate_min_fields_set(row, ["a", "b"], element_path="subs{value}[]") + assert result["subs"]["k"] == [{"a": None, "b": None}] + + def test_require_if_array_then_map(self) -> None: + row = {"items": [{"configs": {"k": {"subtype": "other", "admin_level": 5}}}]} + result = mutate_require_if( + row, + ["admin_level"], + "subtype", + "country", + element_path="items[].configs{value}", + ) + value = result["items"][0]["configs"]["k"] + assert value["subtype"] == "country" + assert value["admin_level"] is None + + def test_require_any_of_map_then_array_stubs_absent_map(self) -> None: + """An absent map-then-array target stubs a list-shaped map value. + + `subs{value}[]` (`dict[K, list[Model]]`) with no `subs` key at all: + the map's sole stubbed entry must itself be a list, since the map + value type is `list[Model]`, not `Model`. Before the shape-aware + stub, `_element_map_value` always stubbed a dict, and the trailing + anonymous array segment then required its parent to already be a + non-empty list -- raising `PathTraversalError` on this absent-map + case instead of producing a mutated row. + """ + row: dict = {} + result = mutate_require_any_of( + row, ["foo", "bar"], element_path="subs{value}[]" + ) + value = next(iter(result["subs"].values())) + assert value == [{"foo": None, "bar": None}] + + def test_require_any_of_map_then_array_then_struct_leaf(self) -> None: + """`subs{value}[].inner` (map, array, then a struct field) also composes. + + Backs the report's "generalizes for free" claim for a composite + descent with a trailing struct leaf, on both a present and an absent + map -- the absent case exercises the shape-aware stub together with + `_scaffold_struct_child`'s struct navigation. + """ + row = {"subs": {"k": [{"inner": {"foo": 1, "bar": 2}}]}} + result = mutate_require_any_of( + row, ["foo", "bar"], element_path="subs{value}[].inner" + ) + assert result["subs"]["k"] == [{"inner": {"foo": None, "bar": None}}] + + def test_require_any_of_map_then_array_then_struct_leaf_stubs_absent_map( + self, + ) -> None: + row: dict = {} + result = mutate_require_any_of( + row, ["foo", "bar"], element_path="subs{value}[].inner" + ) + value = next(iter(result["subs"].values())) + assert value == [{"inner": {"foo": None, "bar": None}}] + + def test_forbid_if_composite_element_path_array_then_map(self) -> None: + """`forbid_if` walks a composite `element_path` (array-then-map) too. + + Backs the report's "generalizes for free" claim: `mutate_forbid_if` + takes the same `element_path` kwarg as `mutate_require_any_of` / + `mutate_require_if`, threaded through the same `_apply_to_targets` -> + `_descend_to_targets` machinery. + """ + row = {"items": [{"configs": {"k": {"subtype": "other", "extra": None}}}]} + result = mutate_forbid_if( + row, + ["extra"], + "subtype", + "country", + element_path="items[].configs{value}", + ) + value = result["items"][0]["configs"]["k"] + assert value["subtype"] == "country" + assert value["extra"] is not None + + class TestMutateForbidIfNegate: def test_negate_changes_condition_value(self) -> None: """negate=True sets condition_field to something != condition_value.""" diff --git a/packages/overture-schema-system/src/overture/schema/system/field_path.py b/packages/overture-schema-system/src/overture/schema/system/field_path.py index df06232fe..aef91e055 100644 --- a/packages/overture-schema-system/src/overture/schema/system/field_path.py +++ b/packages/overture-schema-system/src/overture/schema/system/field_path.py @@ -1,49 +1,61 @@ """Structural representation of a field path through a nested schema. -A `FieldPath` is one of three variants: - -- `ScalarPath` -- a sequence of `StructSegment` values locating a value - that requires no iteration to reach. -- `ArrayPath` -- a sequence of `StructSegment` and `ArraySegment` values, - with at least one `ArraySegment`, locating a value reached by iterating - one or more arrays. Each `ArraySegment` carries `iter_count`, the number - of `[]` markers on its name in the canonical encoding (multi-depth - segments encode nested-list iteration without an intervening struct, - e.g. `list[list[X]]` parses as a single `ArraySegment` with - `iter_count=2`). -- `MapPath` -- struct segments leading to a map column, a single - `MapSegment` projecting the map to its keys or values, then a struct-only - leaf (possibly empty). Locates a value reached by iterating a - `dict[K, V]`'s keys or values, encoded with a `{key}` / `{value}` marker - on the map column and the leaf appended after it (e.g. `names.common{key}` - for a scalar value, `subs{value}.label` for a field inside a - `dict[K, Model]` value). - -The canonical string form (`str(path)`) round-trips through `parse`. -Code that needs to emit a path into source or labels calls `str(path)` -at the boundary; everything else operates on segments. +A `FieldPath` is one of two variants: + +- `Direct` -- a sequence of `StructSegment` values locating a value that + requires no iteration to reach. +- `Iterated` -- a sequence of `StructSegment`, `ArraySegment`, and + `MapSegment` values with at least one iterating (`Array`/`Map`) segment, + locating a value reached by iterating one or more arrays or maps. Each + iterating segment is exactly one iteration frame. A container nested + directly inside another container with no field name between (e.g. + `list[list[X]]`, `dict[K, list[X]]`) is an *anonymous* iterating segment + -- an `ArraySegment` or `MapSegment` whose `name` is empty, meaning "the + parent element is itself this container." Anonymity is read through + `is_anonymous`, never `name == ""` directly. + +Examples map a Pydantic field annotation to the canonical string form of +a check on its innermost value: + + x: int -> "x" (Direct) + parent: Parent (field `value`) -> "parent.value" (Direct) + items: list[Item] (field `v`) -> "items[].v" (Iterated) + tags: dict[str, str] (values) -> "tags{value}" (Iterated) + tags: dict[str, str] (keys) -> "tags{key}" (Iterated) + grid: list[list[int]] -> "grid[][]" (Iterated) + subs: dict[str, list[X]] (values) -> "subs{value}[]" (Iterated) + items: list[dict[str, X]] (values) -> "items[]{value}" (Iterated) + +In the multi-container forms the first marker is a *named* segment (it +carries the field name) and each trailing marker is *anonymous* -- the +parent element is itself the next container, so no field name separates +them. The canonical string (`str(path)`) round-trips through `parse`; +the `[]` / `{key}` / `{value}` sugar is unchanged, so existing encodings +round-trip and only the internal segment list generalizes. Code that +emits a path into source or labels calls `str(path)` at that boundary; +everything else operates on segments. """ from __future__ import annotations +import re from dataclasses import dataclass from enum import Enum from typing import TypeAlias __all__ = [ - "ArrayPath", "ArraySegment", + "Direct", "FieldPath", "FieldSegment", - "MapPath", + "Iterated", "MapProjection", "MapSegment", - "ScalarPath", "StructSegment", "coerce", "parse", - "promote_terminal_array", - "promote_terminal_map", + "promote_terminal", + "terminal_run_start", ] @@ -56,15 +68,19 @@ class StructSegment: @dataclass(frozen=True, slots=True) class ArraySegment: - """An array column entered with one or more levels of iteration. + """An array column entered with one level of iteration. - `iter_count` records the number of `[]` markers immediately following - the segment name; values > 1 correspond to nested lists like - `list[list[X]]`. + An empty `name` marks an *anonymous* segment: the parent element is + itself a list (`list[list[...]]`), iterated once more with no field + navigation. Read anonymity via `is_anonymous`, never `name == ""`. """ - name: str - iter_count: int = 1 + name: str = "" + + @property + def is_anonymous(self) -> bool: + """Whether this segment is a nameless extra iteration of the prior container.""" + return self.name == "" class MapProjection(Enum): @@ -80,121 +96,197 @@ class MapSegment: `projection` selects keys or values; the projected side is iterated like an array, so checks on a `MapSegment` render through the same - element machinery as `ArraySegment`. + element machinery as `ArraySegment`. An empty `name` marks an + *anonymous* segment: the parent element is itself a map + (`dict[K, dict[K2, V]]`, `list[dict]`), projected once more with no + field navigation. Read anonymity via `is_anonymous`, never `name == ""`. """ name: str projection: MapProjection + @property + def is_anonymous(self) -> bool: + """Whether this segment is a nameless extra projection of the prior container.""" + return self.name == "" + + +# The element type of any `FieldPath.segments`. A `Direct` holds only +# `StructSegment`s; an `Iterated` mixes all three. +FieldSegment: TypeAlias = StructSegment | ArraySegment | MapSegment + + +def _is_iterating(seg: FieldSegment) -> bool: + """Whether *seg* is an iterating (`Array`/`Map`) segment.""" + return isinstance(seg, (ArraySegment, MapSegment)) + + +def _first_iterating(segments: tuple[FieldSegment, ...]) -> ArraySegment | MapSegment: + """Return the first iterating segment (Array/Map) in *segments*. + + Callers guarantee at least one exists (the `Iterated` invariant). + """ + for seg in segments: + if isinstance(seg, (ArraySegment, MapSegment)): + return seg + raise AssertionError("no iterating segment; caller violated the invariant") + @dataclass(frozen=True, slots=True) -class ScalarPath: +class Direct: """Locate a non-iterated value in a row.""" segments: tuple[StructSegment, ...] = () - def append_struct(self, name: str) -> ScalarPath: - return ScalarPath(segments=self.segments + (StructSegment(name=name),)) + def append_struct(self, name: str) -> Direct: + """Return a new `Direct` with *name* appended as a struct segment.""" + return Direct(segments=self.segments + (StructSegment(name=name),)) - def append_array(self, name: str, iter_count: int = 1) -> ArrayPath: - return ArrayPath( - segments=self.segments + (ArraySegment(name=name, iter_count=iter_count),) - ) + def append_array(self, name: str) -> Iterated: + """Return an `Iterated` with *name* appended as a named array segment.""" + return Iterated(segments=self.segments + (ArraySegment(name=name),)) def __str__(self) -> str: return ".".join(s.name for s in self.segments) @dataclass(frozen=True, slots=True) -class ArrayPath: +class Iterated: """Locate an iterated value; iteration structure is part of the location. - Invariant: `segments` contains at least one `ArraySegment`. + Segments mix `StructSegment`, `ArraySegment`, and `MapSegment`. + Invariants (enforced in `__post_init__`): at least one iterating + (`Array`/`Map`) segment, and the first iterating segment is named + (anonymity only ever follows another iterating segment). A struct + prefix before the first iterating segment is allowed (e.g. + `parent.items[].value`). """ - segments: tuple[StructSegment | ArraySegment, ...] + segments: tuple[FieldSegment, ...] def __post_init__(self) -> None: - if not any(isinstance(s, ArraySegment) for s in self.segments): - raise ValueError("ArrayPath must contain at least one ArraySegment") + if not any(_is_iterating(s) for s in self.segments): + raise ValueError("Iterated must contain at least one Array/Map segment") + if _first_iterating(self.segments).is_anonymous: + raise ValueError("first iterating segment must be named, not anonymous") - def append_struct(self, name: str) -> ArrayPath: - return ArrayPath(segments=self.segments + (StructSegment(name=name),)) + def append_struct(self, name: str) -> Iterated: + """Return a new `Iterated` with *name* appended to the struct leaf.""" + return Iterated(segments=self.segments + (StructSegment(name=name),)) - def append_array(self, name: str, iter_count: int = 1) -> ArrayPath: - return ArrayPath( - segments=self.segments + (ArraySegment(name=name, iter_count=iter_count),) - ) + def append_array(self, name: str) -> Iterated: + """Return a new `Iterated` with *name* appended as a named array segment.""" + return Iterated(segments=self.segments + (ArraySegment(name=name),)) @property - def column_prefix(self) -> ScalarPath: - """Struct segments before the first ArraySegment. + def outer_column(self) -> str: + """Dotted name of the outermost iterated column. - Returns an empty `ScalarPath(())` when the array is the first - segment. + The struct prefix plus the first iterating segment's name + (unbracketed, unprojected). This is what `F.col(...)`, + `array_check("...", ...)`, or `map_values_check("...", ...)` + consumes. """ - prefix: list[StructSegment] = [] + names: list[str] = [] for seg in self.segments: - if isinstance(seg, ArraySegment): + names.append(seg.name) + if _is_iterating(seg): break - prefix.append(seg) - return ScalarPath(segments=tuple(prefix)) + return ".".join(names) @property - def column_path(self) -> str: - """Dotted name of the outermost array column. + def column_prefix(self) -> Direct: + """Struct segments before the first iterating segment. - The struct prefix plus the first ArraySegment's name (unbracketed). - This is what `F.col(...)` or `array_check("...", ...)` consumes. + Returns an empty `Direct(())` when an iterating segment is first. """ - first_prefix, first_array, _first_iter = self.array_chunks[0] - return ".".join((*first_prefix, first_array)) + prefix: list[StructSegment] = [] + for seg in self.segments: + if _is_iterating(seg): + break + assert isinstance(seg, StructSegment) + prefix.append(seg) + return Direct(segments=tuple(prefix)) @property def leaf(self) -> tuple[str, ...]: - """Names of struct segments after the last ArraySegment.""" - last_array = next( - i - for i in range(len(self.segments) - 1, -1, -1) - if isinstance(self.segments[i], ArraySegment) - ) - return tuple(s.name for s in self.segments[last_array + 1 :]) + """Names of struct segments after the last iterating segment.""" + last_iter = max(i for i, s in enumerate(self.segments) if _is_iterating(s)) + return tuple(s.name for s in self.segments[last_iter + 1 :]) @property - def array_chunks( + def iter_frames( self, - ) -> tuple[tuple[tuple[str, ...], str, int], ...]: - """One chunk per ArraySegment. + ) -> tuple[tuple[tuple[str, ...], ArraySegment | MapSegment], ...]: + """One frame per *named* iterating segment. + + Each entry is `(prefix_structs, segment)` where `prefix_structs` is + the sequence of struct segment names between the previous named + iterating segment (or the start of the path) and this one. An + anonymous iterating segment does not start a new frame -- it is an + extra iteration folded into the preceding named frame, surfaced + separately by `iter_struct_paths`. Carrying the segment lets + downstream pick the runtime helper and read `projection`. + """ + frames: list[tuple[tuple[str, ...], ArraySegment | MapSegment]] = [] + prefix: list[str] = [] + for seg in self.segments: + if isinstance(seg, (ArraySegment, MapSegment)): + if not seg.is_anonymous: + frames.append((tuple(prefix), seg)) + prefix = [] + else: + prefix.append(seg.name) + return tuple(frames) + + @property + def iter_struct_paths(self) -> tuple[tuple[str, ...], ...]: + """Per non-outermost iteration: the struct path that reaches its container. + + For each named iterating segment past the first, emit `(prefix_structs + + segment_name)` -- the navigation FROM the previous iteration's + element TO this container. For each anonymous iterating segment, emit + `()` -- the parent element is already the next container, so there is + no navigation. - Each entry is `(prefix_structs, array_name, iter_count)` where - `prefix_structs` is the sequence of struct segment names between - the previous ArraySegment (or the start of the path) and this - ArraySegment. + Returns an empty tuple when the path iterates only once. """ - chunks: list[tuple[tuple[str, ...], str, int]] = [] + paths: list[tuple[str, ...]] = [] prefix: list[str] = [] + first = True for seg in self.segments: - if isinstance(seg, ArraySegment): - chunks.append((tuple(prefix), seg.name, seg.iter_count)) + if isinstance(seg, (ArraySegment, MapSegment)): + if first: + first = False + elif seg.is_anonymous: + paths.append(()) + else: + paths.append((*prefix, seg.name)) prefix = [] else: prefix.append(seg.name) - return tuple(chunks) + return tuple(paths) def element_relative_gate(self, gate: FieldPath) -> tuple[str, ...] | None: """Path inside this array's element scope that names *gate*. + **Precondition (stated loudly):** valid only when the first iterating + segment is an `ArraySegment`; guaranteed because `check_builder` + zeros the nullable gate whenever it enters any iterated container, so + a map-first path never carries a gate. The `assert` below on the + boundary segment makes the precondition a hard failure if violated. + Three return states: - - ``tuple[str, ...]`` (non-empty) -- "reachable with descent": + - `tuple[str, ...]` (non-empty) -- "reachable with descent": `gate` enters the same outer array as this path and names a struct descendant inside its element. The returned segments name that descendant relative to the element. - - ``()`` -- "reachable, no descent": `gate` is the outer array + - `()` -- "reachable, no descent": `gate` is the outer array itself; the element variable IS the gated value. - - ``None`` -- "not reachable": `gate` does not cross into this + - `None` -- "not reachable": `gate` does not cross into this path's element scope (different outer array, scalar gate, - mismatched struct prefix, mismatched boundary `iter_count`, + mismatched struct prefix, mismatched boundary iteration depth, etc.). Callers must apply the gate at column level instead. Raises `NotImplementedError` when `gate` enters the same outer @@ -207,7 +299,7 @@ def element_relative_gate(self, gate: FieldPath) -> tuple[str, ...] | None: """ column_prefix = self.column_prefix.segments n_prefix = len(column_prefix) - if not isinstance(gate, ArrayPath): + if not isinstance(gate, Iterated): return None gate_segs = gate.segments if len(gate_segs) <= n_prefix: @@ -224,9 +316,11 @@ def element_relative_gate(self, gate: FieldPath) -> tuple[str, ...] | None: return None if gate_boundary.name != target_boundary.name: return None - if gate_boundary.iter_count != target_boundary.iter_count: + target_run = _array_run_length(self.segments, n_prefix) + gate_run = _array_run_length(gate_segs, n_prefix) + if gate_run != target_run: return None - inner_segments = gate_segs[n_prefix + 1 :] + inner_segments = gate_segs[n_prefix + gate_run :] for seg in inner_segments: if not isinstance(seg, StructSegment): raise NotImplementedError( @@ -235,178 +329,121 @@ def element_relative_gate(self, gate: FieldPath) -> tuple[str, ...] | None: ) return tuple(s.name for s in inner_segments) - @property - def iter_struct_paths(self) -> tuple[tuple[str, ...], ...]: - """Per non-outermost iteration: the struct path that reaches its array. - - For each ArraySegment past the first, emit `(prefix_structs + - array_name)` -- the navigation FROM the previous iteration's - element TO this array. For each `iter_count > 1` on an - ArraySegment, emit `iter_count - 1` additional `()` entries - representing extra iterations inside the same (already-named) - array. - - Returns an empty tuple when the path iterates only once. - """ - paths: list[tuple[str, ...]] = [] - for chunk_idx, (prefix_structs, arr_name, iter_count) in enumerate( - self.array_chunks - ): - if chunk_idx > 0: - paths.append(prefix_structs + (arr_name,)) - for _ in range(iter_count - 1): - paths.append(()) - return tuple(paths) - def __str__(self) -> str: - return ".".join(_segment_str(s) for s in self.segments) - - -@dataclass(frozen=True, slots=True) -class MapPath: - """Locate a value inside a map's keys or values via one `MapSegment`. - - Invariant: `segments` is a struct prefix, exactly one `MapSegment` - boundary, then a struct-only leaf (possibly empty). The `MapSegment` - iterates the projected keys or values like an array; the leaf navigates - structs inside each projected element, mirroring `ArrayPath.leaf` for a - `list[Model]`. An empty leaf locates the projected scalar itself - (`dict[K, scalar]`); a non-empty leaf locates a field inside a - `dict[K, Model]` value (or key). - - The map must be reachable without array iteration, and the leaf must be - struct-only -- a map nested inside an array element or a container - nested inside a map element is not representable (and - `promote_terminal_map` / `promote_terminal_array` raise rather than - fabricate one). - """ - - segments: tuple[StructSegment | MapSegment, ...] - - def __post_init__(self) -> None: - map_count = sum(isinstance(s, MapSegment) for s in self.segments) - if map_count != 1: - raise ValueError("MapPath must contain exactly one MapSegment") - if not all(isinstance(s, (StructSegment, MapSegment)) for s in self.segments): - raise ValueError("MapPath segments outside the map must be struct segments") - - @property - def _map_index(self) -> int: - return next(i for i, s in enumerate(self.segments) if isinstance(s, MapSegment)) - - @property - def projection(self) -> MapProjection: - seg = self.segments[self._map_index] - assert isinstance(seg, MapSegment) - return seg.projection - - @property - def map_column(self) -> str: - """Dotted name of the map column (struct prefix + map field name). - - This is what `F.col(...)` consumes; the `{key}` / `{value}` marker - and the leaf belong to `str(path)`, not to the column reference. - """ - return ".".join(s.name for s in self.segments[: self._map_index + 1]) - - @property - def leaf(self) -> tuple[str, ...]: - """Names of struct segments after the `MapSegment`. - - Empty for a bare key/value projection; the field path inside each - projected element otherwise. - """ - return tuple(s.name for s in self.segments[self._map_index + 1 :]) - - def append_struct(self, name: str) -> MapPath: - return MapPath(segments=self.segments + (StructSegment(name=name),)) + parts: list[str] = [] + for seg in self.segments: + token = _segment_str(seg) + if isinstance(seg, (ArraySegment, MapSegment)) and seg.is_anonymous: + parts[-1] += token + else: + parts.append(token) + return ".".join(parts) - def __str__(self) -> str: - base = f"{self.map_column}{{{self.projection.value}}}" - return base + "".join(f".{n}" for n in self.leaf) +FieldPath: TypeAlias = Direct | Iterated -FieldPath: TypeAlias = ScalarPath | ArrayPath | MapPath +def _segment_str(seg: FieldSegment) -> str: + if isinstance(seg, ArraySegment): + return "[]" if seg.is_anonymous else seg.name + "[]" + if isinstance(seg, MapSegment): + marker = f"{{{seg.projection.value}}}" + return marker if seg.is_anonymous else seg.name + marker + return seg.name -# The element type of any `FieldPath.segments`, across all three variants. -# Broader than an `ArrayPath`'s `StructSegment | ArraySegment`: a `MapPath` -# adds a trailing `MapSegment`. Consumers that walk an arbitrary -# `FieldPath`'s segments -- rather than a statically known `ArrayPath` -- -# annotate with this so a `MapSegment` is not a type error. -FieldSegment: TypeAlias = StructSegment | ArraySegment | MapSegment +def _array_run_length(segments: tuple[FieldSegment, ...], start: int) -> int: + """Count the ArraySegment run starting at *start*. -def _segment_str(seg: StructSegment | ArraySegment) -> str: - if isinstance(seg, ArraySegment): - return seg.name + "[]" * seg.iter_count - return seg.name + The run is the named segment at `segments[start]` (which must be an + `ArraySegment`) plus any immediately-following anonymous ArraySegments + -- the total iteration depth of a multi-bracket terminal like + `hierarchies[][]`. + """ + length = 1 + idx = start + 1 + while idx < len(segments): + candidate = segments[idx] + if not (isinstance(candidate, ArraySegment) and candidate.is_anonymous): + break + length += 1 + idx += 1 + return length + + +def terminal_run_start(segments: tuple[FieldSegment, ...]) -> int: + """Return the index where the trailing bracket run's named segment sits. + + Scans backward from the last segment while it is an anonymous + `ArraySegment`, stopping at the first segment that isn't (or at index + 0). The result names the run's *named* segment -- e.g. the first of + the two segments behind a multi-bracket terminal like + `hierarchies[][]` -- or simply the last segment when the path doesn't + end in a bracket run at all. + + The `Iterated` invariant (the first iterating segment is always named) + guarantees the scan never needs to pass index 0. The mirror of + `_array_run_length`, which counts the same run forward from this index: + `_array_run_length(segments, terminal_run_start(segments))` gives the + run's length whenever the terminal segment is an `ArraySegment`. + """ + index = len(segments) - 1 + while index > 0: + candidate = segments[index] + if not (isinstance(candidate, ArraySegment) and candidate.is_anonymous): + break + index -= 1 + return index -def _strip_map_suffix(part: str) -> MapProjection | None: - """Return the `MapProjection` named by a trailing `{key}`/`{value}`, or None.""" - for proj in MapProjection: - if part.endswith(f"{{{proj.value}}}"): - return proj - return None +_MARKER = re.compile(r"\[\]|\{(key|value)\}") def parse(encoded: str) -> FieldPath: """Parse a canonical encoded path like `"items[].nested.value"`. - Trailing `[]` markers on a dotted part produce an `ArraySegment` - with matching `iter_count`; a `{key}`/`{value}` marker produces a - `MapSegment` (and a `MapPath`), with any dotted parts after it forming - the map's struct leaf (e.g. `subs{value}.label`). The empty string - returns the empty `ScalarPath`. Raises `ValueError` when any dotted - part has an empty name (e.g. `".a"`, `"a..b"`, `"[]"`), when more than - one map marker appears, or when an array marker combines with a map - projection (`dict[K, list[V]]` is not representable as a `MapPath`). + A left-to-right marker tokenizer. Splits on `.`; for each dotted part, + reads the `name` up to the first marker (`[` or `{`), then scans markers + (`[]`, `{key}`, `{value}`) left to right against the remainder. A part + with no marker becomes a `StructSegment`. The first marker on a part + becomes a *named* container segment on `name`; each subsequent marker + becomes an *anonymous* container segment (its parent element is itself a + container). Because anonymous segments attach to the previous token, + every `.`-delimited part begins with a name. + + Examples: `hierarchies[][]` -> two `ArraySegment`s (named, anonymous); + `subs{value}[]` -> `MapSegment` then anonymous `ArraySegment`; + `items[]{value}` -> `ArraySegment` then anonymous `MapSegment`; + `subs{value}.label` -> `MapSegment` then `StructSegment` leaf. The empty + string returns the empty `Direct`. Raises `ValueError` when any dotted + part has an empty name (e.g. `".a"`, `"a..b"`, `"[]"`, `"a.{key}"`). """ if not encoded: - return ScalarPath() - segments: list[StructSegment | ArraySegment | MapSegment] = [] - struct_segments: list[StructSegment] = [] - has_array = False - map_seen = False - parts = encoded.split(".") - for part in parts: - projection = _strip_map_suffix(part) - if projection is not None: - if map_seen: - raise ValueError(f"FieldPath has multiple map markers in {encoded!r}") - part = part[: -(len(projection.value) + 2)] - depth = 0 - while part.endswith("[]"): - part = part[:-2] - depth += 1 - if not part: + return Direct() + segments: list[FieldSegment] = [] + has_iter = False + for part in encoded.split("."): + m = _MARKER.search(part) + name = part if m is None else part[: m.start()] + if not name: raise ValueError(f"FieldPath part has empty name in {encoded!r}") - if projection is not None: - if depth > 0: - raise ValueError( - f"map projection marker cannot follow array markers in {encoded!r}" + if m is None: + segments.append(StructSegment(name=name)) + continue + has_iter = True + first = True + for mk in _MARKER.finditer(part): + seg_name = name if first else "" + if mk.group() == "[]": + segments.append(ArraySegment(name=seg_name)) + else: + segments.append( + MapSegment(name=seg_name, projection=MapProjection(mk.group(1))) ) - map_seen = True - segments.append(MapSegment(name=part, projection=projection)) - elif depth > 0: - has_array = True - segments.append(ArraySegment(name=part, iter_count=depth)) - else: - struct = StructSegment(name=part) - segments.append(struct) - struct_segments.append(struct) - if map_seen: - if has_array: - raise ValueError( - f"map projection cannot combine with array markers in {encoded!r}" - ) - return MapPath(segments=tuple(segments)) # type: ignore[arg-type] - if has_array: - # No MapSegment reached this branch (map_seen is False), so the - # tuple holds only Struct/Array segments. - return ArrayPath(segments=tuple(segments)) # type: ignore[arg-type] - return ScalarPath(segments=tuple(struct_segments)) + first = False + if has_iter: + return Iterated(segments=tuple(segments)) + return Direct(segments=tuple(segments)) # type: ignore[arg-type] def coerce(value: FieldPath | str) -> FieldPath: @@ -416,49 +453,33 @@ def coerce(value: FieldPath | str) -> FieldPath: return value -def promote_terminal_array(path: FieldPath) -> ArrayPath: - """Promote *path*'s terminal segment to an iterated `ArraySegment`. +def promote_terminal( + path: FieldPath, *, projection: MapProjection | None = None +) -> Iterated: + """Promote *path*'s terminal into an iterating segment. - A `StructSegment` terminal is *replaced* with `ArraySegment(name, - iter_count=1)`; an `ArraySegment` terminal has its `iter_count` - incremented. This is how a walker records entering a `list[...]` - layer on the field it is already pointing at -- unlike `append_array`, - which adds a new segment for a fresh nested array. Repeated calls - build the multi-iteration terminal of a `list[list[X]]` field. + Records a walker entering a container (`list[...]` when *projection* is + `None`, `dict[K, V]` when set) on the field it already points at: - Raises `ValueError` on an empty path: there is no terminal segment - to promote. Raises `NotImplementedError` for a `MapPath`: a list nested - inside a map element has no representable path, so the gap stays loud. - """ - if not path.segments: - raise ValueError("cannot promote the terminal of an empty path") - if isinstance(path, MapPath): - raise NotImplementedError("list nested inside a map element is not supported") - *prefix, last = path.segments - if isinstance(last, ArraySegment): - promoted = ArraySegment(name=last.name, iter_count=last.iter_count + 1) - else: - promoted = ArraySegment(name=last.name, iter_count=1) - return ArrayPath(segments=(*prefix, promoted)) - - -def promote_terminal_map(path: FieldPath, projection: MapProjection) -> MapPath: - """Promote *path*'s terminal struct segment to a `MapSegment`. - - Records a walker entering a `dict[K, V]` layer on the field it already - points at, projecting to keys or values. Raises `ValueError` on an - empty path and `NotImplementedError` when the map is reached through - array iteration or already projects another map -- a map nested inside - an array element or another map element has no schema field today and - no representable `MapPath`, so the gap stays loud. + - a `StructSegment` terminal is *replaced* with a *named* container + segment taking the struct's name; + - an iterating terminal has an *anonymous* container segment *appended* + (the parent element is itself a container). + + Every nesting a walker can enter -- struct into a list or map, and a + container directly inside another container -- is representable this way, + so the promotion always succeeds. Raises `ValueError` on an empty path: + there is no terminal segment to promote. """ if not path.segments: raise ValueError("cannot promote the terminal of an empty path") - if isinstance(path, ArrayPath): - raise NotImplementedError("map nested under a list layer is not supported") - if isinstance(path, MapPath): - raise NotImplementedError("map nested inside a map element is not supported") *prefix, last = path.segments - return MapPath( - segments=(*prefix, MapSegment(name=last.name, projection=projection)) # type: ignore[arg-type] + named = last.name if isinstance(last, StructSegment) else "" + new: ArraySegment | MapSegment = ( + ArraySegment(name=named) + if projection is None + else MapSegment(name=named, projection=projection) ) + if isinstance(last, StructSegment): + return Iterated(segments=(*prefix, new)) + return Iterated(segments=(*path.segments, new)) diff --git a/packages/overture-schema-system/tests/test_field_path.py b/packages/overture-schema-system/tests/test_field_path.py index 4c10c5e3f..80da80f8e 100644 --- a/packages/overture-schema-system/tests/test_field_path.py +++ b/packages/overture-schema-system/tests/test_field_path.py @@ -7,69 +7,86 @@ import pytest from overture.schema.system.field_path import ( - ArrayPath, ArraySegment, - MapPath, + Direct, + Iterated, MapProjection, MapSegment, - ScalarPath, StructSegment, coerce, parse, - promote_terminal_array, - promote_terminal_map, + promote_terminal, + terminal_run_start, ) class TestParseAndRoundTrip: - def test_empty_path_parses_to_empty_scalar(self) -> None: - assert parse("") == ScalarPath(segments=()) + def test_empty_path_parses_to_empty_direct(self) -> None: + assert parse("") == Direct(segments=()) def test_single_segment(self) -> None: path = parse("name") - assert path == ScalarPath(segments=(StructSegment(name="name"),)) + assert path == Direct(segments=(StructSegment(name="name"),)) def test_dotted_path(self) -> None: path = parse("bbox.xmin") - assert path == ScalarPath( + assert path == Direct( segments=(StructSegment(name="bbox"), StructSegment(name="xmin")) ) def test_array_segment(self) -> None: path = parse("items[]") - assert path == ArrayPath(segments=(ArraySegment(name="items", iter_count=1),)) + assert path == Iterated(segments=(ArraySegment(name="items"),)) def test_array_with_nested_field(self) -> None: path = parse("items[].value") - assert path == ArrayPath( + assert path == Iterated( segments=( - ArraySegment(name="items", iter_count=1), + ArraySegment(name="items"), StructSegment(name="value"), ) ) - def test_nested_list_depth(self) -> None: - path = parse("hierarchies[][]") - assert path == ArrayPath( - segments=(ArraySegment(name="hierarchies", iter_count=2),) + def test_nested_list_parses_to_anonymous_segments(self) -> None: + assert parse("hierarchies[][]") == Iterated( + segments=( + ArraySegment(name="hierarchies"), + ArraySegment(name=""), + ) ) def test_nested_list_with_leaf(self) -> None: path = parse("hierarchies[][].value") - assert path == ArrayPath( + assert path == Iterated( segments=( - ArraySegment(name="hierarchies", iter_count=2), + ArraySegment(name="hierarchies"), + ArraySegment(name=""), StructSegment(name="value"), ) ) + def test_is_anonymous_property(self) -> None: + assert ArraySegment(name="").is_anonymous is True + assert ArraySegment(name="grid").is_anonymous is False + assert MapSegment(name="", projection=MapProjection.VALUE).is_anonymous is True + assert ( + MapSegment(name="subs", projection=MapProjection.VALUE).is_anonymous + is False + ) + + def test_nested_list_round_trips(self) -> None: + assert str(parse("hierarchies[][]")) == "hierarchies[][]" + + def test_nested_list_with_leaf_round_trips(self) -> None: + assert str(parse("hierarchies[][].value")) == "hierarchies[][].value" + def test_complex_path(self) -> None: path = parse("speed_limits[].when.vehicle[].dimension") - assert path == ArrayPath( + assert path == Iterated( segments=( - ArraySegment(name="speed_limits", iter_count=1), + ArraySegment(name="speed_limits"), StructSegment(name="when"), - ArraySegment(name="vehicle", iter_count=1), + ArraySegment(name="vehicle"), StructSegment(name="dimension"), ) ) @@ -86,274 +103,451 @@ def test_complex_path(self) -> None: "hierarchies[][].value", "speed_limits[].when.vehicle[].dimension", "tags_min_length", + "tags{key}", + "tags{value}", + "names.common{key}", + "subs{value}.label", ], ) def test_str_round_trip(self, encoded: str) -> None: assert str(parse(encoded)) == encoded -class TestScalarVsArrayPartition: - def test_no_array_returns_scalar_path(self) -> None: - assert isinstance(parse("a.b.c"), ScalarPath) +class TestTypes: + def test_types(self) -> None: + assert isinstance(parse("a.b.c"), Direct) + assert isinstance(parse("a.b[].c"), Iterated) + assert isinstance(parse("names.common{key}"), Iterated) + + def test_empty_is_direct(self) -> None: + assert isinstance(parse(""), Direct) + + def test_map_first_is_iterated(self) -> None: + assert isinstance(parse("tags{value}"), Iterated) + + +class TestInterleavedGrammar: + @pytest.mark.parametrize( + "encoded", + [ + "hierarchies[][]", + "subs{value}[]", + "items[]{value}", + "a{value}{value}", + "subs{value}.label", + "items[].tags{value}", + "names.common{key}", + "dict_field{value}.label", + ], + ) + def test_interleaved_round_trip(self, encoded: str) -> None: + assert str(parse(encoded)) == encoded - def test_with_array_returns_array_path(self) -> None: - assert isinstance(parse("a.b[].c"), ArrayPath) + def test_dict_of_list_segments(self) -> None: + assert parse("subs{value}[]").segments == ( + MapSegment(name="subs", projection=MapProjection.VALUE), + ArraySegment(name=""), + ) - def test_empty_is_scalar(self) -> None: - assert isinstance(parse(""), ScalarPath) + def test_list_of_dict_segments(self) -> None: + assert parse("items[]{value}").segments == ( + ArraySegment(name="items"), + MapSegment(name="", projection=MapProjection.VALUE), + ) + + def test_map_of_map_segments(self) -> None: + assert parse("a{value}{value}").segments == ( + MapSegment(name="a", projection=MapProjection.VALUE), + MapSegment(name="", projection=MapProjection.VALUE), + ) + + def test_named_map_in_array_segments(self) -> None: + assert parse("items[].tags{value}").segments == ( + ArraySegment(name="items"), + MapSegment(name="tags", projection=MapProjection.VALUE), + ) + + def test_map_with_struct_leaf_segments(self) -> None: + assert parse("subs{value}.label").segments == ( + MapSegment(name="subs", projection=MapProjection.VALUE), + StructSegment(name="label"), + ) class TestStr: def test_empty_renders_as_empty(self) -> None: - assert str(ScalarPath()) == "" + assert str(Direct()) == "" - def test_scalar_path_renders_dotted(self) -> None: - path = ScalarPath( - segments=(StructSegment(name="bbox"), StructSegment(name="xmin")) - ) + def test_direct_renders_dotted(self) -> None: + path = Direct(segments=(StructSegment(name="bbox"), StructSegment(name="xmin"))) assert str(path) == "bbox.xmin" - def test_array_path_renders_with_brackets(self) -> None: - path = ArrayPath( + def test_iterated_renders_with_brackets(self) -> None: + path = Iterated( segments=( - ArraySegment(name="speed_limits", iter_count=1), + ArraySegment(name="speed_limits"), StructSegment(name="when"), ) ) assert str(path) == "speed_limits[].when" - def test_array_path_renders_multi_depth(self) -> None: - path = ArrayPath(segments=(ArraySegment(name="hierarchies", iter_count=2),)) + def test_iterated_renders_multi_depth(self) -> None: + path = Iterated( + segments=(ArraySegment(name="hierarchies"), ArraySegment(name="")) + ) assert str(path) == "hierarchies[][]" + def test_map_renders_with_projection(self) -> None: + path = Iterated( + segments=( + StructSegment(name="names"), + MapSegment(name="common", projection=MapProjection.KEY), + ) + ) + assert str(path) == "names.common{key}" + class TestAppendStruct: - def test_scalar_append_struct_returns_scalar(self) -> None: - path = ScalarPath().append_struct("name") + def test_direct_append_struct_returns_direct(self) -> None: + path = Direct().append_struct("name") assert path == parse("name") - assert isinstance(path, ScalarPath) + assert isinstance(path, Direct) - def test_scalar_chain_struct(self) -> None: - path = ScalarPath().append_struct("bbox").append_struct("xmin") + def test_direct_chain_struct(self) -> None: + path = Direct().append_struct("bbox").append_struct("xmin") assert path == parse("bbox.xmin") - def test_array_append_struct_returns_array(self) -> None: + def test_iterated_append_struct_returns_iterated(self) -> None: path = parse("items[]") - assert isinstance(path, ArrayPath) + assert isinstance(path, Iterated) result = path.append_struct("value") assert result == parse("items[].value") - assert isinstance(result, ArrayPath) + assert isinstance(result, Iterated) + + def test_map_append_struct_extends_leaf(self) -> None: + path = parse("subs{value}") + assert isinstance(path, Iterated) + result = path.append_struct("label") + assert result == parse("subs{value}.label") class TestAppendArray: - def test_scalar_append_array_returns_array_path(self) -> None: - path = ScalarPath().append_array("items") + def test_direct_append_array_returns_iterated(self) -> None: + path = Direct().append_array("items") assert path == parse("items[]") - assert isinstance(path, ArrayPath) + assert isinstance(path, Iterated) - def test_scalar_append_array_after_struct(self) -> None: - path = ScalarPath().append_struct("outer").append_array("items") + def test_direct_append_array_after_struct(self) -> None: + path = Direct().append_struct("outer").append_array("items") assert path == parse("outer.items[]") - def test_scalar_append_array_multi_depth(self) -> None: - path = ScalarPath().append_array("hierarchies", iter_count=2) - assert path == parse("hierarchies[][]") - - def test_array_append_array(self) -> None: + def test_iterated_append_array(self) -> None: path = parse("outer[]") - assert isinstance(path, ArrayPath) + assert isinstance(path, Iterated) result = path.append_array("inner") assert result == parse("outer[].inner[]") -class TestPromoteTerminalArray: - def test_scalar_struct_terminal_becomes_array(self) -> None: - assert promote_terminal_array(parse("tags")) == parse("tags[]") +class TestPromoteTerminal: + def test_struct_terminal_becomes_array(self) -> None: + assert promote_terminal(parse("tags")) == parse("tags[]") def test_struct_prefix_is_preserved(self) -> None: - assert promote_terminal_array(parse("outer.tags")) == parse("outer.tags[]") + assert promote_terminal(parse("outer.tags")) == parse("outer.tags[]") - def test_struct_terminal_inside_array_path(self) -> None: - assert promote_terminal_array(parse("items[].tags")) == parse("items[].tags[]") + def test_struct_terminal_inside_iterated(self) -> None: + assert promote_terminal(parse("items[].tags")) == parse("items[].tags[]") - def test_array_terminal_increments_iter_count(self) -> None: - assert promote_terminal_array(parse("tags[]")) == parse("tags[][]") + def test_promote_array_terminal_appends_anonymous(self) -> None: + assert promote_terminal(parse("tags[]")) == parse("tags[][]") + assert promote_terminal(parse("tags[]")).segments == ( + ArraySegment(name="tags"), + ArraySegment(name=""), + ) def test_consecutive_promotions_stack(self) -> None: - assert promote_terminal_array(promote_terminal_array(parse("grid"))) == parse( - "grid[][]" + assert promote_terminal(promote_terminal(parse("grid"))) == parse("grid[][]") + + def test_array_terminal_inside_iterated(self) -> None: + assert promote_terminal(parse("items[].grid[]")) == parse("items[].grid[][]") + + def test_struct_terminal_becomes_map_key(self) -> None: + assert promote_terminal(parse("tags"), projection=MapProjection.KEY) == parse( + "tags{key}" ) - def test_array_terminal_inside_array_path(self) -> None: - assert promote_terminal_array(parse("items[].grid[]")) == parse( - "items[].grid[][]" + def test_struct_prefix_preserved_for_map_value(self) -> None: + assert promote_terminal( + parse("names.common"), projection=MapProjection.VALUE + ) == parse("names.common{value}") + + def test_promote_terminal_to_array_on_map(self) -> None: # dict[K, list] + assert promote_terminal(parse("subs{value}")).segments[-1] == ArraySegment( + name="" ) + def test_promote_terminal_to_map_on_array(self) -> None: # list[dict] + assert promote_terminal( + parse("items[]"), projection=MapProjection.VALUE + ).segments[-1] == MapSegment(name="", projection=MapProjection.VALUE) + def test_empty_path_raises(self) -> None: with pytest.raises(ValueError, match="empty path"): - promote_terminal_array(ScalarPath()) + promote_terminal(Direct()) + + def test_empty_path_raises_for_map(self) -> None: + with pytest.raises(ValueError, match="empty path"): + promote_terminal(Direct(), projection=MapProjection.KEY) + - def test_map_path_raises(self) -> None: - with pytest.raises(NotImplementedError, match="map"): - promote_terminal_array(parse("subs{value}.inner")) +class TestOuterColumn: + @staticmethod + def _outer_column(encoded: str) -> str: + path = parse(encoded) + assert isinstance(path, Iterated) + return path.outer_column + + def test_array_at_start(self) -> None: + assert self._outer_column("items[].value") == "items" + + def test_struct_prefix_before_array(self) -> None: + assert self._outer_column("parent.items[].value") == "parent.items" + + def test_map_outer_column(self) -> None: + assert self._outer_column("names.common{value}.label") == "names.common" + + def test_map_at_start(self) -> None: + assert self._outer_column("tags{key}") == "tags" class TestColumnPrefix: def test_array_at_start_has_empty_prefix(self) -> None: path = parse("items[].value") - assert isinstance(path, ArrayPath) - assert path.column_prefix == ScalarPath(()) + assert isinstance(path, Iterated) + assert path.column_prefix == Direct(()) def test_struct_prefix_before_array(self) -> None: path = parse("parent.items[].value") - assert isinstance(path, ArrayPath) + assert isinstance(path, Iterated) assert path.column_prefix == parse("parent") def test_dotted_struct_prefix(self) -> None: path = parse("a.b.c[].d") - assert isinstance(path, ArrayPath) + assert isinstance(path, Iterated) assert path.column_prefix == parse("a.b") + def test_struct_prefix_before_map(self) -> None: + path = parse("names.common{value}") + assert isinstance(path, Iterated) + assert path.column_prefix == parse("names") + class TestLeaf: def test_no_leaf_after_array(self) -> None: path = parse("items[]") - assert isinstance(path, ArrayPath) + assert isinstance(path, Iterated) assert path.leaf == () def test_single_struct_leaf(self) -> None: path = parse("items[].value") - assert isinstance(path, ArrayPath) + assert isinstance(path, Iterated) assert path.leaf == ("value",) def test_nested_struct_leaf(self) -> None: path = parse("items[].nested.value") - assert isinstance(path, ArrayPath) + assert isinstance(path, Iterated) assert path.leaf == ("nested", "value") - def test_uses_last_array(self) -> None: + def test_uses_last_iterating_segment(self) -> None: path = parse("speed_limits[].when.vehicle[].dimension") - assert isinstance(path, ArrayPath) + assert isinstance(path, Iterated) assert path.leaf == ("dimension",) + def test_map_leaf(self) -> None: + path = parse("subs{value}.inner.label") + assert isinstance(path, Iterated) + assert path.leaf == ("inner", "label") + + def test_bare_map_has_empty_leaf(self) -> None: + path = parse("subs{value}") + assert isinstance(path, Iterated) + assert path.leaf == () -class TestArrayChunks: + +class TestIterFrames: def test_single_top_level_array(self) -> None: path = parse("items[]") - assert isinstance(path, ArrayPath) - assert path.array_chunks == (((), "items", 1),) + assert isinstance(path, Iterated) + assert path.iter_frames == (((), ArraySegment(name="items")),) def test_single_array_with_struct_prefix(self) -> None: path = parse("parent.items[].value") - assert isinstance(path, ArrayPath) - assert path.array_chunks == ((("parent",), "items", 1),) + assert isinstance(path, Iterated) + assert path.iter_frames == ((("parent",), ArraySegment(name="items")),) def test_nested_arrays(self) -> None: path = parse("speed_limits[].when.vehicle[].dimension") - assert isinstance(path, ArrayPath) - assert path.array_chunks == ( - ((), "speed_limits", 1), - (("when",), "vehicle", 1), + assert isinstance(path, Iterated) + assert path.iter_frames == ( + ((), ArraySegment(name="speed_limits")), + (("when",), ArraySegment(name="vehicle")), ) - def test_multi_depth_array(self) -> None: + def test_multi_depth_folds_anonymous(self) -> None: path = parse("hierarchies[][].value") - assert isinstance(path, ArrayPath) - assert path.array_chunks == (((), "hierarchies", 2),) + assert isinstance(path, Iterated) + assert path.iter_frames == (((), ArraySegment(name="hierarchies")),) + + def test_map_frame(self) -> None: + path = parse("names.common{key}") + assert isinstance(path, Iterated) + assert path.iter_frames == ( + (("names",), MapSegment(name="common", projection=MapProjection.KEY)), + ) + + def test_mixed_map_in_array(self) -> None: + path = parse("items[].tags{value}") + assert isinstance(path, Iterated) + assert path.iter_frames == ( + ((), ArraySegment(name="items")), + ((), MapSegment(name="tags", projection=MapProjection.VALUE)), + ) class TestIterStructPaths: def test_single_iteration_is_empty(self) -> None: path = parse("items[].value") - assert isinstance(path, ArrayPath) + assert isinstance(path, Iterated) assert path.iter_struct_paths == () def test_nested_arrays_emit_navigation_path(self) -> None: path = parse("speed_limits[].when.vehicle[].dimension") - assert isinstance(path, ArrayPath) + assert isinstance(path, Iterated) assert path.iter_struct_paths == (("when", "vehicle"),) - def test_multi_depth_array_expands_extra_iterations(self) -> None: + def test_multi_depth_iter_struct_paths(self) -> None: + # inner anonymous frame contributes () (no navigation) path = parse("hierarchies[][].value") - assert isinstance(path, ArrayPath) + assert isinstance(path, Iterated) assert path.iter_struct_paths == ((),) def test_multi_depth_inner_array_combines_navigation_and_expansion(self) -> None: path = parse("rules[].tags[][].value") - assert isinstance(path, ArrayPath) + assert isinstance(path, Iterated) assert path.iter_struct_paths == (("tags",), ()) + def test_mixed_map_in_array_navigation(self) -> None: + path = parse("items[].tags{value}") + assert isinstance(path, Iterated) + assert path.iter_struct_paths == (("tags",),) + + +class TestTerminalRunStart: + def test_named_array_terminal(self) -> None: + path = parse("items[]") + assert terminal_run_start(path.segments) == 0 + + def test_multi_bracket_terminal_starts_at_named_segment(self) -> None: + path = parse("hierarchies[][]") + assert terminal_run_start(path.segments) == 0 + + def test_struct_leaf_after_array_is_its_own_run(self) -> None: + path = parse("items[].value") + assert terminal_run_start(path.segments) == 1 + + def test_struct_only_path_returns_last_index(self) -> None: + path = parse("a.b.c") + assert terminal_run_start(path.segments) == 2 + + def test_single_segment_returns_zero(self) -> None: + path = parse("a") + assert terminal_run_start(path.segments) == 0 + class TestElementRelativeGate: def test_gate_inside_same_outer_array(self) -> None: target = parse("items[].value") gate = parse("items[].nested") - assert isinstance(target, ArrayPath) + assert isinstance(target, Iterated) assert target.element_relative_gate(gate) == ("nested",) def test_gate_at_outer_array_root_returns_empty(self) -> None: target = parse("items[].value") gate = parse("items[]") - assert isinstance(target, ArrayPath) + assert isinstance(target, Iterated) assert target.element_relative_gate(gate) == () def test_gate_with_dotted_struct_inside_element(self) -> None: target = parse("items[].value") gate = parse("items[].a.b") - assert isinstance(target, ArrayPath) + assert isinstance(target, Iterated) assert target.element_relative_gate(gate) == ("a", "b") def test_scalar_gate_returns_none(self) -> None: target = parse("items[].value") gate = parse("other") - assert isinstance(target, ArrayPath) + assert isinstance(target, Iterated) assert target.element_relative_gate(gate) is None def test_different_outer_array_returns_none(self) -> None: target = parse("items[].value") gate = parse("other[].x") - assert isinstance(target, ArrayPath) + assert isinstance(target, Iterated) assert target.element_relative_gate(gate) is None def test_struct_prefix_must_match(self) -> None: target = parse("parent.items[].value") gate = parse("items[].x") - assert isinstance(target, ArrayPath) + assert isinstance(target, Iterated) assert target.element_relative_gate(gate) is None def test_matching_struct_prefix(self) -> None: target = parse("parent.items[].value") gate = parse("parent.items[].x") - assert isinstance(target, ArrayPath) + assert isinstance(target, Iterated) assert target.element_relative_gate(gate) == ("x",) def test_inner_array_segment_raises(self) -> None: target = parse("items[].value") gate = parse("items[].nested[]") - assert isinstance(target, ArrayPath) + assert isinstance(target, Iterated) with pytest.raises(NotImplementedError, match="nested array segment"): target.element_relative_gate(gate) - def test_mismatched_iter_count_returns_none(self) -> None: - # target iterates items[] (iter_count=1); gate enters items[][] (iter_count=2) - # -- same name, different iteration depth -- not the same element scope + def test_mismatched_iteration_depth_returns_none(self) -> None: target = parse("items[].value") gate = parse("items[][].nested") - assert isinstance(target, ArrayPath) + assert isinstance(target, Iterated) assert target.element_relative_gate(gate) is None - def test_matching_iter_count_still_returns_element_relative_tuple(self) -> None: - # regression: matching iter_count must remain reachable after the fix + def test_matching_iteration_depth_still_returns_element_relative_tuple( + self, + ) -> None: target = parse("items[][].value") gate = parse("items[][].nested") - assert isinstance(target, ArrayPath) + assert isinstance(target, Iterated) assert target.element_relative_gate(gate) == ("nested",) -class TestArrayPathInvariant: - def test_rejects_segments_without_array(self) -> None: - with pytest.raises(ValueError, match="at least one ArraySegment"): - ArrayPath(segments=(StructSegment(name="a"),)) +class TestIteratedInvariant: + def test_iterated_requires_iterating_segment(self) -> None: + with pytest.raises(ValueError, match="at least one"): + Iterated(segments=(StructSegment(name="a"),)) + + def test_first_iterating_segment_named_invariant(self) -> None: + with pytest.raises(ValueError, match="first"): + Iterated(segments=(ArraySegment(name=""),)) + + def test_first_iterating_map_segment_named_invariant(self) -> None: + with pytest.raises(ValueError, match="first"): + Iterated(segments=(MapSegment(name="", projection=MapProjection.VALUE),)) + + def test_struct_prefix_before_first_iterating_is_allowed(self) -> None: + # a struct PREFIX before the first iterating segment is fine + path = Iterated( + segments=(StructSegment(name="parent"), ArraySegment(name="items")) + ) + assert path.outer_column == "parent.items" class TestEqualityAndHashing: @@ -363,7 +557,7 @@ def test_paths_with_same_segments_are_equal(self) -> None: def test_different_paths_unequal(self) -> None: assert parse("items[].value") != parse("items[].other") - def test_scalar_array_unequal(self) -> None: + def test_direct_iterated_unequal(self) -> None: assert parse("items") != parse("items[]") def test_hashable(self) -> None: @@ -375,11 +569,11 @@ def test_string_is_not_equal_to_path(self) -> None: class TestCoerce: - def test_passes_through_scalar(self) -> None: + def test_passes_through_direct(self) -> None: path = parse("a.b") assert coerce(path) is path - def test_passes_through_array(self) -> None: + def test_passes_through_iterated(self) -> None: path = parse("items[].value") assert coerce(path) is path @@ -387,224 +581,8 @@ def test_parses_string(self) -> None: assert coerce("items[].value") == parse("items[].value") -class TestMapPath: - def test_str_top_level_key(self) -> None: - path = MapPath( - segments=(MapSegment(name="tags", projection=MapProjection.KEY),) - ) - assert str(path) == "tags{key}" - - def test_str_top_level_value(self) -> None: - path = MapPath( - segments=(MapSegment(name="tags", projection=MapProjection.VALUE),) - ) - assert str(path) == "tags{value}" - - def test_str_nested_under_struct(self) -> None: - path = MapPath( - segments=( - StructSegment(name="names"), - MapSegment(name="common", projection=MapProjection.KEY), - ) - ) - assert str(path) == "names.common{key}" - - def test_projection_property(self) -> None: - path = MapPath( - segments=(MapSegment(name="tags", projection=MapProjection.VALUE),) - ) - assert path.projection is MapProjection.VALUE - - def test_map_column_top_level(self) -> None: - path = MapPath( - segments=(MapSegment(name="tags", projection=MapProjection.KEY),) - ) - assert path.map_column == "tags" - - def test_map_column_nested(self) -> None: - path = MapPath( - segments=( - StructSegment(name="names"), - MapSegment(name="common", projection=MapProjection.VALUE), - ) - ) - assert path.map_column == "names.common" - - def test_must_contain_a_map_segment(self) -> None: - with pytest.raises(ValueError, match="MapSegment"): - MapPath(segments=(StructSegment(name="names"),)) - - def test_rejects_array_segment_before_map(self) -> None: - with pytest.raises(ValueError, match="struct"): - MapPath( - segments=( # type: ignore[arg-type] # invalid by design: runtime guard under test - ArraySegment(name="items"), - MapSegment(name="tags", projection=MapProjection.KEY), - ) - ) - - def test_rejects_two_map_segments(self) -> None: - with pytest.raises(ValueError, match="MapSegment"): - MapPath( - segments=( - MapSegment(name="a", projection=MapProjection.KEY), - MapSegment(name="b", projection=MapProjection.VALUE), - ) - ) - - @pytest.mark.parametrize( - "encoded", - ["tags{key}", "tags{value}", "names.common{key}", "names.common{value}"], - ) - def test_str_round_trip(self, encoded: str) -> None: - assert str(parse(encoded)) == encoded - - def test_parse_returns_map_path(self) -> None: - assert isinstance(parse("names.common{key}"), MapPath) - - def test_parse_key(self) -> None: - assert parse("tags{key}") == MapPath( - segments=(MapSegment(name="tags", projection=MapProjection.KEY),) - ) - - def test_parse_nested_value(self) -> None: - assert parse("names.common{value}") == MapPath( - segments=( - StructSegment(name="names"), - MapSegment(name="common", projection=MapProjection.VALUE), - ) - ) - - -class TestMapPathLeaf: - """A `MapPath` may carry struct segments after the `MapSegment`. - - These name a value inside a `dict[K, Model]`'s value (or key) struct, - mirroring `ArrayPath.leaf` for `list[Model]`. The `MapSegment` is the - iteration boundary; the leaf is the struct navigation inside each - projected element. - """ - - def test_leaf_empty_for_bare_projection(self) -> None: - path = MapPath( - segments=(MapSegment(name="subs", projection=MapProjection.VALUE),) - ) - assert path.leaf == () - - def test_leaf_names_struct_segments_after_map(self) -> None: - path = MapPath( - segments=( - MapSegment(name="subs", projection=MapProjection.VALUE), - StructSegment(name="label"), - ) - ) - assert path.leaf == ("label",) - - def test_leaf_spans_nested_struct_navigation(self) -> None: - path = MapPath( - segments=( - MapSegment(name="subs", projection=MapProjection.VALUE), - StructSegment(name="inner"), - StructSegment(name="label"), - ) - ) - assert path.leaf == ("inner", "label") - - def test_map_column_excludes_leaf(self) -> None: - path = MapPath( - segments=( - StructSegment(name="names"), - MapSegment(name="common", projection=MapProjection.VALUE), - StructSegment(name="label"), - ) - ) - assert path.map_column == "names.common" - - def test_projection_found_with_leaf_present(self) -> None: - path = MapPath( - segments=( - MapSegment(name="subs", projection=MapProjection.KEY), - StructSegment(name="label"), - ) - ) - assert path.projection is MapProjection.KEY - - def test_str_appends_leaf_after_marker(self) -> None: - path = MapPath( - segments=( - MapSegment(name="subs", projection=MapProjection.VALUE), - StructSegment(name="label"), - ) - ) - assert str(path) == "subs{value}.label" - - def test_append_struct_extends_leaf(self) -> None: - path = MapPath( - segments=(MapSegment(name="subs", projection=MapProjection.VALUE),) - ) - extended = path.append_struct("label") - assert extended == MapPath( - segments=( - MapSegment(name="subs", projection=MapProjection.VALUE), - StructSegment(name="label"), - ) - ) - - def test_rejects_array_segment_in_leaf(self) -> None: - with pytest.raises(ValueError, match="struct"): - MapPath( - segments=( # type: ignore[arg-type] # invalid by design: runtime guard under test - MapSegment(name="subs", projection=MapProjection.VALUE), - ArraySegment(name="items"), - ) - ) - - @pytest.mark.parametrize( - "encoded", - ["subs{value}.label", "names.common{key}.tag", "subs{value}.inner.label"], - ) - def test_str_round_trip_with_leaf(self, encoded: str) -> None: - assert str(parse(encoded)) == encoded - - def test_parse_value_with_leaf(self) -> None: - assert parse("subs{value}.label") == MapPath( - segments=( - MapSegment(name="subs", projection=MapProjection.VALUE), - StructSegment(name="label"), - ) - ) - - def test_parse_rejects_array_marker_in_leaf(self) -> None: - with pytest.raises(ValueError, match="map projection"): - parse("subs{value}.items[]") - - -class TestPromoteTerminalMap: - def test_top_level_struct_becomes_map_key(self) -> None: - assert promote_terminal_map(parse("tags"), MapProjection.KEY) == parse( - "tags{key}" - ) - - def test_struct_prefix_preserved_for_value(self) -> None: - assert promote_terminal_map( - parse("names.common"), MapProjection.VALUE - ) == parse("names.common{value}") - - def test_empty_path_raises(self) -> None: - with pytest.raises(ValueError, match="empty path"): - promote_terminal_map(ScalarPath(), MapProjection.KEY) - - def test_array_path_raises(self) -> None: - with pytest.raises(NotImplementedError, match="list"): - promote_terminal_map(parse("items[].tags"), MapProjection.KEY) - - def test_map_path_raises(self) -> None: - with pytest.raises(NotImplementedError, match="map"): - promote_terminal_map(parse("subs{value}.inner"), MapProjection.VALUE) - - class TestParseRejectsEmptyParts: - @pytest.mark.parametrize("encoded", [".a", "a..b", "[]", "a.[]", ".[]"]) + @pytest.mark.parametrize("encoded", [".a", "a..b", "[]", "a.[]", ".[]", "a.{key}"]) def test_raises_value_error_on_empty_part(self, encoded: str) -> None: with pytest.raises(ValueError, match="empty name"): parse(encoded)