From fe9bf64544444f58b1666bc1abfaf7742260c8d3 Mon Sep 17 00:00:00 2001 From: Alex Iannicelli Date: Tue, 4 Aug 2026 15:17:23 -0400 Subject: [PATCH 1/2] [REFACTOR] Make sources dataset field optional Change the dataset field on SourceItem from required to optional in both the pydantic model and the JSON schema (schema/defs.yaml), and regenerate baseline schemas. Relaxing a constraint is a minor schema change, so this can ship ahead of the planned removal of dataset. Refs OvertureMaps/tf-data-platform#4684 Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: Alex Iannicelli --- .../tests/address_baseline_schema.json | 3 +-- .../tests/bathymetry_baseline_schema.json | 3 +-- .../tests/infrastructure_baseline_schema.json | 3 +-- .../tests/land_baseline_schema.json | 3 +-- .../tests/land_cover_baseline_schema.json | 3 +-- .../tests/land_use_baseline_schema.json | 3 +-- .../tests/water_baseline_schema.json | 3 +-- .../tests/building_baseline_schema.json | 3 +-- .../tests/building_part_baseline_schema.json | 3 +-- .../tests/test_pyspark_scaffold.py | 5 +++-- .../src/overture/schema/common/sources.py | 13 ++++++++----- .../overture-schema-common/tests/test_models.py | 2 +- .../tests/division_area_baseline_schema.json | 3 +-- .../tests/division_baseline_schema.json | 3 +-- .../tests/division_boundary_baseline_schema.json | 3 +-- .../tests/place_baseline_schema.json | 3 +-- .../tests/connector_baseline_schema.json | 3 +-- .../tests/segment_baseline_schema.json | 3 +-- schema/defs.yaml | 2 +- 19 files changed, 28 insertions(+), 39 deletions(-) diff --git a/packages/overture-schema-addresses-theme/tests/address_baseline_schema.json b/packages/overture-schema-addresses-theme/tests/address_baseline_schema.json index c678131da..d123f9f47 100644 --- a/packages/overture-schema-addresses-theme/tests/address_baseline_schema.json +++ b/packages/overture-schema-addresses-theme/tests/address_baseline_schema.json @@ -88,8 +88,7 @@ } }, "required": [ - "property", - "dataset" + "property" ], "title": "SourceItem", "type": "object" diff --git a/packages/overture-schema-base-theme/tests/bathymetry_baseline_schema.json b/packages/overture-schema-base-theme/tests/bathymetry_baseline_schema.json index 0b0b1fb57..f2f12e054 100644 --- a/packages/overture-schema-base-theme/tests/bathymetry_baseline_schema.json +++ b/packages/overture-schema-base-theme/tests/bathymetry_baseline_schema.json @@ -109,8 +109,7 @@ } }, "required": [ - "property", - "dataset" + "property" ], "title": "SourceItem", "type": "object" diff --git a/packages/overture-schema-base-theme/tests/infrastructure_baseline_schema.json b/packages/overture-schema-base-theme/tests/infrastructure_baseline_schema.json index 2789a1d21..67c8473a5 100644 --- a/packages/overture-schema-base-theme/tests/infrastructure_baseline_schema.json +++ b/packages/overture-schema-base-theme/tests/infrastructure_baseline_schema.json @@ -418,8 +418,7 @@ } }, "required": [ - "property", - "dataset" + "property" ], "title": "SourceItem", "type": "object" diff --git a/packages/overture-schema-base-theme/tests/land_baseline_schema.json b/packages/overture-schema-base-theme/tests/land_baseline_schema.json index 6dac56f7b..12211f56d 100644 --- a/packages/overture-schema-base-theme/tests/land_baseline_schema.json +++ b/packages/overture-schema-base-theme/tests/land_baseline_schema.json @@ -291,8 +291,7 @@ } }, "required": [ - "property", - "dataset" + "property" ], "title": "SourceItem", "type": "object" diff --git a/packages/overture-schema-base-theme/tests/land_cover_baseline_schema.json b/packages/overture-schema-base-theme/tests/land_cover_baseline_schema.json index 68bc91e6d..b1401d222 100644 --- a/packages/overture-schema-base-theme/tests/land_cover_baseline_schema.json +++ b/packages/overture-schema-base-theme/tests/land_cover_baseline_schema.json @@ -126,8 +126,7 @@ } }, "required": [ - "property", - "dataset" + "property" ], "title": "SourceItem", "type": "object" diff --git a/packages/overture-schema-base-theme/tests/land_use_baseline_schema.json b/packages/overture-schema-base-theme/tests/land_use_baseline_schema.json index 2065c7732..1e12dc470 100644 --- a/packages/overture-schema-base-theme/tests/land_use_baseline_schema.json +++ b/packages/overture-schema-base-theme/tests/land_use_baseline_schema.json @@ -369,8 +369,7 @@ } }, "required": [ - "property", - "dataset" + "property" ], "title": "SourceItem", "type": "object" diff --git a/packages/overture-schema-base-theme/tests/water_baseline_schema.json b/packages/overture-schema-base-theme/tests/water_baseline_schema.json index d2edfe0be..f4f2309ef 100644 --- a/packages/overture-schema-base-theme/tests/water_baseline_schema.json +++ b/packages/overture-schema-base-theme/tests/water_baseline_schema.json @@ -222,8 +222,7 @@ } }, "required": [ - "property", - "dataset" + "property" ], "title": "SourceItem", "type": "object" diff --git a/packages/overture-schema-buildings-theme/tests/building_baseline_schema.json b/packages/overture-schema-buildings-theme/tests/building_baseline_schema.json index fd835b8b1..1ee6e1143 100644 --- a/packages/overture-schema-buildings-theme/tests/building_baseline_schema.json +++ b/packages/overture-schema-buildings-theme/tests/building_baseline_schema.json @@ -405,8 +405,7 @@ } }, "required": [ - "property", - "dataset" + "property" ], "title": "SourceItem", "type": "object" diff --git a/packages/overture-schema-buildings-theme/tests/building_part_baseline_schema.json b/packages/overture-schema-buildings-theme/tests/building_part_baseline_schema.json index 759e2a510..ba487e8cc 100644 --- a/packages/overture-schema-buildings-theme/tests/building_part_baseline_schema.json +++ b/packages/overture-schema-buildings-theme/tests/building_part_baseline_schema.json @@ -291,8 +291,7 @@ } }, "required": [ - "property", - "dataset" + "property" ], "title": "SourceItem", "type": "object" diff --git a/packages/overture-schema-codegen/tests/test_pyspark_scaffold.py b/packages/overture-schema-codegen/tests/test_pyspark_scaffold.py index 84cdace4f..80d99cb8a 100644 --- a/packages/overture-schema-codegen/tests/test_pyspark_scaffold.py +++ b/packages/overture-schema-codegen/tests/test_pyspark_scaffold.py @@ -140,8 +140,9 @@ def test_array_nested_field_builds_path(self, connector_spec: ModelSpec) -> None assert isinstance(scaffold["sources"], list) assert len(scaffold["sources"]) == 1 elem = scaffold["sources"][0] - # Required sibling 'dataset' populated - assert "dataset" in elem + # Required sibling 'property' populated; optional 'dataset' omitted + assert "property" in elem + assert "dataset" not in elem def test_scaffold_is_dict(self, connector_spec: ModelSpec) -> None: field_nodes, _ = build_checks(connector_spec) diff --git a/packages/overture-schema-common/src/overture/schema/common/sources.py b/packages/overture-schema-common/src/overture/schema/common/sources.py index 0b3dc4cdc..d97c7fd69 100644 --- a/packages/overture-schema-common/src/overture/schema/common/sources.py +++ b/packages/overture-schema-common/src/overture/schema/common/sources.py @@ -44,14 +44,17 @@ class SourceItem(BaseModel): it applies to the feature geometry. """).strip() ) - dataset: str = Field( - description=textwrap.dedent(""" - Name of the dataset where the source data can be found. - """).strip() - ) # Optional + dataset: Annotated[ + str | None, + Field( + description=textwrap.dedent(""" + Name of the dataset where the source data can be found. + """).strip() + ), + ] = None license: Annotated[ StrippedString | None, Field( diff --git a/packages/overture-schema-common/tests/test_models.py b/packages/overture-schema-common/tests/test_models.py index d29d00df7..0b58aa609 100644 --- a/packages/overture-schema-common/tests/test_models.py +++ b/packages/overture-schema-common/tests/test_models.py @@ -85,7 +85,7 @@ def test_feature_json_schema() -> None: }, "confidence": {"maximum": 1.0, "minimum": 0.0, "type": "number"}, }, - "required": ["property", "dataset"], + "required": ["property"], "type": "object", } }, diff --git a/packages/overture-schema-divisions-theme/tests/division_area_baseline_schema.json b/packages/overture-schema-divisions-theme/tests/division_area_baseline_schema.json index 4e44c64d8..2be15985f 100644 --- a/packages/overture-schema-divisions-theme/tests/division_area_baseline_schema.json +++ b/packages/overture-schema-divisions-theme/tests/division_area_baseline_schema.json @@ -250,8 +250,7 @@ } }, "required": [ - "property", - "dataset" + "property" ], "title": "SourceItem", "type": "object" diff --git a/packages/overture-schema-divisions-theme/tests/division_baseline_schema.json b/packages/overture-schema-divisions-theme/tests/division_baseline_schema.json index 487fec682..76a42caa8 100644 --- a/packages/overture-schema-divisions-theme/tests/division_baseline_schema.json +++ b/packages/overture-schema-divisions-theme/tests/division_baseline_schema.json @@ -353,8 +353,7 @@ } }, "required": [ - "property", - "dataset" + "property" ], "title": "SourceItem", "type": "object" diff --git a/packages/overture-schema-divisions-theme/tests/division_boundary_baseline_schema.json b/packages/overture-schema-divisions-theme/tests/division_boundary_baseline_schema.json index 43bb98b03..e9ff5b24f 100644 --- a/packages/overture-schema-divisions-theme/tests/division_boundary_baseline_schema.json +++ b/packages/overture-schema-divisions-theme/tests/division_boundary_baseline_schema.json @@ -140,8 +140,7 @@ } }, "required": [ - "property", - "dataset" + "property" ], "title": "SourceItem", "type": "object" diff --git a/packages/overture-schema-places-theme/tests/place_baseline_schema.json b/packages/overture-schema-places-theme/tests/place_baseline_schema.json index 7009bb3c7..9d18eca6d 100644 --- a/packages/overture-schema-places-theme/tests/place_baseline_schema.json +++ b/packages/overture-schema-places-theme/tests/place_baseline_schema.json @@ -317,8 +317,7 @@ } }, "required": [ - "property", - "dataset" + "property" ], "title": "SourceItem", "type": "object" diff --git a/packages/overture-schema-transportation-theme/tests/connector_baseline_schema.json b/packages/overture-schema-transportation-theme/tests/connector_baseline_schema.json index 14cda8519..04fe5e759 100644 --- a/packages/overture-schema-transportation-theme/tests/connector_baseline_schema.json +++ b/packages/overture-schema-transportation-theme/tests/connector_baseline_schema.json @@ -73,8 +73,7 @@ } }, "required": [ - "property", - "dataset" + "property" ], "title": "SourceItem", "type": "object" diff --git a/packages/overture-schema-transportation-theme/tests/segment_baseline_schema.json b/packages/overture-schema-transportation-theme/tests/segment_baseline_schema.json index 0bf9d1962..a8c7d9304 100644 --- a/packages/overture-schema-transportation-theme/tests/segment_baseline_schema.json +++ b/packages/overture-schema-transportation-theme/tests/segment_baseline_schema.json @@ -1253,8 +1253,7 @@ } }, "required": [ - "property", - "dataset" + "property" ], "title": "SourceItem", "type": "object" diff --git a/schema/defs.yaml b/schema/defs.yaml index 53bf92d92..6ea7b47eb 100644 --- a/schema/defs.yaml +++ b/schema/defs.yaml @@ -269,7 +269,7 @@ description: Common schema definitions shared by all themes Additional metadata such as license, record_id, update_time, and confidence may also be provided. type: object - required: [property, dataset] + required: [property] allOf: - { "$ref": "#/$defs/propertyContainers/geometricRangeScopeContainer" } unevaluatedProperties: false From e4cd7eaa3577cc1ca3d4751594f2258fa5f47789 Mon Sep 17 00:00:00 2001 From: Alex Iannicelli Date: Thu, 6 Aug 2026 09:33:37 -0400 Subject: [PATCH 2/2] Add towncrier changelog fragments Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: Alex Iannicelli --- packages/overture-schema-addresses-theme/changelog.d/633.misc.md | 1 + packages/overture-schema-base-theme/changelog.d/633.misc.md | 1 + packages/overture-schema-buildings-theme/changelog.d/633.misc.md | 1 + packages/overture-schema-codegen/changelog.d/633.misc.md | 1 + packages/overture-schema-common/changelog.d/633.feature.md | 1 + packages/overture-schema-divisions-theme/changelog.d/633.misc.md | 1 + packages/overture-schema-places-theme/changelog.d/633.misc.md | 1 + .../overture-schema-transportation-theme/changelog.d/633.misc.md | 1 + 8 files changed, 8 insertions(+) create mode 100644 packages/overture-schema-addresses-theme/changelog.d/633.misc.md create mode 100644 packages/overture-schema-base-theme/changelog.d/633.misc.md create mode 100644 packages/overture-schema-buildings-theme/changelog.d/633.misc.md create mode 100644 packages/overture-schema-codegen/changelog.d/633.misc.md create mode 100644 packages/overture-schema-common/changelog.d/633.feature.md create mode 100644 packages/overture-schema-divisions-theme/changelog.d/633.misc.md create mode 100644 packages/overture-schema-places-theme/changelog.d/633.misc.md create mode 100644 packages/overture-schema-transportation-theme/changelog.d/633.misc.md diff --git a/packages/overture-schema-addresses-theme/changelog.d/633.misc.md b/packages/overture-schema-addresses-theme/changelog.d/633.misc.md new file mode 100644 index 000000000..c4c80e187 --- /dev/null +++ b/packages/overture-schema-addresses-theme/changelog.d/633.misc.md @@ -0,0 +1 @@ +Regenerated baseline schemas for optional `sources[].dataset`. diff --git a/packages/overture-schema-base-theme/changelog.d/633.misc.md b/packages/overture-schema-base-theme/changelog.d/633.misc.md new file mode 100644 index 000000000..c4c80e187 --- /dev/null +++ b/packages/overture-schema-base-theme/changelog.d/633.misc.md @@ -0,0 +1 @@ +Regenerated baseline schemas for optional `sources[].dataset`. diff --git a/packages/overture-schema-buildings-theme/changelog.d/633.misc.md b/packages/overture-schema-buildings-theme/changelog.d/633.misc.md new file mode 100644 index 000000000..c4c80e187 --- /dev/null +++ b/packages/overture-schema-buildings-theme/changelog.d/633.misc.md @@ -0,0 +1 @@ +Regenerated baseline schemas for optional `sources[].dataset`. diff --git a/packages/overture-schema-codegen/changelog.d/633.misc.md b/packages/overture-schema-codegen/changelog.d/633.misc.md new file mode 100644 index 000000000..4123a1edd --- /dev/null +++ b/packages/overture-schema-codegen/changelog.d/633.misc.md @@ -0,0 +1 @@ +Updated scaffold test expectations for optional `sources[].dataset`. diff --git a/packages/overture-schema-common/changelog.d/633.feature.md b/packages/overture-schema-common/changelog.d/633.feature.md new file mode 100644 index 000000000..e55147bb5 --- /dev/null +++ b/packages/overture-schema-common/changelog.d/633.feature.md @@ -0,0 +1 @@ +Made `sources[].dataset` optional; sources entries no longer require a `dataset` value. diff --git a/packages/overture-schema-divisions-theme/changelog.d/633.misc.md b/packages/overture-schema-divisions-theme/changelog.d/633.misc.md new file mode 100644 index 000000000..c4c80e187 --- /dev/null +++ b/packages/overture-schema-divisions-theme/changelog.d/633.misc.md @@ -0,0 +1 @@ +Regenerated baseline schemas for optional `sources[].dataset`. diff --git a/packages/overture-schema-places-theme/changelog.d/633.misc.md b/packages/overture-schema-places-theme/changelog.d/633.misc.md new file mode 100644 index 000000000..c4c80e187 --- /dev/null +++ b/packages/overture-schema-places-theme/changelog.d/633.misc.md @@ -0,0 +1 @@ +Regenerated baseline schemas for optional `sources[].dataset`. diff --git a/packages/overture-schema-transportation-theme/changelog.d/633.misc.md b/packages/overture-schema-transportation-theme/changelog.d/633.misc.md new file mode 100644 index 000000000..c4c80e187 --- /dev/null +++ b/packages/overture-schema-transportation-theme/changelog.d/633.misc.md @@ -0,0 +1 @@ +Regenerated baseline schemas for optional `sources[].dataset`.