From 61c37f2cc4b4728b750695268b3d99f5a6163609 Mon Sep 17 00:00:00 2001 From: Adam Lastowka Date: Thu, 13 Aug 2026 14:30:35 -0400 Subject: [PATCH 1/3] [FIX] Make pyspark an optional extra on overture-schema-pyspark pyspark was a hard dependency, so installing overture-schema-pyspark resolved pyspark on every runtime, including ones (Glue, EMR) that already bundle their own PySpark and don't want this package re-resolving it. Moved it to a spark optional-dependency extra instead. Standalone environments building their own PySpark now install overture-schema-pyspark[spark]; runtime-provided-PySpark environments install the bare package. No other package in this repo depends on overture-schema-pyspark or imports pyspark directly, and CI's make check already runs uv sync --all-extras, so test coverage is unaffected. Signed-off-by: Adam Lastowka --- .../overture-schema-pyspark/changelog.d/659.bugfix.md | 1 + packages/overture-schema-pyspark/pyproject.toml | 4 +++- uv.lock | 11 ++++++++--- 3 files changed, 12 insertions(+), 4 deletions(-) create mode 100644 packages/overture-schema-pyspark/changelog.d/659.bugfix.md diff --git a/packages/overture-schema-pyspark/changelog.d/659.bugfix.md b/packages/overture-schema-pyspark/changelog.d/659.bugfix.md new file mode 100644 index 000000000..920bcb523 --- /dev/null +++ b/packages/overture-schema-pyspark/changelog.d/659.bugfix.md @@ -0,0 +1 @@ +Moved `pyspark` from a hard dependency to the `spark` optional extra, so installing `overture-schema-pyspark` no longer resolves `pyspark` on runtimes (Glue, EMR) that already bundle their own. Standalone environments need `pip install overture-schema-pyspark[spark]`. diff --git a/packages/overture-schema-pyspark/pyproject.toml b/packages/overture-schema-pyspark/pyproject.toml index 6e72bc93c..a280bf72c 100644 --- a/packages/overture-schema-pyspark/pyproject.toml +++ b/packages/overture-schema-pyspark/pyproject.toml @@ -6,7 +6,6 @@ build-backend = "uv_build" dependencies = [ "click>=8.0", "overture-schema-system>=0.1.1", - "pyspark>=3.4", ] description = "PySpark validation expressions for Overture Maps data" license = "MIT" @@ -14,6 +13,9 @@ name = "overture-schema-pyspark" requires-python = ">=3.10" version = "0.1.1" +[project.optional-dependencies] +spark = ["pyspark>=3.4"] + [project.scripts] overture-validate = "overture.schema.pyspark.cli:validate_cli" diff --git a/uv.lock b/uv.lock index f9ecf5b6c..9330abc55 100644 --- a/uv.lock +++ b/uv.lock @@ -9,7 +9,7 @@ resolution-markers = [ ] [options] -exclude-newer = "0001-01-01T00:00:00Z" # This has no effect and is included for backwards compatibility when using relative exclude-newer values. +exclude-newer = "2026-08-06T18:30:08.560594Z" exclude-newer-span = "P1W" [manifest] @@ -363,7 +363,7 @@ name = "exceptiongroup" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } wheels = [ @@ -1042,6 +1042,10 @@ source = { editable = "packages/overture-schema-pyspark" } dependencies = [ { name = "click" }, { name = "overture-schema-system" }, +] + +[package.optional-dependencies] +spark = [ { name = "pyspark" }, ] @@ -1049,8 +1053,9 @@ dependencies = [ requires-dist = [ { name = "click", specifier = ">=8.0" }, { name = "overture-schema-system", editable = "packages/overture-schema-system" }, - { name = "pyspark", specifier = ">=3.4" }, + { name = "pyspark", marker = "extra == 'spark'", specifier = ">=3.4" }, ] +provides-extras = ["spark"] [[package]] name = "overture-schema-system" From 70db0ab9046f077cdd0a50a1234737cb343e1e96 Mon Sep 17 00:00:00 2001 From: Adam Lastowka Date: Thu, 13 Aug 2026 21:23:35 -0400 Subject: [PATCH 2/3] Raise actionable error when pyspark extra is missing Importing overture.schema.pyspark without the spark extra installed surfaced a bare ModuleNotFoundError: No module named 'pyspark', with no indication that installing overture-schema-pyspark[spark] fixes it. Guard the package's top-level imports and re-raise with an actionable message pointing at the extra. Narrowed to ModuleNotFoundError on pyspark specifically, so a genuinely broken pyspark install or an unrelated missing dependency still surfaces its own real error. Signed-off-by: Adam Lastowka --- .../changelog.d/659.bugfix.md | 2 +- .../src/overture/schema/pyspark/__init__.py | 36 +++++++++++++------ 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/packages/overture-schema-pyspark/changelog.d/659.bugfix.md b/packages/overture-schema-pyspark/changelog.d/659.bugfix.md index 920bcb523..133bc11dd 100644 --- a/packages/overture-schema-pyspark/changelog.d/659.bugfix.md +++ b/packages/overture-schema-pyspark/changelog.d/659.bugfix.md @@ -1 +1 @@ -Moved `pyspark` from a hard dependency to the `spark` optional extra, so installing `overture-schema-pyspark` no longer resolves `pyspark` on runtimes (Glue, EMR) that already bundle their own. Standalone environments need `pip install overture-schema-pyspark[spark]`. +Moved `pyspark` from a hard dependency to the `spark` optional extra, so installing `overture-schema-pyspark` no longer resolves `pyspark` on runtimes (Glue, EMR) that already bundle their own. Standalone environments need `pip install overture-schema-pyspark[spark]`. Importing the package without PySpark installed now raises an actionable `ModuleNotFoundError` pointing at that extra, instead of a bare "No module named 'pyspark'". diff --git a/packages/overture-schema-pyspark/src/overture/schema/pyspark/__init__.py b/packages/overture-schema-pyspark/src/overture/schema/pyspark/__init__.py index 8841ae9f4..06d8ad47e 100644 --- a/packages/overture-schema-pyspark/src/overture/schema/pyspark/__init__.py +++ b/packages/overture-schema-pyspark/src/overture/schema/pyspark/__init__.py @@ -1,16 +1,30 @@ """PySpark validation expressions for Overture Maps data.""" -from .check import Check, CheckShape -from .schema_check import SchemaMismatch, compare_schemas -from .validate import ( - ValidationResult, - evaluate_checks, - explain_errors, - filter_errors, - model_keys, - model_names, - validate_model, -) +try: + from .check import Check, CheckShape + from .schema_check import SchemaMismatch, compare_schemas + from .validate import ( + ValidationResult, + evaluate_checks, + explain_errors, + filter_errors, + model_keys, + model_names, + validate_model, + ) +except ModuleNotFoundError as e: + # pyspark is an optional extra (the `spark` extra): a bare install lets + # this package's console script and metadata resolve, but every submodule + # needs pyspark itself to do anything. Narrow to exactly that case so a + # genuinely broken pyspark install (or an unrelated missing module) still + # raises its own real error instead of this actionable one. + if e.name == "pyspark" or (e.name or "").startswith("pyspark."): + raise ModuleNotFoundError( + "overture-schema-pyspark requires PySpark, which isn't installed. " + "Install it with `pip install overture-schema-pyspark[spark]`, or run " + "in an environment that already provides PySpark (e.g. a Spark cluster)." + ) from e + raise __all__ = [ "Check", From 0027e5efd2607ab4031263d3e1004df888a786f6 Mon Sep 17 00:00:00 2001 From: Adam Lastowka Date: Fri, 14 Aug 2026 02:02:18 -0400 Subject: [PATCH 3/3] Guard pyspark import directly, not the re-export chain Testing the optional dependency at a direct import of pyspark is more precise than wrapping the package's own re-exports and disambiguating the exception by module name: an unrelated missing dependency now surfaces its own error through the real imports instead of being caught and inspected. Per review feedback on the extras change. Signed-off-by: Adam Lastowka --- .../src/overture/schema/pyspark/__init__.py | 46 +++++++++---------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/packages/overture-schema-pyspark/src/overture/schema/pyspark/__init__.py b/packages/overture-schema-pyspark/src/overture/schema/pyspark/__init__.py index 06d8ad47e..33b794e27 100644 --- a/packages/overture-schema-pyspark/src/overture/schema/pyspark/__init__.py +++ b/packages/overture-schema-pyspark/src/overture/schema/pyspark/__init__.py @@ -1,30 +1,30 @@ """PySpark validation expressions for Overture Maps data.""" +# pyspark is an optional extra (the `spark` extra): a bare install lets this +# package's console script and metadata resolve, but every submodule needs +# pyspark to do anything. Guard the import directly so a missing pyspark gives +# an actionable message; anything else this package imports surfaces its own +# error through the real imports below. try: - from .check import Check, CheckShape - from .schema_check import SchemaMismatch, compare_schemas - from .validate import ( - ValidationResult, - evaluate_checks, - explain_errors, - filter_errors, - model_keys, - model_names, - validate_model, - ) + import pyspark # noqa: F401 except ModuleNotFoundError as e: - # pyspark is an optional extra (the `spark` extra): a bare install lets - # this package's console script and metadata resolve, but every submodule - # needs pyspark itself to do anything. Narrow to exactly that case so a - # genuinely broken pyspark install (or an unrelated missing module) still - # raises its own real error instead of this actionable one. - if e.name == "pyspark" or (e.name or "").startswith("pyspark."): - raise ModuleNotFoundError( - "overture-schema-pyspark requires PySpark, which isn't installed. " - "Install it with `pip install overture-schema-pyspark[spark]`, or run " - "in an environment that already provides PySpark (e.g. a Spark cluster)." - ) from e - raise + raise ModuleNotFoundError( + "overture-schema-pyspark requires PySpark, which isn't installed. " + "Install it with `pip install overture-schema-pyspark[spark]`, or run " + "in an environment that already provides PySpark (e.g. a Spark cluster)." + ) from e + +from .check import Check, CheckShape +from .schema_check import SchemaMismatch, compare_schemas +from .validate import ( + ValidationResult, + evaluate_checks, + explain_errors, + filter_errors, + model_keys, + model_names, + validate_model, +) __all__ = [ "Check",