From 12806c14cdd7376a4c0469199e4716e2b87018ce Mon Sep 17 00:00:00 2001 From: Seth Fitzsimmons Date: Wed, 19 Aug 2026 10:50:22 -0700 Subject: [PATCH 1/2] fix(pyspark): make pyspark an optional extra pyspark was a hard dependency, so installing overture-schema-pyspark resolved pyspark on every runtime, including ones (Glue, EMR) that already bundle their own PySpark and don't want this package re-resolving it underneath them. Moved it to a `spark` optional-dependency extra. Standalone environments building their own install overture-schema-pyspark[spark]; runtime-provided-PySpark environments install the bare package. Probe for pyspark in the package's __init__ so a bare install without PySpark raises an actionable ModuleNotFoundError naming the extra rather than "No module named 'pyspark'". __init__ runs on any import of any submodule, generated ones included, so one probe covers every entry path. It catches nothing, so an unrelated missing dependency -- or a pyspark that is present but broken -- still surfaces its own real error from the imports that follow. Supersedes #660, which is where this change and its changelog wording originated. Refs #659 Co-authored-by: Adam Lastowka Signed-off-by: Seth Fitzsimmons --- .../changelog.d/659.bugfix.md | 1 + .../overture-schema-pyspark/pyproject.toml | 4 +- .../src/overture/schema/pyspark/__init__.py | 15 ++++ .../tests/test_optional_pyspark.py | 75 +++++++++++++++++++ uv.lock | 7 +- 5 files changed, 100 insertions(+), 2 deletions(-) create mode 100644 packages/overture-schema-pyspark/changelog.d/659.bugfix.md create mode 100644 packages/overture-schema-pyspark/tests/test_optional_pyspark.py diff --git a/packages/overture-schema-pyspark/changelog.d/659.bugfix.md b/packages/overture-schema-pyspark/changelog.d/659.bugfix.md new file mode 100644 index 000000000..133bc11dd --- /dev/null +++ b/packages/overture-schema-pyspark/changelog.d/659.bugfix.md @@ -0,0 +1 @@ +Moved `pyspark` from a hard dependency to the `spark` optional extra, so installing `overture-schema-pyspark` no longer resolves `pyspark` on runtimes (Glue, EMR) that already bundle their own. Standalone environments need `pip install overture-schema-pyspark[spark]`. Importing the package without PySpark installed now raises an actionable `ModuleNotFoundError` pointing at that extra, instead of a bare "No module named 'pyspark'". diff --git a/packages/overture-schema-pyspark/pyproject.toml b/packages/overture-schema-pyspark/pyproject.toml index 6e72bc93c..a280bf72c 100644 --- a/packages/overture-schema-pyspark/pyproject.toml +++ b/packages/overture-schema-pyspark/pyproject.toml @@ -6,7 +6,6 @@ build-backend = "uv_build" dependencies = [ "click>=8.0", "overture-schema-system>=0.1.1", - "pyspark>=3.4", ] description = "PySpark validation expressions for Overture Maps data" license = "MIT" @@ -14,6 +13,9 @@ name = "overture-schema-pyspark" requires-python = ">=3.10" version = "0.1.1" +[project.optional-dependencies] +spark = ["pyspark>=3.4"] + [project.scripts] overture-validate = "overture.schema.pyspark.cli:validate_cli" diff --git a/packages/overture-schema-pyspark/src/overture/schema/pyspark/__init__.py b/packages/overture-schema-pyspark/src/overture/schema/pyspark/__init__.py index 8841ae9f4..1bc7b3b69 100644 --- a/packages/overture-schema-pyspark/src/overture/schema/pyspark/__init__.py +++ b/packages/overture-schema-pyspark/src/overture/schema/pyspark/__init__.py @@ -1,5 +1,20 @@ """PySpark validation expressions for Overture Maps data.""" +import importlib.util + +# pyspark is an optional extra (the `spark` extra): a bare install lets this +# package's metadata and console script resolve, but every module below needs +# pyspark itself to do anything. Probe for it here -- this module runs on any +# import of any submodule -- so a bare install gets an actionable message. A +# pyspark that is present but broken still raises its own error, because the +# imports below run for real. +if importlib.util.find_spec("pyspark") is None: + raise ModuleNotFoundError( + "overture-schema-pyspark requires PySpark, which isn't installed. " + "Install it with `pip install overture-schema-pyspark[spark]`, or run " + "in an environment that already provides PySpark (e.g. a Spark cluster)." + ) + from .check import Check, CheckShape from .schema_check import SchemaMismatch, compare_schemas from .validate import ( diff --git a/packages/overture-schema-pyspark/tests/test_optional_pyspark.py b/packages/overture-schema-pyspark/tests/test_optional_pyspark.py new file mode 100644 index 000000000..fe1d2be21 --- /dev/null +++ b/packages/overture-schema-pyspark/tests/test_optional_pyspark.py @@ -0,0 +1,75 @@ +"""Tests for the missing-PySpark guard in the package's __init__.""" + +import importlib +import sys +from types import ModuleType + +import pytest + + +class _BlockedFinder: + """A meta_path finder that reports one module as genuinely absent. + + Raising from `find_spec` reproduces what an uninstalled module does at an + `import` statement -- a `ModuleNotFoundError` carrying `name`. Setting + `sys.modules[name] = None` would raise `ImportError` instead, which is not + the error a missing package actually produces. + """ + + def __init__(self, name: str) -> None: + self.name = name + + def find_spec( + self, fullname: str, path: object = None, target: ModuleType | None = None + ) -> None: + if fullname == self.name or fullname.startswith(f"{self.name}."): + raise ModuleNotFoundError(f"No module named {fullname!r}", name=fullname) + return None + + +def _unload(monkeypatch: pytest.MonkeyPatch, *roots: str) -> None: + """Drop modules from sys.modules so importing them re-runs their top level. + + monkeypatch restores the original module objects at teardown, so modules + already imported by the rest of the suite keep their identity. + """ + for name in [ + name + for name in sys.modules + if any(name == root or name.startswith(f"{root}.") for root in roots) + ]: + monkeypatch.delitem(sys.modules, name) + + +def test_missing_pyspark_names_the_extra(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setitem(sys.modules, "pyspark", None) + _unload(monkeypatch, "overture.schema.pyspark") + + with pytest.raises(ModuleNotFoundError) as excinfo: + importlib.import_module("overture.schema.pyspark") + + assert "overture-schema-pyspark[spark]" in str(excinfo.value) + + +def test_unrelated_missing_dependency_surfaces_its_own_error( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The guard reports only a missing pyspark, never another package's.""" + _unload(monkeypatch, "overture.schema.pyspark", "overture.schema.system", "shapely") + monkeypatch.setattr(sys, "meta_path", [_BlockedFinder("shapely"), *sys.meta_path]) + + with pytest.raises(ImportError) as excinfo: + importlib.import_module("overture.schema.pyspark") + + assert "shapely" in str(excinfo.value) + assert "overture-schema-pyspark[spark]" not in str(excinfo.value) + + +def test_package_imports_when_pyspark_is_installed( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _unload(monkeypatch, "overture.schema.pyspark") + + module = importlib.import_module("overture.schema.pyspark") + + assert module.Check.__name__ == "Check" diff --git a/uv.lock b/uv.lock index 5e0bcfd1a..a5a248a3f 100644 --- a/uv.lock +++ b/uv.lock @@ -1042,6 +1042,10 @@ source = { editable = "packages/overture-schema-pyspark" } dependencies = [ { name = "click" }, { name = "overture-schema-system" }, +] + +[package.optional-dependencies] +spark = [ { name = "pyspark" }, ] @@ -1049,8 +1053,9 @@ dependencies = [ requires-dist = [ { name = "click", specifier = ">=8.0" }, { name = "overture-schema-system", editable = "packages/overture-schema-system" }, - { name = "pyspark", specifier = ">=3.4" }, + { name = "pyspark", marker = "extra == 'spark'", specifier = ">=3.4" }, ] +provides-extras = ["spark"] [[package]] name = "overture-schema-system" From 2c1e62e8ed9eccf0d04eef885f04d3fee3a5d149 Mon Sep 17 00:00:00 2001 From: Seth Fitzsimmons Date: Thu, 20 Aug 2026 10:55:24 -0700 Subject: [PATCH 2/2] fix(pyspark): enforce the declared PySpark floor at import Moving pyspark to an extra left the `>=3.4` floor unenforced on exactly the install the extra exists for: a bare install on a runtime that brings its own PySpark, where no resolver ever sees the requirement. The extra itself still carries the floor -- `overture-schema-pyspark[spark]` with `pyspark==3.3.4` resolves as unsatisfiable -- so only the bring-your-own path was open. Check it at import instead, reading the floor back out of the installed distribution's Requires-Dist metadata rather than restating it in code, so there is still one declaration and the check cannot drift from pyproject.toml. The installed version comes from `pyspark.__version__`, not PySpark's own metadata: a PySpark supplied by a Spark distribution sits on sys.path with no dist-info to read. Both lookups fail open by design -- a PySpark that reports no version, or a source tree with no metadata installed, leaves nothing to judge -- and a distribution name that stopped resolving would fail open the same silent way, so a test pins that the one named here finds real metadata. Prereleases satisfy the floor, since Spark ships release candidates and dev builds. Adds a `packaging` dependency for requirement and version parsing. Refs #659 Signed-off-by: Seth Fitzsimmons --- .../changelog.d/659.bugfix.md | 2 +- .../overture-schema-pyspark/pyproject.toml | 1 + .../src/overture/schema/pyspark/__init__.py | 10 +++ .../schema/pyspark/_pyspark_version.py | 62 ++++++++++++++ .../tests/test_optional_pyspark.py | 80 ++++++++++++++++++- uv.lock | 2 + 6 files changed, 155 insertions(+), 2 deletions(-) create mode 100644 packages/overture-schema-pyspark/src/overture/schema/pyspark/_pyspark_version.py diff --git a/packages/overture-schema-pyspark/changelog.d/659.bugfix.md b/packages/overture-schema-pyspark/changelog.d/659.bugfix.md index 133bc11dd..b4d382763 100644 --- a/packages/overture-schema-pyspark/changelog.d/659.bugfix.md +++ b/packages/overture-schema-pyspark/changelog.d/659.bugfix.md @@ -1 +1 @@ -Moved `pyspark` from a hard dependency to the `spark` optional extra, so installing `overture-schema-pyspark` no longer resolves `pyspark` on runtimes (Glue, EMR) that already bundle their own. Standalone environments need `pip install overture-schema-pyspark[spark]`. Importing the package without PySpark installed now raises an actionable `ModuleNotFoundError` pointing at that extra, instead of a bare "No module named 'pyspark'". +Moved `pyspark` from a hard dependency to the `spark` optional extra, so installing `overture-schema-pyspark` no longer resolves `pyspark` on runtimes (Glue, EMR) that already bundle their own. Standalone environments need `pip install overture-schema-pyspark[spark]`. Importing the package without PySpark installed now raises an actionable `ModuleNotFoundError` pointing at that extra, instead of a bare "No module named 'pyspark'". Importing it with a PySpark older than the declared floor raises an `ImportError` naming both versions, so the floor still applies on installs that skip the extra and bring their own PySpark. diff --git a/packages/overture-schema-pyspark/pyproject.toml b/packages/overture-schema-pyspark/pyproject.toml index a280bf72c..10a190395 100644 --- a/packages/overture-schema-pyspark/pyproject.toml +++ b/packages/overture-schema-pyspark/pyproject.toml @@ -6,6 +6,7 @@ build-backend = "uv_build" dependencies = [ "click>=8.0", "overture-schema-system>=0.1.1", + "packaging>=22", ] description = "PySpark validation expressions for Overture Maps data" license = "MIT" diff --git a/packages/overture-schema-pyspark/src/overture/schema/pyspark/__init__.py b/packages/overture-schema-pyspark/src/overture/schema/pyspark/__init__.py index 1bc7b3b69..7af28fa48 100644 --- a/packages/overture-schema-pyspark/src/overture/schema/pyspark/__init__.py +++ b/packages/overture-schema-pyspark/src/overture/schema/pyspark/__init__.py @@ -2,6 +2,8 @@ import importlib.util +from ._pyspark_version import pyspark_version_problem + # pyspark is an optional extra (the `spark` extra): a bare install lets this # package's metadata and console script resolve, but every module below needs # pyspark itself to do anything. Probe for it here -- this module runs on any @@ -15,6 +17,14 @@ "in an environment that already provides PySpark (e.g. a Spark cluster)." ) +import pyspark + +# Installing without the extra leaves no resolver to enforce the version floor +# declared alongside it, so enforce it here, against the PySpark that actually +# turned up. The floor is read back out of this package's own metadata. +if _problem := pyspark_version_problem(getattr(pyspark, "__version__", None)): + raise ImportError(_problem) + from .check import Check, CheckShape from .schema_check import SchemaMismatch, compare_schemas from .validate import ( diff --git a/packages/overture-schema-pyspark/src/overture/schema/pyspark/_pyspark_version.py b/packages/overture-schema-pyspark/src/overture/schema/pyspark/_pyspark_version.py new file mode 100644 index 000000000..7b2410c95 --- /dev/null +++ b/packages/overture-schema-pyspark/src/overture/schema/pyspark/_pyspark_version.py @@ -0,0 +1,62 @@ +"""The PySpark version floor this package declares, checked at import time.""" + +import importlib.metadata + +from packaging.requirements import Requirement +from packaging.specifiers import SpecifierSet +from packaging.utils import canonicalize_name + +# The distribution this module ships in. A name that stopped resolving would +# disable the check below in silence -- no floor found reads exactly like a +# floor satisfied -- so a test pins that this one finds real metadata. +_DISTRIBUTION = "overture-schema-pyspark" + + +def declared_pyspark_specifier() -> SpecifierSet | None: + """Return the PySpark version range this package declares, if readable. + + The range lives in pyproject.toml (`pyspark>=N` on the `spark` extra) and + reaches the installed distribution as a `Requires-Dist` entry. Reading it + back from there rather than restating it here means one declaration, so a + check against it cannot drift from what the package actually requires. + + Returns None when the metadata isn't installed -- a source tree run + without an install -- because there is then no declaration to enforce. + """ + try: + declared = importlib.metadata.requires(_DISTRIBUTION) or () + except importlib.metadata.PackageNotFoundError: + return None + for raw in declared: + requirement = Requirement(raw) + if canonicalize_name(requirement.name) == "pyspark": + return requirement.specifier + return None + + +def pyspark_version_problem(version: str | None) -> str | None: + """Describe how `version` falls outside the declared range, or None. + + Installing without the `spark` extra is the case the extra exists for -- + a runtime that provides its own PySpark -- and it is also the case no + resolver sees, so nothing enforces the range at install time. This is + where it gets enforced instead. + + `version` is `pyspark.__version__` rather than the version recorded in + PySpark's own metadata, because a PySpark supplied by a Spark + distribution is on `sys.path` without a `dist-info` directory to read. + A PySpark that reports no version passes: unjudgeable is not out of range. + """ + specifier = declared_pyspark_specifier() + if version is None or specifier is None: + return None + # Prereleases count: Spark ships release candidates and dev builds, and a + # `>=` floor is a statement about the release they belong to. + if specifier.contains(version, prereleases=True): + return None + return ( + f"overture-schema-pyspark requires PySpark {specifier}, but PySpark " + f"{version} is installed. Upgrade the PySpark in this environment, or " + f"install this package with its extra (`pip install " + f"overture-schema-pyspark[spark]`) to let the resolver choose one." + ) diff --git a/packages/overture-schema-pyspark/tests/test_optional_pyspark.py b/packages/overture-schema-pyspark/tests/test_optional_pyspark.py index fe1d2be21..89054b7fb 100644 --- a/packages/overture-schema-pyspark/tests/test_optional_pyspark.py +++ b/packages/overture-schema-pyspark/tests/test_optional_pyspark.py @@ -1,11 +1,15 @@ -"""Tests for the missing-PySpark guard in the package's __init__.""" +"""Tests for the PySpark guards in the package's __init__.""" import importlib +import importlib.metadata import sys from types import ModuleType +import pyspark import pytest +from overture.schema.pyspark._pyspark_version import declared_pyspark_specifier + class _BlockedFinder: """A meta_path finder that reports one module as genuinely absent. @@ -73,3 +77,77 @@ def test_package_imports_when_pyspark_is_installed( module = importlib.import_module("overture.schema.pyspark") assert module.Check.__name__ == "Check" + + +def test_pyspark_below_the_declared_floor_names_both_versions( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(pyspark, "__version__", "3.3.4") + _unload(monkeypatch, "overture.schema.pyspark") + + with pytest.raises(ImportError) as excinfo: + importlib.import_module("overture.schema.pyspark") + + message = str(excinfo.value) + assert "3.3.4" in message + assert ">=3.4" in message + + +def test_the_floor_is_read_from_package_metadata( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The declared floor is the one that applies, not one restated in code. + + A guard carrying its own copy of the version passes the test above and + fails this one, which is the drift this indirection exists to prevent. + """ + monkeypatch.setattr( + importlib.metadata, + "requires", + lambda name: ["pyspark>=99.0 ; extra == 'spark'"], + ) + _unload(monkeypatch, "overture.schema.pyspark") + + with pytest.raises(ImportError) as excinfo: + importlib.import_module("overture.schema.pyspark") + + assert ">=99.0" in str(excinfo.value) + assert pyspark.__version__ in str(excinfo.value) # type: ignore[attr-defined] + + +def test_unreadable_package_metadata_does_not_block_import( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """No declaration to read means no floor to enforce, not a failed import.""" + + def _absent(name: str) -> list[str]: + raise importlib.metadata.PackageNotFoundError(name) + + monkeypatch.setattr(importlib.metadata, "requires", _absent) + _unload(monkeypatch, "overture.schema.pyspark") + + module = importlib.import_module("overture.schema.pyspark") + + assert module.Check.__name__ == "Check" + + +def test_pyspark_without_a_reported_version_does_not_block_import( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A PySpark that reports no version is unjudgeable, not out of range.""" + monkeypatch.delattr(pyspark, "__version__") + _unload(monkeypatch, "overture.schema.pyspark") + + module = importlib.import_module("overture.schema.pyspark") + + assert module.Check.__name__ == "Check" + + +def test_the_declared_floor_is_readable_here() -> None: + """The metadata lookup resolves, so the guard is live and not inert. + + Every other test in this file passes whether or not it does: a lookup that + finds nothing reports no floor, which reads exactly like a floor that is + satisfied. + """ + assert declared_pyspark_specifier() is not None diff --git a/uv.lock b/uv.lock index a5a248a3f..aca61aa76 100644 --- a/uv.lock +++ b/uv.lock @@ -1042,6 +1042,7 @@ source = { editable = "packages/overture-schema-pyspark" } dependencies = [ { name = "click" }, { name = "overture-schema-system" }, + { name = "packaging" }, ] [package.optional-dependencies] @@ -1053,6 +1054,7 @@ spark = [ requires-dist = [ { name = "click", specifier = ">=8.0" }, { name = "overture-schema-system", editable = "packages/overture-schema-system" }, + { name = "packaging", specifier = ">=22" }, { name = "pyspark", marker = "extra == 'spark'", specifier = ">=3.4" }, ] provides-extras = ["spark"]