[BUG] Make pyspark an optional extra on overture-schema-pyspark

packages/overture-schema-pyspark/changelog.d/659.bugfix.md

@@ -0,0 +1 @@
+Moved `pyspark` from a hard dependency to the `spark` optional extra, so installing `overture-schema-pyspark` no longer resolves `pyspark` on runtimes (Glue, EMR) that already bundle their own. Standalone environments need `pip install overture-schema-pyspark[spark]`. Importing the package without PySpark installed now raises an actionable `ModuleNotFoundError` pointing at that extra, instead of a bare "No module named 'pyspark'". Importing it with a PySpark older than the declared floor raises an `ImportError` naming both versions, so the floor still applies on installs that skip the extra and bring their own PySpark.

packages/overture-schema-pyspark/pyproject.toml

@@ -6,14 +6,17 @@ build-backend = "uv_build"
 dependencies = [
     "click>=8.0",
     "overture-schema-system>=0.1.1",
-    "pyspark>=3.4",
+    "packaging>=22",
 ]
 description = "PySpark validation expressions for Overture Maps data"
 license = "MIT"
 name = "overture-schema-pyspark"
 requires-python = ">=3.10"
 version = "0.1.1"
 
+[project.optional-dependencies]
+spark = ["pyspark>=3.4"]
+
 [project.scripts]
 overture-validate = "overture.schema.pyspark.cli:validate_cli"
 

packages/overture-schema-pyspark/src/overture/schema/pyspark/__init__.py

@@ -1,5 +1,30 @@
 """PySpark validation expressions for Overture Maps data."""
 
+import importlib.util
+
+from ._pyspark_version import pyspark_version_problem
+
+# pyspark is an optional extra (the `spark` extra): a bare install lets this
+# package's metadata and console script resolve, but every module below needs
+# pyspark itself to do anything. Probe for it here -- this module runs on any
+# import of any submodule -- so a bare install gets an actionable message. A
+# pyspark that is present but broken still raises its own error, because the
+# imports below run for real.
+if importlib.util.find_spec("pyspark") is None:
+    raise ModuleNotFoundError(
+        "overture-schema-pyspark requires PySpark, which isn't installed. "
+        "Install it with `pip install overture-schema-pyspark[spark]`, or run "
+        "in an environment that already provides PySpark (e.g. a Spark cluster)."
+    )
+
+import pyspark
+
+# Installing without the extra leaves no resolver to enforce the version floor
+# declared alongside it, so enforce it here, against the PySpark that actually
+# turned up. The floor is read back out of this package's own metadata.
+if _problem := pyspark_version_problem(getattr(pyspark, "__version__", None)):
+    raise ImportError(_problem)
+
 from .check import Check, CheckShape
 from .schema_check import SchemaMismatch, compare_schemas
 from .validate import (

packages/overture-schema-pyspark/src/overture/schema/pyspark/_pyspark_version.py

@@ -0,0 +1,62 @@
+"""The PySpark version floor this package declares, checked at import time."""
+
+import importlib.metadata
+
+from packaging.requirements import Requirement
+from packaging.specifiers import SpecifierSet
+from packaging.utils import canonicalize_name
+
+# The distribution this module ships in. A name that stopped resolving would
+# disable the check below in silence -- no floor found reads exactly like a
+# floor satisfied -- so a test pins that this one finds real metadata.
+_DISTRIBUTION = "overture-schema-pyspark"
+
+
+def declared_pyspark_specifier() -> SpecifierSet | None:
+    """Return the PySpark version range this package declares, if readable.
+
+    The range lives in pyproject.toml (`pyspark>=N` on the `spark` extra) and
+    reaches the installed distribution as a `Requires-Dist` entry. Reading it
+    back from there rather than restating it here means one declaration, so a
+    check against it cannot drift from what the package actually requires.
+
+    Returns None when the metadata isn't installed -- a source tree run
+    without an install -- because there is then no declaration to enforce.
+    """
+    try:
+        declared = importlib.metadata.requires(_DISTRIBUTION) or ()
+    except importlib.metadata.PackageNotFoundError:
+        return None
+    for raw in declared:
+        requirement = Requirement(raw)
+        if canonicalize_name(requirement.name) == "pyspark":
+            return requirement.specifier
+    return None
+
+
+def pyspark_version_problem(version: str | None) -> str | None:
+    """Describe how `version` falls outside the declared range, or None.
+
+    Installing without the `spark` extra is the case the extra exists for --
+    a runtime that provides its own PySpark -- and it is also the case no
+    resolver sees, so nothing enforces the range at install time. This is
+    where it gets enforced instead.
+
+    `version` is `pyspark.__version__` rather than the version recorded in
+    PySpark's own metadata, because a PySpark supplied by a Spark
+    distribution is on `sys.path` without a `dist-info` directory to read.
+    A PySpark that reports no version passes: unjudgeable is not out of range.
+    """
+    specifier = declared_pyspark_specifier()
+    if version is None or specifier is None:
+        return None
+    # Prereleases count: Spark ships release candidates and dev builds, and a
+    # `>=` floor is a statement about the release they belong to.
+    if specifier.contains(version, prereleases=True):
+        return None
+    return (
+        f"overture-schema-pyspark requires PySpark {specifier}, but PySpark "
+        f"{version} is installed. Upgrade the PySpark in this environment, or "
+        f"install this package with its extra (`pip install "
+        f"overture-schema-pyspark[spark]`) to let the resolver choose one."
+    )

packages/overture-schema-pyspark/tests/test_optional_pyspark.py

@@ -0,0 +1,153 @@
+"""Tests for the PySpark guards in the package's __init__."""
+
+import importlib
+import importlib.metadata
+import sys
+from types import ModuleType
+
+import pyspark
+import pytest
+
+from overture.schema.pyspark._pyspark_version import declared_pyspark_specifier
+
+
+class _BlockedFinder:
+    """A meta_path finder that reports one module as genuinely absent.
+
+    Raising from `find_spec` reproduces what an uninstalled module does at an
+    `import` statement -- a `ModuleNotFoundError` carrying `name`. Setting
+    `sys.modules[name] = None` would raise `ImportError` instead, which is not
+    the error a missing package actually produces.
+    """
+
+    def __init__(self, name: str) -> None:
+        self.name = name
+
+    def find_spec(
+        self, fullname: str, path: object = None, target: ModuleType | None = None
+    ) -> None:
+        if fullname == self.name or fullname.startswith(f"{self.name}."):
+            raise ModuleNotFoundError(f"No module named {fullname!r}", name=fullname)
+        return None
+
+
+def _unload(monkeypatch: pytest.MonkeyPatch, *roots: str) -> None:
+    """Drop modules from sys.modules so importing them re-runs their top level.
+
+    monkeypatch restores the original module objects at teardown, so modules
+    already imported by the rest of the suite keep their identity.
+    """
+    for name in [
+        name
+        for name in sys.modules
+        if any(name == root or name.startswith(f"{root}.") for root in roots)
+    ]:
+        monkeypatch.delitem(sys.modules, name)
+
+
+def test_missing_pyspark_names_the_extra(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setitem(sys.modules, "pyspark", None)
+    _unload(monkeypatch, "overture.schema.pyspark")
+
+    with pytest.raises(ModuleNotFoundError) as excinfo:
+        importlib.import_module("overture.schema.pyspark")
+
+    assert "overture-schema-pyspark[spark]" in str(excinfo.value)
+
+
+def test_unrelated_missing_dependency_surfaces_its_own_error(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """The guard reports only a missing pyspark, never another package's."""
+    _unload(monkeypatch, "overture.schema.pyspark", "overture.schema.system", "shapely")
+    monkeypatch.setattr(sys, "meta_path", [_BlockedFinder("shapely"), *sys.meta_path])
+
+    with pytest.raises(ImportError) as excinfo:
+        importlib.import_module("overture.schema.pyspark")
+
+    assert "shapely" in str(excinfo.value)
+    assert "overture-schema-pyspark[spark]" not in str(excinfo.value)
+
+
+def test_package_imports_when_pyspark_is_installed(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    _unload(monkeypatch, "overture.schema.pyspark")
+
+    module = importlib.import_module("overture.schema.pyspark")
+
+    assert module.Check.__name__ == "Check"
+
+
+def test_pyspark_below_the_declared_floor_names_both_versions(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setattr(pyspark, "__version__", "3.3.4")
+    _unload(monkeypatch, "overture.schema.pyspark")
+
+    with pytest.raises(ImportError) as excinfo:
+        importlib.import_module("overture.schema.pyspark")
+
+    message = str(excinfo.value)
+    assert "3.3.4" in message
+    assert ">=3.4" in message
+
+
+def test_the_floor_is_read_from_package_metadata(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """The declared floor is the one that applies, not one restated in code.
+
+    A guard carrying its own copy of the version passes the test above and
+    fails this one, which is the drift this indirection exists to prevent.
+    """
+    monkeypatch.setattr(
+        importlib.metadata,
+        "requires",
+        lambda name: ["pyspark>=99.0 ; extra == 'spark'"],
+    )
+    _unload(monkeypatch, "overture.schema.pyspark")
+
+    with pytest.raises(ImportError) as excinfo:
+        importlib.import_module("overture.schema.pyspark")
+
+    assert ">=99.0" in str(excinfo.value)
+    assert pyspark.__version__ in str(excinfo.value)  # type: ignore[attr-defined]
+
+
+def test_unreadable_package_metadata_does_not_block_import(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """No declaration to read means no floor to enforce, not a failed import."""
+
+    def _absent(name: str) -> list[str]:
+        raise importlib.metadata.PackageNotFoundError(name)
+
+    monkeypatch.setattr(importlib.metadata, "requires", _absent)
+    _unload(monkeypatch, "overture.schema.pyspark")
+
+    module = importlib.import_module("overture.schema.pyspark")
+
+    assert module.Check.__name__ == "Check"
+
+
+def test_pyspark_without_a_reported_version_does_not_block_import(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """A PySpark that reports no version is unjudgeable, not out of range."""
+    monkeypatch.delattr(pyspark, "__version__")
+    _unload(monkeypatch, "overture.schema.pyspark")
+
+    module = importlib.import_module("overture.schema.pyspark")
+
+    assert module.Check.__name__ == "Check"
+
+
+def test_the_declared_floor_is_readable_here() -> None:
+    """The metadata lookup resolves, so the guard is live and not inert.
+
+    Every other test in this file passes whether or not it does: a lookup that
+    finds nothing reports no floor, which reads exactly like a floor that is
+    satisfied.
+    """
+    assert declared_pyspark_specifier() is not None