Adam Lastowka (Rachmanin0xFF) · Aug 13, 2026 · Aug 14, 2026 · Aug 14, 2026 · Aug 14, 2026
diff --git a/packages/overture-schema-pyspark/changelog.d/659.bugfix.md b/packages/overture-schema-pyspark/changelog.d/659.bugfix.md
@@ -0,0 +1 @@
+Moved `pyspark` from a hard dependency to the `spark` optional extra, so installing `overture-schema-pyspark` no longer resolves `pyspark` on runtimes (Glue, EMR) that already bundle their own. Standalone environments need `pip install overture-schema-pyspark[spark]`. Importing the package without PySpark installed now raises an actionable `ModuleNotFoundError` pointing at that extra, instead of a bare "No module named 'pyspark'".
diff --git a/packages/overture-schema-pyspark/pyproject.toml b/packages/overture-schema-pyspark/pyproject.toml
@@ -6,14 +6,16 @@ build-backend = "uv_build"
 dependencies = [
     "click>=8.0",
     "overture-schema-system>=0.1.1",
-    "pyspark>=3.4",
 ]
 description = "PySpark validation expressions for Overture Maps data"
 license = "MIT"
 name = "overture-schema-pyspark"
 requires-python = ">=3.10"
 version = "0.1.1"
 
+[project.optional-dependencies]
+spark = ["pyspark>=3.4"]
+
 [project.scripts]
 overture-validate = "overture.schema.pyspark.cli:validate_cli"
 

diff --git a/packages/overture-schema-pyspark/src/overture/schema/pyspark/__init__.py b/packages/overture-schema-pyspark/src/overture/schema/pyspark/__init__.py
@@ -1,5 +1,19 @@
 """PySpark validation expressions for Overture Maps data."""
 
+# pyspark is an optional extra (the `spark` extra): a bare install lets this
+# package's console script and metadata resolve, but every submodule needs
+# pyspark to do anything. Guard the import directly so a missing pyspark gives
+# an actionable message; anything else this package imports surfaces its own
+# error through the real imports below.
+try:
+    import pyspark  # noqa: F401
+except ModuleNotFoundError as e:
+    raise ModuleNotFoundError(
+        "overture-schema-pyspark requires PySpark, which isn't installed. "
+        "Install it with `pip install overture-schema-pyspark[spark]`, or run "
+        "in an environment that already provides PySpark (e.g. a Spark cluster)."
+    ) from e
+
 from .check import Check, CheckShape
 from .schema_check import SchemaMismatch, compare_schemas
 from .validate import (

diff --git a/uv.lock b/uv.lock
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		Moved `pyspark` from a hard dependency to the `spark` optional extra, so installing `overture-schema-pyspark` no longer resolves `pyspark` on runtimes (Glue, EMR) that already bundle their own. Standalone environments need `pip install overture-schema-pyspark[spark]`. Importing the package without PySpark installed now raises an actionable `ModuleNotFoundError` pointing at that extra, instead of a bare "No module named 'pyspark'".