diff --git a/packages/overture-schema-pyspark/changelog.d/659.bugfix.md b/packages/overture-schema-pyspark/changelog.d/659.bugfix.md new file mode 100644 index 000000000..133bc11dd --- /dev/null +++ b/packages/overture-schema-pyspark/changelog.d/659.bugfix.md @@ -0,0 +1 @@ +Moved `pyspark` from a hard dependency to the `spark` optional extra, so installing `overture-schema-pyspark` no longer resolves `pyspark` on runtimes (Glue, EMR) that already bundle their own. Standalone environments need `pip install overture-schema-pyspark[spark]`. Importing the package without PySpark installed now raises an actionable `ModuleNotFoundError` pointing at that extra, instead of a bare "No module named 'pyspark'". diff --git a/packages/overture-schema-pyspark/pyproject.toml b/packages/overture-schema-pyspark/pyproject.toml index 6e72bc93c..a280bf72c 100644 --- a/packages/overture-schema-pyspark/pyproject.toml +++ b/packages/overture-schema-pyspark/pyproject.toml @@ -6,7 +6,6 @@ build-backend = "uv_build" dependencies = [ "click>=8.0", "overture-schema-system>=0.1.1", - "pyspark>=3.4", ] description = "PySpark validation expressions for Overture Maps data" license = "MIT" @@ -14,6 +13,9 @@ name = "overture-schema-pyspark" requires-python = ">=3.10" version = "0.1.1" +[project.optional-dependencies] +spark = ["pyspark>=3.4"] + [project.scripts] overture-validate = "overture.schema.pyspark.cli:validate_cli" diff --git a/packages/overture-schema-pyspark/src/overture/schema/pyspark/__init__.py b/packages/overture-schema-pyspark/src/overture/schema/pyspark/__init__.py index 8841ae9f4..33b794e27 100644 --- a/packages/overture-schema-pyspark/src/overture/schema/pyspark/__init__.py +++ b/packages/overture-schema-pyspark/src/overture/schema/pyspark/__init__.py @@ -1,5 +1,19 @@ """PySpark validation expressions for Overture Maps data.""" +# pyspark is an optional extra (the `spark` extra): a bare install lets this +# package's console script and metadata resolve, but every submodule needs +# pyspark to do anything. Guard the import directly so a missing pyspark gives +# an actionable message; anything else this package imports surfaces its own +# error through the real imports below. +try: + import pyspark # noqa: F401 +except ModuleNotFoundError as e: + raise ModuleNotFoundError( + "overture-schema-pyspark requires PySpark, which isn't installed. " + "Install it with `pip install overture-schema-pyspark[spark]`, or run " + "in an environment that already provides PySpark (e.g. a Spark cluster)." + ) from e + from .check import Check, CheckShape from .schema_check import SchemaMismatch, compare_schemas from .validate import ( diff --git a/uv.lock b/uv.lock index 5e0bcfd1a..bead56ba9 100644 --- a/uv.lock +++ b/uv.lock @@ -9,7 +9,7 @@ resolution-markers = [ ] [options] -exclude-newer = "0001-01-01T00:00:00Z" # This has no effect and is included for backwards compatibility when using relative exclude-newer values. +exclude-newer = "2026-08-06T18:30:08.560594Z" exclude-newer-span = "P1W" [manifest] @@ -363,7 +363,7 @@ name = "exceptiongroup" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } wheels = [ @@ -1042,6 +1042,10 @@ source = { editable = "packages/overture-schema-pyspark" } dependencies = [ { name = "click" }, { name = "overture-schema-system" }, +] + +[package.optional-dependencies] +spark = [ { name = "pyspark" }, ] @@ -1049,8 +1053,9 @@ dependencies = [ requires-dist = [ { name = "click", specifier = ">=8.0" }, { name = "overture-schema-system", editable = "packages/overture-schema-system" }, - { name = "pyspark", specifier = ">=3.4" }, + { name = "pyspark", marker = "extra == 'spark'", specifier = ">=3.4" }, ] +provides-extras = ["spark"] [[package]] name = "overture-schema-system"