From bfa1422c961fbba4f2c17a359cebe5f037dafd40 Mon Sep 17 00:00:00 2001 From: Roel <75250264+RoelBollens-TomTom@users.noreply.github.com> Date: Tue, 11 Aug 2026 21:19:49 +0200 Subject: [PATCH] Make the pyspark test suite hermetic --- .github/workflows/check-python-code.yaml | 12 +-- .../overture-schema-pyspark/tests/conftest.py | 86 ++++++++++++++++++- pyproject.toml | 8 ++ uv.lock | 17 +++- 4 files changed, 108 insertions(+), 15 deletions(-) diff --git a/.github/workflows/check-python-code.yaml b/.github/workflows/check-python-code.yaml index f2de60dc6..a0924647a 100644 --- a/.github/workflows/check-python-code.yaml +++ b/.github/workflows/check-python-code.yaml @@ -61,15 +61,9 @@ jobs: with: python-version: ${{ matrix.python }} - # PySpark 3.4 (the declared minimum) does not support Java 21, which is - # the default JDK on ubuntu-latest runners. Pin to Java 17 for the - # lowest-direct cell so the resolved pyspark==3.4.0 can actually start. - - name: Set up JDK 17 - if: matrix.resolution == 'lowest-direct' - uses: actions/setup-java@03ad4de0992f5dab5e18fcb136590ce7c4a0ac95 # v5.6.0 - with: - distribution: temurin - java-version: '17' + # No setup-java: the test suite runs on the jdk4py JDK from the dev + # group. Its floor (JDK 17) is what lowest-direct resolves, matching + # pyspark 3.4, which does not support the runners' default Java 21. # UV_RESOLUTION=lowest-direct makes `uv sync` re-resolve every direct # dependency to the lowest version permitted by pyproject.toml. This diff --git a/packages/overture-schema-pyspark/tests/conftest.py b/packages/overture-schema-pyspark/tests/conftest.py index ccacb1aac..df3f4f9de 100644 --- a/packages/overture-schema-pyspark/tests/conftest.py +++ b/packages/overture-schema-pyspark/tests/conftest.py @@ -1,18 +1,96 @@ """Shared pytest fixtures for overture-schema-pyspark tests.""" +import atexit import os +import shutil import socket import sys +import tempfile from collections.abc import Callable +from pathlib import Path from typing import Any +import pyspark import pytest from pyspark.sql import SparkSession -# Ensure PySpark workers use the same Python as the driver to avoid -# version mismatch errors when a different system Python is on PATH. -os.environ.setdefault("PYSPARK_PYTHON", sys.executable) -os.environ.setdefault("PYSPARK_DRIVER_PYTHON", sys.executable) +# Tests must be hermetic against ambient Spark/JVM configuration: developers +# commonly have a system-wide Spark install or JVM flags exported for other +# projects, and any of these leaking into the test JVM changes -- or breaks -- +# every session. Networking overrides such as SPARK_LOCAL_IP are deliberately +# left alone: those are set to make local sessions *work* (e.g. macOS/VPN +# hostname resolution). +for _var in ( + "PYSPARK_SUBMIT_ARGS", # injects launcher/JVM flags, jars, even --master + "SPARK_CONF_DIR", # points at another install's spark-defaults.conf + "HADOOP_CONF_DIR", # redirects filesystem/cluster defaults + "YARN_CONF_DIR", + "JAVA_TOOL_OPTIONS", # picked up unconditionally by every JVM + "_JAVA_OPTIONS", + "JDK_JAVA_OPTIONS", +): + os.environ.pop(_var, None) +# Pin worker and driver Python to this interpreter (overriding, not +# setdefault): an exported PYSPARK_PYTHON pointing at a non-venv Python +# fails with pickle/version-mismatch errors inside executors. +os.environ["PYSPARK_PYTHON"] = sys.executable +os.environ["PYSPARK_DRIVER_PYTHON"] = sys.executable +# Pin Spark to the venv's bundled distribution: an ambient SPARK_HOME +# pointing at a system-wide Spark install would launch that JVM under this +# venv's PySpark client, and mismatched versions fail at session setup with +# opaque py4j "'JavaPackage' object is not callable" errors. +os.environ["SPARK_HOME"] = os.path.dirname(pyspark.__file__) + + +def _shimmed_java_home(java_home: Path) -> Path: + """Mirror *java_home*, replacing bin/java with a flag-filtering wrapper. + + jdk4py ships a jlink-stripped runtime without jdk.incubator.vector + (upstream declined to bundle it for size: atoti/jdk4py#95), while + Spark 4.x passes --add-modules=jdk.incubator.vector to every JVM it + launches -- and a JVM refuses to boot when an --add-modules target is + missing. The module only enables vectorized BLAS, which Spark treats + as optional at runtime, so the wrapper drops that one flag and execs + the real java with everything else intact. + """ + shim_home = Path(tempfile.mkdtemp(prefix="overture-pyspark-test-jdk-")) + atexit.register(shutil.rmtree, shim_home, ignore_errors=True) + for entry in java_home.iterdir(): + if entry.name != "bin": + (shim_home / entry.name).symlink_to(entry) + bin_dir = shim_home / "bin" + bin_dir.mkdir() + for entry in (java_home / "bin").iterdir(): + if entry.name != "java": + (bin_dir / entry.name).symlink_to(entry) + real_java = java_home / "bin" / "java" + shim_java = bin_dir / "java" + shim_java.write_text( + "#!/bin/sh\n" + "# Drop --add-modules=jdk.incubator.vector; see conftest.py.\n" + "for arg do\n" + " shift\n" + ' [ "$arg" = "--add-modules=jdk.incubator.vector" ] && continue\n' + ' set -- "$@" "$arg"\n' + "done\n" + f'exec "{real_java}" "$@"\n' + ) + shim_java.chmod(0o755) + return shim_home + + +# Pin the JVM to the venv's bundled JDK (jdk4py, a dev dependency on the +# platforms that have wheels for it). On platforms without a wheel -- or +# when the dev group isn't installed -- fall back to the machine's Java. +# The shim wrapper is a POSIX shell script, so non-POSIX platforms also +# fall back. +try: + import jdk4py +except ImportError: + pass +else: + if os.name == "posix": + os.environ["JAVA_HOME"] = str(_shimmed_java_home(jdk4py.JAVA_HOME)) def pytest_configure(config: pytest.Config) -> None: diff --git a/pyproject.toml b/pyproject.toml index 9f5eea2bc..18a3824b5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,6 +50,14 @@ warn_unused_configs = true [dependency-groups] dev = [ + # Bundled JDK for the pyspark test suite, so tests never depend on the + # machine's Java. <22 keeps every resolution inside Spark's supported + # range: the floor (17.0.9.2 = JDK 17) is what lowest-direct resolves + # for pyspark 3.4, and the lock picks the maintained 21.x line for + # pyspark 4.x. jdk4py ships wheels only and the conftest shim that + # adapts it for Spark is POSIX shell, so gate it to darwin/linux; + # elsewhere conftest falls back to the system Java. + "jdk4py>=17.0.9.2,<22; sys_platform == 'darwin' or sys_platform == 'linux'", "mypy>=1.17.0", "pdoc>=15.0.4", "pydocstyle>=6.3.0", diff --git a/uv.lock b/uv.lock index 4a025e9bc..a3cbbcae5 100644 --- a/uv.lock +++ b/uv.lock @@ -9,7 +9,7 @@ resolution-markers = [ ] [options] -exclude-newer = "0001-01-01T00:00:00Z" # This has no effect and is included for backwards compatibility when using relative exclude-newer values. +exclude-newer = "2026-08-04T18:57:39.255576Z" exclude-newer-span = "P1W" [manifest] @@ -362,7 +362,7 @@ name = "exceptiongroup" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } wheels = [ @@ -387,6 +387,17 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, ] +[[package]] +name = "jdk4py" +version = "21.0.8.2" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9b/91/faab5f5b2bdcaf74c20304c5c3a6a9e304a6c609a1d755e927c07a87f883/jdk4py-21.0.8.2-py3-none-macosx_10_9_x86_64.whl", hash = "sha256:7b2b397f2999e5a71728691477857a5a41215b3c95f6e8b15a4d53bc76203dff", size = 30900008, upload-time = "2026-04-10T20:50:45.913Z" }, + { url = "https://files.pythonhosted.org/packages/e6/19/449f69b8706c2cb5fa46fdc3e75650983e44e4e025d6ade78580b2c8572d/jdk4py-21.0.8.2-py3-none-macosx_11_0_arm64.whl", hash = "sha256:0667b81b3ddc60157f773e0d05d00f4d46530f82bdc59e295b322b8d33f69417", size = 29692099, upload-time = "2026-04-10T20:50:37.405Z" }, + { url = "https://files.pythonhosted.org/packages/9d/0c/24a7d8e4c96c3843b8163ba7a9be939b48fc181fb3263bd4fd9289ed8f41/jdk4py-21.0.8.2-py3-none-manylinux_2_17_aarch64.whl", hash = "sha256:2519dce230b3ed13a93a3efed15c6b8b163aeacd77859ddf59eef3faa69c635d", size = 33195566, upload-time = "2026-04-10T20:50:34.609Z" }, + { url = "https://files.pythonhosted.org/packages/a2/61/f3b5936908ff6de66c61aef69bf1074cb468fb883f5a0bb9e15e67a6b484/jdk4py-21.0.8.2-py3-none-manylinux_2_17_x86_64.whl", hash = "sha256:85addfcb57c7051dad6145b9f816fc519337e9a0c705ef01edc9dc7818ee0356", size = 33794570, upload-time = "2026-04-10T20:50:40.113Z" }, +] + [[package]] name = "jinja2" version = "3.1.6" @@ -1207,6 +1218,7 @@ source = { virtual = "." } [package.dev-dependencies] dev = [ + { name = "jdk4py", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "mypy" }, { name = "pdoc" }, { name = "pydocstyle" }, @@ -1221,6 +1233,7 @@ dev = [ [package.metadata.requires-dev] dev = [ + { name = "jdk4py", marker = "sys_platform == 'darwin' or sys_platform == 'linux'", specifier = ">=17.0.9.2,<22" }, { name = "mypy", specifier = ">=1.17.0" }, { name = "pdoc", specifier = ">=15.0.4" }, { name = "pydocstyle", specifier = ">=6.3.0" },