diff --git a/eng/pipelines/pr-validation-pipeline.yml b/eng/pipelines/pr-validation-pipeline.yml index 99507c8e6..3882f8ec7 100644 --- a/eng/pipelines/pr-validation-pipeline.yml +++ b/eng/pipelines/pr-validation-pipeline.yml @@ -66,6 +66,9 @@ jobs: LocalDB_Python314: sqlVersion: 'LocalDB' pythonVersion: '3.14' + LocalDB_Python315Preview: + sqlVersion: 'LocalDB' + pythonVersion: '3.15.0-rc.3' steps: - checkout: self @@ -74,6 +77,7 @@ jobs: inputs: versionSpec: '$(pythonVersion)' addToPath: true + allowUnstable: true githubToken: $(GITHUB_TOKEN) displayName: 'Use Python $(pythonVersion)' @@ -582,6 +586,10 @@ jobs: sqlServerImage: 'mcr.microsoft.com/mssql/server:2025-latest' sqlVersion: 'SQL2025' pythonVersion: '3.14' + SQL2025_Python315Preview: + sqlServerImage: 'mcr.microsoft.com/mssql/server:2025-latest' + sqlVersion: 'SQL2025' + pythonVersion: '3.15.0-rc.3' steps: - checkout: self @@ -591,6 +599,7 @@ jobs: inputs: versionSpec: '$(pythonVersion)' addToPath: true + allowUnstable: true displayName: 'Use Python $(pythonVersion) on macOS' - task: Cache@2 @@ -768,6 +777,11 @@ jobs: distroName: 'Debian-SQL2025' sqlServerImage: 'mcr.microsoft.com/mssql/server:2025-latest' useAzureSQL: 'false' + Debian_Python315Preview: + dockerImage: 'python:3.15.0rc2-bookworm' + distroName: 'Debian-Python315Preview' + sqlServerImage: 'mcr.microsoft.com/mssql/server:2025-latest' + useAzureSQL: 'false' steps: - checkout: self @@ -797,7 +811,16 @@ jobs: - script: | # Install dependencies in the container - if [ "$(distroName)" = "Ubuntu" ]; then + if [ "$(distroName)" = "Debian-Python315Preview" ]; then + docker exec test-container-$(distroName) bash -c " + export DEBIAN_FRONTEND=noninteractive + export TZ=UTC + ln -snf /usr/share/zoneinfo/\$TZ /etc/localtime && echo \$TZ > /etc/timezone + apt-get update && + apt-get install -y cmake curl wget gnupg build-essential + python3 --version + " + elif [ "$(distroName)" = "Ubuntu" ]; then docker exec test-container-$(distroName) bash -c " export DEBIAN_FRONTEND=noninteractive export TZ=UTC @@ -866,6 +889,7 @@ jobs: displayName: 'Install Python dependencies in $(distroName) container' - script: | + set -e # Build pybind bindings in the container PROFILER_BUILD=0 if [ "$(Build.Reason)" = "PullRequest" ] && @@ -1078,6 +1102,10 @@ jobs: dockerImage: 'python:3.11-bookworm' distroName: 'Debian' archName: 'arm64' + Debian_Python315Preview_ARM64: + dockerImage: 'python:3.15.0rc2-bookworm' + distroName: 'Debian-Python315Preview' + archName: 'arm64' steps: - script: | diff --git a/mssql_python/cursor.py b/mssql_python/cursor.py index 0825ea1b5..76dd2c262 100644 --- a/mssql_python/cursor.py +++ b/mssql_python/cursor.py @@ -2747,20 +2747,17 @@ def executemany( # pylint: disable=too-many-locals,too-many-branches,too-many-s f"{row_index}, column {i} (value type: {type(val).__name__})" ) # Split str(val) from the decimal parse so we only chain a - # cause we know is value-free. decimal.DecimalException - # messages (e.g. ConversionSyntax) never echo the input, so - # they are safe to preserve for debugging. str(val) itself - # or any other error could carry the value in its message - # and surface through __cause__ / formatted tracebacks, so - # those are re-raised with the chain suppressed (from None). + # cause. Python 3.15's pure-Python decimal implementation + # can retain the rejected input in traceback state, so every + # conversion failure suppresses chaining. try: val_text = str(val) except Exception: # pylint: disable=broad-exception-caught raise ValueError(err_msg) from None try: processed_row[i] = format(decimal.Decimal(val_text), "f") - except decimal.DecimalException as e: - raise ValueError(err_msg) from e + except decimal.DecimalException: + raise ValueError(err_msg) from None except Exception: # pylint: disable=broad-exception-caught raise ValueError(err_msg) from None processed_parameters.append(processed_row) diff --git a/mssql_python/pybind/CMakeLists.txt b/mssql_python/pybind/CMakeLists.txt index 77d599bd5..bb94e749d 100644 --- a/mssql_python/pybind/CMakeLists.txt +++ b/mssql_python/pybind/CMakeLists.txt @@ -255,6 +255,13 @@ endif() message(STATUS "Final Python library directory: ${PYTHON_LIB_DIR}") +# Single-config POSIX generators ignore `cmake --build --config Release`. +# Make the optimized release mode explicit instead of relying on simdutf to +# populate this project-wide cache variable as a FetchContent side effect. +if(UNIX AND NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) + set(CMAKE_BUILD_TYPE Release CACHE STRING "Build type" FORCE) +endif() + include(FetchContent) message(STATUS "Downloading simdutf v8.2.0 source archive with FetchContent") set(simdutf_fetchcontent_args @@ -387,6 +394,47 @@ if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" OR CMAKE_CXX_COMPILER_ID STREQUAL "Clang endif() endif() +# Harden the Python extension against exploitation of a separate memory-safety +# defect. Mach-O receives stack protection; ELF-specific flags stay Linux-only. +if(UNIX) + # Python 3.15 defines newer POSIX feature levels than glibc's C++ headers. + # Force Python.h to be processed first in every translation unit. + target_compile_options(ddbc_bindings PRIVATE + -include Python.h + -fstack-protector-strong + ) +endif() + +if(UNIX AND NOT APPLE) + include(CheckCXXSourceCompiles) + set(DDBC_REQUIRED_FLAGS_SAVED "${CMAKE_REQUIRED_FLAGS}") + set(CMAKE_REQUIRED_FLAGS + "${CMAKE_REQUIRED_FLAGS} -O2 -Werror -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=3" + ) + check_cxx_source_compiles(" + #include + #if !defined(__USE_FORTIFY_LEVEL) || __USE_FORTIFY_LEVEL < 3 + #error _FORTIFY_SOURCE=3 is unavailable + #endif + int main() { return 0; } + " DDBC_SUPPORTS_FORTIFY_SOURCE_3) + set(CMAKE_REQUIRED_FLAGS "${DDBC_REQUIRED_FLAGS_SAVED}") + if(DDBC_SUPPORTS_FORTIFY_SOURCE_3) + set(DDBC_FORTIFY_LEVEL 3) + else() + set(DDBC_FORTIFY_LEVEL 2) + endif() + target_compile_options(ddbc_bindings PRIVATE + $<$>:-U_FORTIFY_SOURCE> + $<$>:-D_FORTIFY_SOURCE=${DDBC_FORTIFY_LEVEL}> + ) + target_link_options(ddbc_bindings PRIVATE + -Wl,-z,relro + -Wl,-z,now + -Wl,-z,noexecstack + ) +endif() + # Add macOS-specific string conversion fix if(APPLE) message(STATUS "Enabling macOS string conversion fix") diff --git a/mssql_python/pybind/ddbc_bindings.h b/mssql_python/pybind/ddbc_bindings.h index 32d9f8067..748ecc5ca 100644 --- a/mssql_python/pybind/ddbc_bindings.h +++ b/mssql_python/pybind/ddbc_bindings.h @@ -3,16 +3,19 @@ #pragma once -// pybind11.h must be the first include -#include -#include -#include +// Python.h must precede standard-library headers so its feature-test macros +// are established before libc headers consume them. +#include #include #include #include #include #include // Add this line for datetime support #include + +#include +#include +#include #include #include diff --git a/requirements.txt b/requirements.txt index daffd1a1c..1f4a90d4a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,9 @@ zstandard coverage unittest-xml-reporting psutil -pyarrow +--extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple +pyarrow; python_version < "3.15" +pyarrow==26.0.0.dev323; python_version >= "3.15" polars # Runtime dependencies needed for tests diff --git a/tests/test_008_auth.py b/tests/test_008_auth.py index 6fd2d6574..2b3ef50da 100644 --- a/tests/test_008_auth.py +++ b/tests/test_008_auth.py @@ -1565,7 +1565,13 @@ def test_multiple_connections_share_same_token_provider(self, mock_ddbc_conn): @patch("mssql_python.connection.ddbc_bindings.Connection") def test_concurrent_connections_with_same_token_provider(self, mock_ddbc_conn): """Concurrent connect() calls with one token provider should succeed.""" - mock_ddbc_conn.return_value = MagicMock() + + def create_native_connection(*_args, **_kwargs): + native_connection = MagicMock() + native_connection.get_autocommit.return_value = True + return native_connection + + mock_ddbc_conn.side_effect = create_native_connection mock_cred = MagicMock() mock_cred.get_token.return_value = MagicMock(token=SAMPLE_TOKEN) from mssql_python import connect diff --git a/tests/test_009_pooling.py b/tests/test_009_pooling.py index c8cafed8f..28b7465bc 100644 --- a/tests/test_009_pooling.py +++ b/tests/test_009_pooling.py @@ -1351,7 +1351,6 @@ def collect_children(): collect_barrier.wait() assert free_entered.wait(10), "Cursor finalizer did not enter free" assert not errors, errors - assert cursor_ref() is None, "GC did not clear the cursor weakref" assert not connection._cursors, "Connection.close would still see the cursor" if not explicit_close: @@ -1369,6 +1368,15 @@ def collect_children(): native = None collect_barrier.wait() + remaining_cursor = cursor_ref() + if remaining_cursor is not None: + # Python 3.15 may finalize a cyclic object before reclaiming + # its self-cycle. Break only the synthetic test cycle, then + # verify that no production reference keeps the cursor alive. + remaining_cursor.cycle = None + del remaining_cursor + gc.collect() + assert cursor_ref() is None, "GC did not clear the cursor weakref" assert finalizer_statement.calls == 1 assert finalizer_statement.completed, errors assert not errors, errors diff --git a/tests/test_038_mssql_odbc_daily_validation.py b/tests/test_038_mssql_odbc_daily_validation.py index 539e65ea6..2bcc7f216 100644 --- a/tests/test_038_mssql_odbc_daily_validation.py +++ b/tests/test_038_mssql_odbc_daily_validation.py @@ -12,6 +12,7 @@ ROOT = Path(__file__).parents[1] RUNNER = ROOT / "eng" / "scripts" / "run-mssql-odbc-tests.sh" PIPELINE = ROOT / "eng" / "pipelines" / "mssql-odbc-daily-validation-pipeline.yml" +PR_PIPELINE = ROOT / "eng" / "pipelines" / "pr-validation-pipeline.yml" PREFLIGHT = ROOT / "eng" / "scripts" / "verify_mssql_odbc_provider.py" @@ -150,6 +151,23 @@ def test_stable_rs_transport_is_pinned(self): self.assertEqual(version.strip(), "0.3.0") + def test_python_315_validation_adds_preview_matrix_legs(self): + pipeline = PR_PIPELINE.read_text(encoding="utf-8") + active_python_versions = { + line.split(":", 1)[1].strip(" '\"") + for line in pipeline.splitlines() + if line.lstrip().startswith("pythonVersion:") + } + + self.assertEqual(active_python_versions, {"3.13", "3.14", "3.15.0-rc.3"}) + self.assertIn("python:3.15.0rc2-bookworm", pipeline) + + def test_python_315_validation_installs_pyarrow_nightly(self): + requirements = (ROOT / "requirements.txt").read_text(encoding="utf-8") + + self.assertIn("scientific-python-nightly-wheels", requirements) + self.assertIn('pyarrow==26.0.0.dev323; python_version >= "3.15"', requirements) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_040_native_binary_hardening.py b/tests/test_040_native_binary_hardening.py new file mode 100644 index 000000000..694d86e42 --- /dev/null +++ b/tests/test_040_native_binary_hardening.py @@ -0,0 +1,190 @@ +"""Regression guards for POSIX native-extension hardening.""" + +import struct +import sys +from pathlib import Path + +import pytest + +_ROOT = Path(__file__).resolve().parents[1] +_CMAKE = _ROOT / "mssql_python" / "pybind" / "CMakeLists.txt" +_PT_DYNAMIC = 2 +_PT_GNU_STACK = 0x6474E551 +_PT_GNU_RELRO = 0x6474E552 +_PF_X = 0x1 +_DT_NULL = 0 +_DT_BIND_NOW = 24 +_DT_FLAGS = 30 +_DF_BIND_NOW = 0x8 +_DT_FLAGS_1 = 0x6FFFFFFB +_DF_1_NOW = 0x1 + + +def _make_elf64(*, relro=True, bind_now=True, executable_stack=False): + header_size = 64 + program_header_size = 56 + program_count = 3 + dynamic_offset = header_size + program_header_size * program_count + dynamic = struct.pack("" + program_offset = struct.unpack_from(endian + "Q", data, 0x20)[0] + entry_size = struct.unpack_from(endian + "H", data, 0x36)[0] + entry_count = struct.unpack_from(endian + "H", data, 0x38)[0] + + if ( + not program_offset + or not entry_count + or entry_size < 56 + or program_offset + entry_count * entry_size > len(data) + ): + raise ValueError("invalid ELF program headers") + + has_relro = False + stack_executable = None + dynamic_segment = None + for index in range(entry_count): + offset = program_offset + index * entry_size + program_type = struct.unpack_from(endian + "I", data, offset)[0] + flags = struct.unpack_from(endian + "I", data, offset + 4)[0] + file_offset = struct.unpack_from(endian + "Q", data, offset + 8)[0] + file_size = struct.unpack_from(endian + "Q", data, offset + 32)[0] + if file_offset + file_size > len(data): + raise ValueError("ELF segment extends beyond the file") + if program_type == _PT_GNU_RELRO: + has_relro = True + elif program_type == _PT_GNU_STACK: + stack_executable = bool(flags & _PF_X) + elif program_type == _PT_DYNAMIC: + dynamic_segment = file_offset, file_size + + if dynamic_segment is None: + raise ValueError("ELF has no PT_DYNAMIC segment") + + dynamic_offset, dynamic_size = dynamic_segment + dynamic_entry_size = 16 + if dynamic_size % dynamic_entry_size: + raise ValueError("ELF dynamic segment has a partial entry") + + bind_now = False + terminated = False + for offset in range(dynamic_offset, dynamic_offset + dynamic_size, dynamic_entry_size): + tag = struct.unpack_from(endian + "q", data, offset)[0] + value = struct.unpack_from(endian + "Q", data, offset + 8)[0] + if tag == _DT_NULL: + terminated = True + break + bind_now = bind_now or tag == _DT_BIND_NOW + bind_now = bind_now or tag == _DT_FLAGS and bool(value & _DF_BIND_NOW) + bind_now = bind_now or tag == _DT_FLAGS_1 and bool(value & _DF_1_NOW) + + if not terminated: + raise ValueError("ELF dynamic segment lacks DT_NULL") + return has_relro, bind_now, stack_executable + + +def test_elf_hardening_parser_reads_program_and_dynamic_flags(): + assert _elf_hardening(_make_elf64()) == (True, True, False) + assert _elf_hardening(_make_elf64(relro=False)) == (False, True, False) + assert _elf_hardening(_make_elf64(bind_now=False)) == (True, False, False) + assert _elf_hardening(_make_elf64(executable_stack=True)) == (True, True, True) + + +@pytest.mark.skipif( + not _CMAKE.is_file(), + reason="requires a source checkout; isolated wheel tests omit the source tree", +) +def test_posix_hardening_flags_are_explicit(): + cmake = _CMAKE.read_text(encoding="utf-8") + + assert "set(CMAKE_BUILD_TYPE Release" in cmake + assert "-fstack-protector-strong" in cmake + assert "-U_FORTIFY_SOURCE" in cmake + assert "DDBC_SUPPORTS_FORTIFY_SOURCE_3" in cmake + assert "-D_FORTIFY_SOURCE=${DDBC_FORTIFY_LEVEL}" in cmake + assert "-include Python.h" in cmake + assert "$>" in cmake + assert "if(UNIX AND NOT APPLE)" in cmake + for flag in ("-Wl,-z,relro", "-Wl,-z,now", "-Wl,-z,noexecstack"): + assert flag in cmake + + +def test_python_headers_precede_standard_library_headers(): + header = (_ROOT / "mssql_python" / "pybind" / "ddbc_bindings.h").read_text(encoding="utf-8") + + assert header.index("#include ") < header.index("#include ") + + +@pytest.mark.skipif(sys.platform != "linux", reason="ELF hardening applies to Linux") +def test_linux_extension_has_linker_hardening(): + from mssql_python import ddbc_bindings + + extension = Path(ddbc_bindings.module_path) + has_relro, bind_now, stack_executable = _elf_hardening(extension.read_bytes()) + assert has_relro, "native extension is missing PT_GNU_RELRO" + assert bind_now, "native extension is missing immediate binding (BIND_NOW)" + assert stack_executable is not None, "native extension has no PT_GNU_STACK declaration" + assert stack_executable is False, "native extension requests an executable stack"