From b033843a4f99103150ed1a11207d0160ff9c40b0 Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Fri, 16 Oct 2026 12:59:55 -0400 Subject: [PATCH] fix(conda): pin cryptography 48.0.1 on Intel macOS, which has no newer wheel (#1827) - conda/requirements.txt: cryptography gets a line per platform pin. osx-64 takes 48.0.1, the newest release with an Intel macOS wheel (universal2), since 49.0.0 dropped them; every other build takes the installed release. Each conda leg's pip download evaluates the markers on its own runner, and build.sh installs what it fetched. - util/conda_vendor.py: PLATFORM_PINS holds that pin and its reason. The writer emits a line per pin; --check accepts them, and flags a pin that moved or went and a table entry no build can use. - tests/project/test_conda_cryptography_pin_1827.py pins the committed lines per build and covers the writer and --check, offline; the #1814 tests read the new parse shape. - cron-conda.yml: the comments say where the markers are evaluated. tests/project passes and --check passes on the committed file. pip resolves 48.0.1's universal2 wheel for osx-64 and 50.0.2 elsewhere, on each supported Python. Closes #1827 --- .github/workflows/cron-conda.yml | 9 +- conda/requirements.txt | 3 +- .../test_conda_cryptography_pin_1827.py | 168 ++++++++++++++++++ tests/project/test_conda_vendor_1814.py | 22 ++- util/conda_vendor.py | 104 +++++++++-- 5 files changed, 277 insertions(+), 29 deletions(-) create mode 100644 tests/project/test_conda_cryptography_pin_1827.py diff --git a/.github/workflows/cron-conda.yml b/.github/workflows/cron-conda.yml index 7e0052eba..531c3919e 100644 --- a/.github/workflows/cron-conda.yml +++ b/.github/workflows/cron-conda.yml @@ -159,9 +159,9 @@ jobs: # `.[all]`, which is what `util/conda_vendor.py` below vendors (#1814): # it derives `conda/requirements.txt` from `dependencies` and the `all` # extra, walks what they require through the installed metadata, and - # pins each package at the version installed here. So this install is - # the step that resolves the newest releases, and the script itself - # reads nothing from the network. + # pins each package at the version installed here, but for its + # `PLATFORM_PINS` (#1827). So this install is the step that resolves the + # newest releases, and the script itself reads nothing from the network. - name: Install and Setup run: | set -x @@ -459,6 +459,9 @@ jobs: python -m pip install -U pip setuptools wheel python -m pip install -e .[all] + # #1827: pip evaluates conda/requirements.txt's platform markers here, + # on this leg's own runner, so Intel macOS fetches its own cryptography + # pin; the recipe's build.sh installs whatever this downloaded. - name: Setup Conda Distribution if: ${{ matrix.os != 'windows-latest' }} run: | diff --git a/conda/requirements.txt b/conda/requirements.txt index 54c58ae18..69ac7517f 100644 --- a/conda/requirements.txt +++ b/conda/requirements.txt @@ -1,7 +1,8 @@ aenum==3.1.17 cffi==2.1.1 chardet==7.6.0 -cryptography==50.0.2 +cryptography==48.0.1; sys_platform == "darwin" and platform_machine == "x86_64" +cryptography==50.0.2; sys_platform != "darwin" or platform_machine != "x86_64" dictdumper==0.8.4.post6 emoji==2.16.0 pycparser==3.11 diff --git a/tests/project/test_conda_cryptography_pin_1827.py b/tests/project/test_conda_cryptography_pin_1827.py new file mode 100644 index 000000000..1f40861ea --- /dev/null +++ b/tests/project/test_conda_cryptography_pin_1827.py @@ -0,0 +1,168 @@ +# -*- coding: utf-8 -*- +"""cryptography is pinned apart on Intel macOS, which has no wheel of its newest releases (#1827). + +* :file:`conda/requirements.txt` carries a cryptography line per platform pin, + and on each conda build exactly one of them applies: 48.0.1 on ``osx-64``, + the installed release elsewhere. +* :file:`util/conda_vendor.py` writes those lines from ``PLATFORM_PINS``, and + ``--check`` passes on them and fails when the table entry or the file's pin + goes, when the pin moves, and when the table names a package or platform no + build has. All offline, on stand-in metadata. + +""" + +from __future__ import annotations + +import contextlib +import importlib.util +import io +import pathlib +import re +import tempfile +import unittest +from typing import TYPE_CHECKING +from unittest import mock + +try: + import tomllib +except ImportError: # pragma: no cover + tomllib = None # type: ignore[assignment] + +if TYPE_CHECKING: + from typing import Any + +ROOT = pathlib.Path(__file__).resolve().parents[2] +REQUIREMENTS = ROOT / 'conda' / 'requirements.txt' +INTEL = 'cryptography==48.0.1; sys_platform == "darwin" and platform_machine == "x86_64"' +#: The other line, whose pin the weekly Conda Update moves. +REST = re.compile(r'cryptography==(\S+); sys_platform != "darwin" or platform_machine != "x86_64"') + +_MODULE = [] # type: list[Any] + + +def vendor() -> 'Any': + """:file:`util/conda_vendor.py`, loaded once; skipped without :mod:`tomllib`, which it imports.""" + if tomllib is None: # pragma: no cover + raise unittest.SkipTest('tomllib, which util/conda_vendor.py imports, needs Python 3.11+') + if not _MODULE: + spec = importlib.util.spec_from_file_location('conda_vendor', ROOT / 'util' / 'conda_vendor.py') + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) # type: ignore[union-attr] + _MODULE.append(module) + return _MODULE[0] + + +def committed() -> 'list[str]': + """The committed file's cryptography lines.""" + return [line for line in REQUIREMENTS.read_text(encoding='utf-8').splitlines() + if line.startswith('cryptography')] + + +def cryptography(module: 'Any', envs: 'list[dict[str, str]]') -> 'dict[str, Any]': + """What the conda builds vendor for ``pyproject.toml``'s cryptography line, on stand-in metadata.""" + return module.derive([req for req in module.direct() if req.name == 'cryptography'], envs, lambda name: []) + + +class CommittedLinesTests(unittest.TestCase): + """The committed cryptography lines, evaluated for each conda build.""" + + def test_each_build_reads_one_cryptography_line(self) -> None: + from packaging.requirements import Requirement + module = vendor() + lines = committed() + self.assertIn(INTEL, lines) + self.assertEqual(len([line for line in lines if REST.fullmatch(line)]), 1, lines) + self.assertEqual(module.PLATFORM_PINS, {'cryptography': {'osx-64': '48.0.1'}}) + for env in module.environments(module.conda_pythons()): + subdir = next(subdir for subdir, (sys_platform, _, machine) in module.PLATFORMS.items() + if (sys_platform, machine) == (env['sys_platform'], env['platform_machine'])) + with self.subTest(subdir=subdir, python=env['python_version']): + live = [line for line in lines if Requirement(line).marker.evaluate(env)] + self.assertEqual(len(live), 1, live) + self.assertEqual(live[0] == INTEL, subdir == 'osx-64', live) + + +class WriterTests(unittest.TestCase): + """The writer emits a line per platform pin.""" + + def test_the_writer_emits_the_platform_lines(self) -> None: + module = vendor() + envs = module.environments(module.conda_pythons()) + self.assertEqual(module.lines(cryptography(module, envs), envs, lambda name: '50.0.2'), + [INTEL, 'cryptography==50.0.2; sys_platform != "darwin" or platform_machine != "x86_64"']) + + def test_a_python_marker_is_kept_beside_the_platform_one(self) -> None: + from packaging.requirements import Requirement + module = vendor() + envs = module.environments(['3.11', '3.12']) + found = module.derive([Requirement('cryptography; python_version >= "3.12"')], envs, lambda name: []) + lines = module.lines(found, envs, lambda name: '50.0.2') + self.assertEqual(lines, [ + 'cryptography==48.0.1; (python_version == "3.12") and ' + '(sys_platform == "darwin" and platform_machine == "x86_64")', + 'cryptography==50.0.2; (python_version == "3.12") and ' + '(sys_platform != "darwin" or platform_machine != "x86_64")', + ]) + for env in envs: + with self.subTest(env=(env['python_version'], env['sys_platform'], env['platform_machine'])): + live = [line for line in lines if Requirement(line).marker.evaluate(env)] + self.assertEqual(len(live), env['python_version'] == '3.12') + + +class CheckTests(unittest.TestCase): + """``--check`` against the committed cryptography lines.""" + + def problems(self, text: 'str') -> 'list[str]': + module = vendor() + envs = module.environments(module.conda_pythons()) + return module.compare(text, cryptography(module, envs), envs) + + def test_the_committed_lines_pass(self) -> None: + lines = committed() + self.assertIn(INTEL, lines) + self.assertEqual(self.problems('\n'.join(lines)), []) + + def test_without_the_override_the_platform_lines_fail(self) -> None: + with mock.patch.object(vendor(), 'PLATFORM_PINS', {}): + problems = self.problems('\n'.join(committed())) + self.assertEqual(len(problems), 1, problems) + self.assertTrue(problems[0].startswith('marker: cryptography has ['), problems) + + def test_a_dropped_or_moved_platform_pin_fails(self) -> None: + rest = [line for line in committed() if line != INTEL] + dropped = self.problems('\n'.join(rest)) + self.assertEqual([problem.split(':')[0] for problem in dropped], ['marker'], dropped) + unmarked = self.problems('\n'.join(rest + ['cryptography==48.0.1'])) + self.assertEqual([problem.split(':')[0] for problem in unmarked], ['marker'], unmarked) + moved = self.problems('\n'.join(rest + [INTEL.replace('48.0.1', '49.0.0')])) + self.assertEqual(moved, ["pin: cryptography pins '49.0.0' where sys_platform == \"darwin\" and " + "platform_machine == \"x86_64\", and PLATFORM_PINS '48.0.1'"]) + + def test_an_override_no_build_can_use_fails(self) -> None: + pins = {'cryptography': {'osx-64': '48.0.1', 'win-64': '1.0'}, 'gone': {'osx-64': '1.0'}} + with mock.patch.object(vendor(), 'PLATFORM_PINS', pins): + problems = self.problems('\n'.join(committed())) + self.assertEqual(problems, ['override: cryptography is pinned for win-64, which no conda leg builds', + 'override: gone is pinned for osx-64, and no conda build vendors it']) + + def test_the_writer_refuses_an_override_no_build_can_use(self) -> None: + module = vendor() + with tempfile.TemporaryDirectory() as temp: + target = pathlib.Path(temp) / 'requirements.txt' + saved = module.FILE, module.ROOT, module._requires, module._version + module.FILE, module.ROOT = target, pathlib.Path(temp) + module._requires = lambda name: [] + module._version = lambda name: '1.0' + try: + with mock.patch.object(module, 'PLATFORM_PINS', {'gone': {'osx-64': '1.0'}}), \ + contextlib.redirect_stderr(io.StringIO()) as errors: + self.assertEqual(module.main([]), 1) + self.assertFalse(target.exists()) + self.assertIn('override: gone is pinned for osx-64', errors.getvalue()) + finally: + module.FILE, module.ROOT, module._requires, module._version = saved + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/project/test_conda_vendor_1814.py b/tests/project/test_conda_vendor_1814.py index a56664d72..ed99294ae 100644 --- a/tests/project/test_conda_vendor_1814.py +++ b/tests/project/test_conda_vendor_1814.py @@ -33,6 +33,7 @@ import textwrap import unittest import zipfile +from unittest import mock from typing import TYPE_CHECKING try: @@ -211,13 +212,15 @@ def test_compare_passes_on_written_lines_and_names_each_stale_one(self) -> None: module = vendor() envs = module.environments(self.PYTHONS) found = self.found('core', 'crypt', 'backport; python_version < "3.11"') - written = requirement_lines(*module.lines(found, envs, VERSIONS.__getitem__)) - self.assertEqual(module.compare(written, found, envs), []) - repinned = written.replace('core==1.0', 'core==9.9') - self.assertEqual(module.compare(repinned, found, envs), []) - stale = requirement_lines('backport==1.0; python_version < "3.11"', 'core==1.0', - 'crypt==1.0', 'ffi==1.0', 'helper==1.0', 'lxml==6.1.3') - problems = module.compare(stale, found, envs) + # The stand-in packages have no platform pins (#1827's are its own tests'). + with mock.patch.object(module, 'PLATFORM_PINS', {}): + written = requirement_lines(*module.lines(found, envs, VERSIONS.__getitem__)) + self.assertEqual(module.compare(written, found, envs), []) + repinned = written.replace('core==1.0', 'core==9.9') + self.assertEqual(module.compare(repinned, found, envs), []) + stale = requirement_lines('backport==1.0; python_version < "3.11"', 'core==1.0', + 'crypt==1.0', 'ffi==1.0', 'helper==1.0', 'lxml==6.1.3') + problems = module.compare(stale, found, envs) self.assertEqual([problem.split(':')[0] + ':' + problem.split()[1] for problem in problems], ['dead:backport', 'stale:lxml', 'missing:parser']) @@ -237,8 +240,9 @@ def test_every_live_direct_requirement_is_listed_and_no_dead_one(self) -> None: self.assertNotIn(name, have) continue self.assertIn(name, have) - marker = module.render_marker(entry['builds'], envs) - self.assertEqual(have[name], None if marker is None else str(Marker(marker))) + # #1827: a line per platform pin, each with its own marker. + self.assertEqual(set(have[name]), {None if marker is None else str(Marker(marker)) + for marker, _ in module.pins(name, entry, envs)}) def test_check_passes_on_its_own_output_and_fails_on_a_stale_file(self) -> None: module = vendor() diff --git a/util/conda_vendor.py b/util/conda_vendor.py index 5330b19f6..d1c0087ad 100644 --- a/util/conda_vendor.py +++ b/util/conda_vendor.py @@ -29,12 +29,19 @@ Each line pins the version installed here, as ``conda-update`` in :file:`.github/workflows/cron-conda.yml` installs ``.[all]`` before running this script. Resolving the newest releases is that install step's job, and needs the -network; this script reads only what is installed, so it works offline. +network; this script reads only what is installed, so it works offline. The one +exception is :data:`PLATFORM_PINS`: a package whose newest release has no wheel +for some platform keeps an older pin for it, on a line of its own whose marker +names that platform, beside the installed pin for the rest (#1827). The conda +legs build natively, so each leg's ``pip download`` evaluates those markers on +its own runner. Run without arguments it rewrites :file:`conda/requirements.txt`. With ``--check`` it writes nothing, and exits 1 when the file names a different set of -packages or markers than it derives -- the pins aside, since they move with every -release and the weekly run refreshes them. With ``--smoke PACKAGE`` it unpacks a +packages or markers than it derives, a platform pin other than +:data:`PLATFORM_PINS`, or when that table names a package no build vendors -- +the other pins aside, since they move with every release and the weekly run +refreshes them. With ``--smoke PACKAGE`` it unpacks a built ``.tar.bz2`` conda package and checks that every distribution vendored in it shipped every file its ``RECORD`` lists, and that each of its modules imports from :file:`pcapkit/_extern` with nothing else on the path. It also checks that @@ -85,6 +92,11 @@ 'osx-64': ('darwin', 'Darwin', 'x86_64'), 'osx-arm64': ('darwin', 'Darwin', 'arm64'), } +#: The packages pinned apart on some of :data:`PLATFORMS`, as {package: {conda +#: subdir: version}} (#1827). cryptography stopped shipping Intel macOS wheels at +#: 49.0.0, so osx-64 vendors 48.0.1, the newest release with one (universal2); +#: otherwise its build would compile cryptography from source. +PLATFORM_PINS = {'cryptography': {'osx-64': '48.0.1'}} #: The object format and the machine of every native binary a package for each #: of :data:`PLATFORMS` may carry (#1822). A universal Mach-O passes when one of #: its slices is the machine. @@ -244,42 +256,92 @@ def render_marker(builds: 'frozenset[int]', envs: 'list[dict[str, str]]') -> 'Op return ' or '.join(f'python_version == "{python}"' for python in pythons) +def platform_marker(subdirs: 'Iterable[str]', negate: 'bool' = False) -> 'str': + """The marker selecting the builds on ``subdirs`` of :data:`PLATFORMS`, or with ``negate`` all others.""" + terms = [] + for subdir in sorted(subdirs): + sys_platform, _, machine = PLATFORMS[subdir] + terms.append(f'sys_platform != "{sys_platform}" or platform_machine != "{machine}"' if negate else + f'sys_platform == "{sys_platform}" and platform_machine == "{machine}"') + if len(terms) == 1: + return terms[0] + return (' and ' if negate else ' or ').join(f'({term})' for term in terms) + + +def pins(name: 'str', entry: 'dict[str, object]', + envs: 'list[dict[str, str]]') -> 'list[tuple[Optional[str], Optional[str]]]': + """Each line ``name`` gets: its marker, and the version :data:`PLATFORM_PINS` fixes or :data:`None`.""" + python = render_marker(entry['builds'], envs) # type: ignore[arg-type] + # One for a platform no leg builds is unpinnable()'s to report. + fixed = {subdir: version for subdir, version in PLATFORM_PINS.get(name, {}).items() if subdir in PLATFORMS} + if not fixed: + return [(python, None)] + groups = {} # type: dict[str, set[str]] + for subdir, version in fixed.items(): + groups.setdefault(version, set()).add(subdir) + found = [(platform_marker(subdirs), version) + for version, subdirs in sorted(groups.items())] # type: list[tuple[str, Optional[str]]] + found.append((platform_marker(fixed, negate=True), None)) + return [(marker if python is None else f'({python}) and ({marker})', version) + for marker, version in found] + + def lines(found: 'dict[str, dict[str, object]]', envs: 'list[dict[str, str]]', version: 'Callable[[str], str]') -> 'list[str]': - """:file:`conda/requirements.txt`'s lines, sorted, one pin per vendored package.""" + """:file:`conda/requirements.txt`'s lines, sorted: a pin per vendored package, and per platform pin.""" result = [] for name, entry in sorted(found.items()): if entry['class'] == 'dead': continue - marker = render_marker(entry['builds'], envs) # type: ignore[arg-type] - pin = f'{name}=={version(name)}' - result.append(pin if marker is None else f'{pin}; {marker}') + for marker, fixed in pins(name, entry, envs): + pin = f'{name}=={fixed or version(name)}' + result.append(pin if marker is None else f'{pin}; {marker}') return result -def parse(text: 'str') -> 'dict[str, Optional[str]]': - """A requirements file's packages, by canonical name, each with its marker.""" +def parse(text: 'str') -> 'dict[str, dict[Optional[str], str]]': + """A requirements file's packages, by canonical name, each as {its markers: the versions they pin}.""" from packaging.requirements import Requirement # pylint: disable=import-outside-toplevel from packaging.utils import canonicalize_name # pylint: disable=import-outside-toplevel - result = {} # type: dict[str, Optional[str]] + result = {} # type: dict[str, dict[Optional[str], str]] for line in text.splitlines(): line = line.split('#', 1)[0].strip() if not line: continue req = Requirement(line) - result[canonicalize_name(req.name)] = None if req.marker is None else str(req.marker) + versions = [spec.version for spec in req.specifier if spec.operator == '=='] + markers = result.setdefault(canonicalize_name(req.name), {}) + markers[None if req.marker is None else str(req.marker)] = versions[0] if versions else '' return result +def unpinnable(found: 'dict[str, dict[str, object]]') -> 'list[str]': + """Each :data:`PLATFORM_PINS` entry no conda build can use, one line apiece.""" + problems = [] + for name, fixed in sorted(PLATFORM_PINS.items()): + entry = found.get(name) + if entry is None or entry['class'] == 'dead': + problems.append(f'override: {name} is pinned for {", ".join(sorted(fixed))}, ' + 'and no conda build vendors it') + for subdir in sorted(set(fixed) - set(PLATFORMS)): + problems.append(f'override: {name} is pinned for {subdir}, which no conda leg builds') + return problems + + +def _show(markers: 'Iterable[Optional[str]]') -> 'str': + shown = sorted(markers, key=str) + return repr(shown[0]) if len(shown) == 1 else repr(shown) + + def compare(text: 'str', found: 'dict[str, dict[str, object]]', envs: 'list[dict[str, str]]') -> 'list[str]': """Why ``text`` is stale against ``found``, one line per package; empty when current.""" from packaging.markers import Marker # pylint: disable=import-outside-toplevel + problems = unpinnable(found) have = parse(text) - want = {name: render_marker(entry['builds'], envs) # type: ignore[arg-type] + want = {name: {None if marker is None else str(Marker(marker)): fixed + for marker, fixed in pins(name, entry, envs)} for name, entry in found.items() if entry['class'] != 'dead'} - want = {name: None if marker is None else str(Marker(marker)) for name, marker in want.items()} - problems = [] for name in sorted(set(have) | set(want)): if name not in have: problems.append(f'missing: {name} ({found[name]["class"]})') @@ -290,8 +352,13 @@ def compare(text: 'str', found: 'dict[str, dict[str, object]]', else: problems.append(f'stale: {name} -- neither the dependencies nor the ' f'{", ".join(EXTRAS)} extra need it, directly or not') - elif have[name] != want[name]: - problems.append(f'marker: {name} has {have[name]!r}, derived {want[name]!r}') + elif set(have[name]) != set(want[name]): + problems.append(f'marker: {name} has {_show(have[name])}, derived {_show(want[name])}') + else: + for marker, fixed in sorted(want[name].items(), key=str): + if fixed is not None and have[name][marker] != fixed: + problems.append(f'pin: {name} pins {have[name][marker]!r} where {marker}, ' + f'and PLATFORM_PINS {fixed!r}') return problems @@ -482,6 +549,11 @@ def main(argv: 'Optional[list[str]]' = None) -> 'int': print(f'{FILE.relative_to(ROOT)}: {problem}') return 1 if problems else 0 + problems = unpinnable(found) + if problems: + print('\n'.join(problems), file=sys.stderr) + return 1 + FILE.write_text(''.join(f'{line}\n' for line in lines(found, envs, _version)), encoding='utf-8') return 0