From 44fe6d42a1e0a631dd85723f0c8f474a875117de Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Fri, 16 Oct 2026 22:28:52 -0400 Subject: [PATCH] ci(conda): move platform pins up and give them a review-by date (#1851) - Move PLATFORM_PINS into conda/platform-pins.toml, so the weekly job can rewrite it; each pin carries a version, a review-by date and a reason. - Add util/conda_vendor.py --refresh-pins: it reads PyPI's JSON, finds the newest release up to the installed one with a wheel every conda Python installs on the pinned platform, abi3 and universal2 included, and moves the pin there, due for review REVIEW_HORIZON out. A pin that reaches the installed release, or already pins it, is dropped. A PyPI failure warns and keeps the pin. - Read PyPI through requests, imported only for that request so --smoke still needs only the standard library, with each requests error raised as OSError; conda-update installs requests, which .[all] does not bring. - Run it in conda-update before the requirements are written. - --check stays offline and now fails on a pin past its review-by date. - Keep the table out of BOT_MANAGED_PATHS, pinned by a test, and correct that comment's claim about which files tests read. - Read the pin from the table in the #1827 tests, and fix today in the #1814 --check tests, so a moved or overdue pin does not break them. Closes #1851 --- .github/workflows/cron-conda.yml | 20 +- conda/platform-pins.toml | 18 + .../test_conda_cryptography_pin_1827.py | 107 ++- .../test_conda_platform_pin_refresh_1851.py | 639 ++++++++++++++++++ tests/project/test_conda_vendor_1814.py | 13 +- util/conda_vendor.py | 351 +++++++++- util/verdict_covers_checkout.py | 10 +- 7 files changed, 1097 insertions(+), 61 deletions(-) create mode 100644 conda/platform-pins.toml create mode 100644 tests/project/test_conda_platform_pin_refresh_1851.py diff --git a/.github/workflows/cron-conda.yml b/.github/workflows/cron-conda.yml index b4cce5d47..5a704d5ab 100644 --- a/.github/workflows/cron-conda.yml +++ b/.github/workflows/cron-conda.yml @@ -161,14 +161,30 @@ jobs: # extra, walks what they require through the installed metadata, and # pins each package at the version installed here, but for its # `PLATFORM_PINS` (#1827). So this install is the step that resolves the - # newest releases, and the script itself reads nothing from the network. + # newest releases, and of the script's runs only the next one reads the + # network, through `requests`, which `.[all]` does not bring. - name: Install and Setup run: | set -x - python -m pip install -U pip setuptools wheel packaging + python -m pip install -U pip setuptools wheel packaging requests python -m pip install -e .[all] + # #1851: asks PyPI for each `PLATFORM_PINS` package's releases and moves + # its pin in conda/platform-pins.toml up to the newest release, up to the + # one installed above, with a wheel for the pinned platform, renewing its + # review-by date; a pin that reaches the installed release, or already + # pins it, goes. + # The step below then writes the moved pin into conda/requirements.txt, + # and Commit Changes commits both. The table is not one of + # util/verdict_covers_checkout.py's bot-managed paths, which says why: + # the workflows chained off Unit Tests wait for such a commit's own run + # (#1643). A package PyPI cannot be read for keeps its pin, with a + # warning, and a pin past its review-by date gets one too; either way the + # step passes. + - name: Refresh the platform pins + run: python util/conda_vendor.py --refresh-pins + - name: Setup Conda Distribution run: | export PCAPKIT_DEVMODE=1 diff --git a/conda/platform-pins.toml b/conda/platform-pins.toml new file mode 100644 index 000000000..43c6d5a73 --- /dev/null +++ b/conda/platform-pins.toml @@ -0,0 +1,18 @@ +# The packages conda/requirements.txt pins apart on some conda platforms, as a +# table per package and conda subdir (#1827): the version that platform vendors +# in place of the installed release, the date the pin is due for review by, and +# why the platform cannot take the installed release. +# +# util/conda_vendor.py reads this file. Conda Update's `conda-update` job runs +# its `--refresh-pins` first, which asks PyPI for the newest release, up to the +# installed one, with a wheel each conda Python installs on the pinned platform. +# It moves the pin up to that release with a fresh review-by date, and drops a +# pin that reaches the installed release or already pins it (#1851). `--check` +# fails on a pin past its review-by date: check by hand that the pin still +# holds, then set a new date. The job writes this whole file, this comment +# included, so anything a pin needs said goes in its `reason`. + +[cryptography.osx-64] +version = "48.0.1" +review-by = 2027-01-15 +reason = "Its newest releases ship no Intel or universal2 macOS wheel, so the osx-64 build would compile them from source; 49.0.0 was the first (#1827)." diff --git a/tests/project/test_conda_cryptography_pin_1827.py b/tests/project/test_conda_cryptography_pin_1827.py index 1f40861ea..f4740c8ce 100644 --- a/tests/project/test_conda_cryptography_pin_1827.py +++ b/tests/project/test_conda_cryptography_pin_1827.py @@ -1,19 +1,23 @@ # -*- coding: utf-8 -*- """cryptography is pinned apart on Intel macOS, which has no wheel of its newest releases (#1827). -* :file:`conda/requirements.txt` carries a cryptography line per platform pin, - and on each conda build exactly one of them applies: 48.0.1 on ``osx-64``, - the installed release elsewhere. +* :file:`conda/requirements.txt` carries a line per platform pin of each package + :file:`conda/platform-pins.toml` pins, and of cryptography whether or not it + is, and on each conda build exactly one of a package's lines applies: the + table's pin on its platform, the installed release elsewhere. Read from the + table rather than spelt here, since the weekly Conda Update moves and drops + pins (#1851). * :file:`util/conda_vendor.py` writes those lines from ``PLATFORM_PINS``, and ``--check`` passes on them and fails when the table entry or the file's pin goes, when the pin moves, and when the table names a package or platform no - build has. All offline, on stand-in metadata. + build has. All offline, on stand-in metadata and a stand-in 48.0.1 pin. """ from __future__ import annotations import contextlib +import datetime import importlib.util import io import pathlib @@ -33,6 +37,7 @@ ROOT = pathlib.Path(__file__).resolve().parents[2] REQUIREMENTS = ROOT / 'conda' / 'requirements.txt' +#: The Intel macOS line :func:`stand_in`'s pin writes. INTEL = 'cryptography==48.0.1; sys_platform == "darwin" and platform_machine == "x86_64"' #: The other line, whose pin the weekly Conda Update moves. REST = re.compile(r'cryptography==(\S+); sys_platform != "darwin" or platform_machine != "x86_64"') @@ -59,45 +64,74 @@ def committed() -> 'list[str]': if line.startswith('cryptography')] +def pinned(*pins: 'tuple[str, str, str]') -> 'dict[str, dict[str, Any]]': + """A ``PLATFORM_PINS`` of (package, conda subdir, version), with a stand-in date and reason.""" + module, table = vendor(), {} # type: Any, dict[str, dict[str, Any]] + for name, subdir, version in pins: + table.setdefault(name, {})[subdir] = module.Pin(version, datetime.date(2027, 1, 15), 'a stand-in') + return table + + +def stand_in() -> 'Any': + """``PLATFORM_PINS`` patched to cryptography 48.0.1 on osx-64, as #1827 committed it.""" + return mock.patch.object(vendor(), 'PLATFORM_PINS', pinned(('cryptography', 'osx-64', '48.0.1'))) + + +def written() -> 'list[str]': + """The cryptography lines the writer gives :func:`stand_in`'s pin, with 50.0.2 installed.""" + module = vendor() + envs = module.environments(module.conda_pythons()) + with stand_in(): + return module.lines(cryptography(module, envs), envs, lambda name: '50.0.2') + + def cryptography(module: 'Any', envs: 'list[dict[str, str]]') -> 'dict[str, Any]': """What the conda builds vendor for ``pyproject.toml``'s cryptography line, on stand-in metadata.""" return module.derive([req for req in module.direct() if req.name == 'cryptography'], envs, lambda name: []) class CommittedLinesTests(unittest.TestCase): - """The committed cryptography lines, evaluated for each conda build.""" + """The committed lines of each pinned package, and of cryptography, evaluated for each conda build.""" - def test_each_build_reads_one_cryptography_line(self) -> None: + def test_each_build_reads_one_line_per_pinned_package(self) -> None: from packaging.requirements import Requirement module = vendor() - lines = committed() - self.assertIn(INTEL, lines) - self.assertEqual(len([line for line in lines if REST.fullmatch(line)]), 1, lines) - self.assertEqual(module.PLATFORM_PINS, {'cryptography': {'osx-64': '48.0.1'}}) - for env in module.environments(module.conda_pythons()): - subdir = next(subdir for subdir, (sys_platform, _, machine) in module.PLATFORMS.items() - if (sys_platform, machine) == (env['sys_platform'], env['platform_machine'])) - with self.subTest(subdir=subdir, python=env['python_version']): - live = [line for line in lines if Requirement(line).marker.evaluate(env)] - self.assertEqual(len(live), 1, live) - self.assertEqual(live[0] == INTEL, subdir == 'osx-64', live) + lines = [Requirement(line) for line in REQUIREMENTS.read_text(encoding='utf-8').splitlines() + if line.split('#', 1)[0].strip()] + for name in sorted({'cryptography', *module.PLATFORM_PINS}): + pins = module.PLATFORM_PINS.get(name, {}) + reads = {} # type: dict[tuple[str, str], str] + for env in module.environments(module.conda_pythons()): + subdir = next(subdir for subdir, (sys_platform, _, machine) in module.PLATFORMS.items() + if (sys_platform, machine) == (env['sys_platform'], env['platform_machine'])) + with self.subTest(package=name, subdir=subdir, python=env['python_version']): + live = [str(req) for req in lines + if req.name == name and (req.marker is None or req.marker.evaluate(env))] + self.assertEqual(len(live), 1, live) + reads[subdir, env['python_version']] = live[0] + if subdir in pins: + self.assertEqual(str(Requirement(live[0]).specifier), f'=={pins[subdir].version}') + # The builds no pin names read none of the pinned lines. + on_pinned = {line for (subdir, _), line in reads.items() if subdir in pins} + self.assertFalse({line for (subdir, _), line in reads.items() if subdir not in pins} & on_pinned, + reads) class WriterTests(unittest.TestCase): """The writer emits a line per platform pin.""" def test_the_writer_emits_the_platform_lines(self) -> None: - module = vendor() - envs = module.environments(module.conda_pythons()) - self.assertEqual(module.lines(cryptography(module, envs), envs, lambda name: '50.0.2'), + self.assertEqual(written(), [INTEL, 'cryptography==50.0.2; sys_platform != "darwin" or platform_machine != "x86_64"']) + self.assertIsNotNone(REST.fullmatch(written()[1])) def test_a_python_marker_is_kept_beside_the_platform_one(self) -> None: from packaging.requirements import Requirement module = vendor() envs = module.environments(['3.11', '3.12']) found = module.derive([Requirement('cryptography; python_version >= "3.12"')], envs, lambda name: []) - lines = module.lines(found, envs, lambda name: '50.0.2') + with stand_in(): + lines = module.lines(found, envs, lambda name: '50.0.2') self.assertEqual(lines, [ 'cryptography==48.0.1; (python_version == "3.12") and ' '(sys_platform == "darwin" and platform_machine == "x86_64")', @@ -111,7 +145,7 @@ def test_a_python_marker_is_kept_beside_the_platform_one(self) -> None: class CheckTests(unittest.TestCase): - """``--check`` against the committed cryptography lines.""" + """``--check`` against the committed cryptography lines, and against :func:`stand_in`'s.""" def problems(self, text: 'str') -> 'list[str]': module = vendor() @@ -119,30 +153,37 @@ def problems(self, text: 'str') -> 'list[str]': return module.compare(text, cryptography(module, envs), envs) def test_the_committed_lines_pass(self) -> None: - lines = committed() - self.assertIn(INTEL, lines) - self.assertEqual(self.problems('\n'.join(lines)), []) + self.assertEqual(self.problems('\n'.join(committed())), []) + + def test_the_stand_in_lines_pass(self) -> None: + lines = written() + with stand_in(): + self.assertEqual(self.problems('\n'.join(lines)), []) def test_without_the_override_the_platform_lines_fail(self) -> None: + lines = written() with mock.patch.object(vendor(), 'PLATFORM_PINS', {}): - problems = self.problems('\n'.join(committed())) + problems = self.problems('\n'.join(lines)) self.assertEqual(len(problems), 1, problems) self.assertTrue(problems[0].startswith('marker: cryptography has ['), problems) def test_a_dropped_or_moved_platform_pin_fails(self) -> None: - rest = [line for line in committed() if line != INTEL] - dropped = self.problems('\n'.join(rest)) + rest = [line for line in written() if line != INTEL] + with stand_in(): + dropped = self.problems('\n'.join(rest)) + unmarked = self.problems('\n'.join(rest + ['cryptography==48.0.1'])) + moved = self.problems('\n'.join(rest + [INTEL.replace('48.0.1', '49.0.0')])) self.assertEqual([problem.split(':')[0] for problem in dropped], ['marker'], dropped) - unmarked = self.problems('\n'.join(rest + ['cryptography==48.0.1'])) self.assertEqual([problem.split(':')[0] for problem in unmarked], ['marker'], unmarked) - moved = self.problems('\n'.join(rest + [INTEL.replace('48.0.1', '49.0.0')])) self.assertEqual(moved, ["pin: cryptography pins '49.0.0' where sys_platform == \"darwin\" and " "platform_machine == \"x86_64\", and PLATFORM_PINS '48.0.1'"]) def test_an_override_no_build_can_use_fails(self) -> None: - pins = {'cryptography': {'osx-64': '48.0.1', 'win-64': '1.0'}, 'gone': {'osx-64': '1.0'}} + lines = written() + pins = pinned(('cryptography', 'osx-64', '48.0.1'), ('cryptography', 'win-64', '1.0'), + ('gone', 'osx-64', '1.0')) with mock.patch.object(vendor(), 'PLATFORM_PINS', pins): - problems = self.problems('\n'.join(committed())) + problems = self.problems('\n'.join(lines)) self.assertEqual(problems, ['override: cryptography is pinned for win-64, which no conda leg builds', 'override: gone is pinned for osx-64, and no conda build vendors it']) @@ -155,7 +196,7 @@ def test_the_writer_refuses_an_override_no_build_can_use(self) -> None: module._requires = lambda name: [] module._version = lambda name: '1.0' try: - with mock.patch.object(module, 'PLATFORM_PINS', {'gone': {'osx-64': '1.0'}}), \ + with mock.patch.object(module, 'PLATFORM_PINS', pinned(('gone', 'osx-64', '1.0'))), \ contextlib.redirect_stderr(io.StringIO()) as errors: self.assertEqual(module.main([]), 1) self.assertFalse(target.exists()) diff --git a/tests/project/test_conda_platform_pin_refresh_1851.py b/tests/project/test_conda_platform_pin_refresh_1851.py new file mode 100644 index 000000000..2a9604337 --- /dev/null +++ b/tests/project/test_conda_platform_pin_refresh_1851.py @@ -0,0 +1,639 @@ +# -*- coding: utf-8 -*- +"""The weekly Conda Update moves each platform pin up, and each pin has a review-by date (#1851). + +* :func:`conda_vendor.installs` and :func:`conda_vendor.newest_wheel` find the + newest release with a wheel every conda Python installs on a platform, on + PyPI's JSON for cryptography as recorded here: ``abi3`` from an older + CPython, ``universal2`` on either Mac, and not a free-threaded, arm64-only, + musllinux, yanked or pre-release file. +* :func:`conda_vendor.refresh` moves a pin up to that release with a fresh + review-by date, drops it once it reaches the installed release or if it + already pins it, and leaves it, with a warning, when PyPI cannot be read. + ``--refresh-pins`` rewrites :file:`conda/platform-pins.toml` only on a change. +* :func:`conda_vendor._fetch` reads PyPI through :mod:`requests`, a stand-in + here, and raises each of its errors as :class:`OSError`. It imports requests + itself, so the script loads, and ``--smoke`` passes, without it. +* The table is not one of :file:`util/verdict_covers_checkout.py`'s bot-managed + paths, so a commit moving a pin waits for its own ``Unit Tests`` run. +* ``--check`` fails on a pin past its review-by date, and passes on one in date, + without reading the network. +* :file:`conda/platform-pins.toml` is in the form the job writes, and + ``conda-update`` in :file:`.github/workflows/cron-conda.yml` installs + requests and runs ``--refresh-pins`` between installing ``.[all]`` and + writing the requirements, read with :func:`yaml.safe_load`. + +""" + +from __future__ import annotations + +import bz2 +import contextlib +import datetime +import importlib.util +import io +import json +import pathlib +import shlex +import sys +import tarfile +import tempfile +import types +import unittest +from typing import TYPE_CHECKING +from unittest import mock + +try: + import tomllib +except ImportError: # pragma: no cover + tomllib = None # type: ignore[assignment] + +try: + import yaml +except ImportError: # pragma: no cover + yaml = None # type: ignore[assignment] + +if TYPE_CHECKING: + from typing import Any, Optional + +ROOT = pathlib.Path(__file__).resolve().parents[2] +WORKFLOW = ROOT / '.github' / 'workflows' / 'cron-conda.yml' +#: The conda Pythons the stand-ins are built for; the matcher takes any list. +PYTHONS = ('3.11', '3.12', '3.13', '3.14', '3.15') +#: The day the stand-in runs happen on. +TODAY = datetime.date(2026, 10, 17) +#: What every file of the recorded releases says of the Python it needs. +REQUIRES = '!=3.9.0,!=3.9.1,>=3.9' + +_MODULE = [] # type: list[Any] + + +def vendor() -> 'Any': + """:file:`util/conda_vendor.py`, loaded once; skipped without :mod:`tomllib`, which it imports.""" + if tomllib is None: # pragma: no cover + raise unittest.SkipTest('tomllib, which util/conda_vendor.py imports, needs Python 3.11+') + if not _MODULE: + spec = importlib.util.spec_from_file_location('conda_vendor', ROOT / 'util' / 'conda_vendor.py') + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) # type: ignore[union-attr] + _MODULE.append(module) + return _MODULE[0] + + +def files(version: 'str', *tags: 'str', yanked: 'bool' = False, + requires_python: 'str' = REQUIRES) -> 'list[dict[str, Any]]': + """A release's files as PyPI's JSON lists them, cut to the fields :func:`newest_wheel` reads. + + Each of ``tags`` is a wheel's ``{python}-{abi}-{platform}``; the sdist comes last. + + """ + found = [{'filename': f'cryptography-{version}-{tag}.whl', 'packagetype': 'bdist_wheel', + 'requires_python': requires_python, 'yanked': yanked} for tag in tags] + found.append({'filename': f'cryptography-{version}.tar.gz', 'packagetype': 'sdist', + 'requires_python': requires_python, 'yanked': yanked}) + return found + + +def recorded(**more: 'list[dict[str, Any]]') -> 'dict[str, Any]': + """PyPI's JSON for cryptography, with ``more`` releases besides. + + Recorded from https://pypi.org/pypi/cryptography/json on 2026-10-16, cut to + some of its releases and of their files. 45.0.0 is yanked whole; 48.0.1 is the + newest with a universal2 wheel, which #1827 pins osx-64 to; 49.0.0 and 50.0.2 + have arm64 macOS wheels alone. + + """ + releases = { + '45.0.0': files('45.0.0', 'cp311-abi3-macosx_10_9_universal2', 'cp311-abi3-manylinux_2_34_x86_64', + 'cp37-abi3-macosx_10_9_universal2', yanked=True, + requires_python='!=3.9.0,!=3.9.1,>=3.7'), + '48.0.1': files('48.0.1', 'cp311-abi3-macosx_10_9_universal2', 'cp311-abi3-manylinux_2_34_x86_64', + 'cp311-abi3-musllinux_1_2_x86_64', 'cp311-abi3-win_amd64', + 'cp314-cp314t-macosx_10_9_universal2', 'cp39-abi3-macosx_10_9_universal2'), + '49.0.0': files('49.0.0', 'cp311-abi3-macosx_11_0_arm64', 'cp311-abi3-manylinux_2_34_x86_64', + 'cp311-abi3-musllinux_1_2_x86_64', 'cp314-cp314t-macosx_11_0_arm64', + 'cp39-abi3-macosx_11_0_arm64'), + '50.0.2': files('50.0.2', 'cp311-abi3-macosx_11_0_arm64', + 'cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64', + 'cp311-abi3-musllinux_1_2_x86_64', 'cp315-abi3.abi3t-macosx_11_0_arm64', + 'cp39-abi3-macosx_11_0_arm64'), + } + releases.update(more) + return {'info': {'name': 'cryptography', 'version': '50.0.2'}, 'releases': releases} + + +#: A later release with an Intel macOS wheel again, as the stand-in for upstream shipping one. +INTEL_AGAIN = files('51.0.0', 'cp311-abi3-macosx_10_9_universal2', 'cp311-abi3-manylinux_2_34_x86_64') + + +def table(version: 'str' = '48.0.1', review_by: 'datetime.date' = datetime.date(2026, 12, 1), + ) -> 'dict[str, dict[str, Any]]': + """A ``PLATFORM_PINS`` pinning cryptography to ``version`` on osx-64.""" + return {'cryptography': {'osx-64': vendor().Pin(version, review_by, 'no Intel wheel (#1827)')}} + + +def stand_in_requests(payload: 'Any' = None, *, error: 'Optional[str]' = None, status: 'Optional[str]' = None, + undecodable: 'bool' = False) -> 'tuple[types.ModuleType, list[tuple[str, dict[str, Any]]]]': + """A stand-in :mod:`requests` whose ``get`` answers ``payload``, and the ``get`` calls it is asked. + + ``error`` fails the request itself, ``status`` its ``raise_for_status``, and + ``undecodable`` its ``json``. Its ``RequestException`` derives from + :class:`Exception` alone, unlike requests' own, so :func:`conda_vendor._fetch` + has to catch it by name for :func:`conda_vendor.refresh` to catch it at all. + + """ + calls = [] # type: list[tuple[str, dict[str, Any]]] + fake = types.ModuleType('requests') + + class RequestException(Exception): + """requests' base error, here not an :class:`OSError`.""" + + class Response: + """A reply, which fails as the arguments say.""" + + def raise_for_status(self) -> None: + if status is not None: + raise RequestException(status) + + def json(self) -> 'Any': + if undecodable: + raise RequestException('Expecting value: line 1 column 1 (char 0)') + return payload + + def get(url: 'str', **kwargs: 'Any') -> 'Response': + calls.append((url, kwargs)) + if error is not None: + raise RequestException(error) + return Response() + + fake.RequestException = RequestException # type: ignore[attr-defined] + fake.get = get # type: ignore[attr-defined] + return fake, calls + + +def no_urllib() -> 'Any': + """:func:`urllib.request.urlopen` patched to fail, so a test reading PyPI some other way never reaches it.""" + return mock.patch('urllib.request.urlopen', side_effect=AssertionError('PyPI was read through urllib')) + + +#: A built conda package's files, as --smoke passes them: pcapkit's _extern shim +#: and one vendored distribution. +SMOKE_FILES = { + 'pcapkit/__init__.py': ("import os, sys\n_extern = os.path.join(os.path.dirname(os.path.realpath(__file__)), " + "'_extern')\nif os.path.exists(_extern):\n sys.path.append(_extern)\n"), + 'pcapkit/_extern/bundle/__init__.py': '', + 'pcapkit/_extern/bundle-1.0.dist-info/METADATA': 'Name: bundle\n', + 'pcapkit/_extern/bundle-1.0.dist-info/RECORD': + 'bundle/__init__.py,,\nbundle-1.0.dist-info/METADATA,,\nbundle-1.0.dist-info/RECORD,,\n', +} + + +def smoke_package(module: 'Any', directory: 'pathlib.Path') -> 'pathlib.Path': + """A ``.tar.bz2`` conda package of :data:`SMOKE_FILES` for this runner, in ``directory``.""" + runner = module.runner_platform() + if runner is None: # pragma: no cover + raise unittest.SkipTest('this machine is none of the platforms the conda legs build on') + site = f'lib/python{sys.version_info.major}.{sys.version_info.minor}/site-packages/' + members = {site + name: text for name, text in SMOKE_FILES.items()} + members['info/index.json'] = json.dumps({'name': 'pypcapkit', 'subdir': runner}) + buffer = io.BytesIO() + with tarfile.open(fileobj=buffer, mode='w') as archive: + for name, text in members.items(): + data = text.encode('utf-8') + info = tarfile.TarInfo(name) + info.size = len(data) + archive.addfile(info, io.BytesIO(data)) + path = directory / 'pypcapkit-0-py_0.tar.bz2' + path.write_bytes(bz2.compress(buffer.getvalue())) + return path + + +class TagMatcherTests(unittest.TestCase): + """:func:`installs` on single wheels, and :func:`newest_wheel` on the recording.""" + + def test_abi3_installs_on_its_own_and_every_later_python(self) -> None: + installs = vendor().installs + wheel = 'cryptography-48.0.1-cp311-abi3-macosx_10_9_universal2.whl' + for python in PYTHONS: + with self.subTest(python=python): + self.assertTrue(installs(wheel, 'osx-64', python, REQUIRES)) + self.assertFalse(installs(wheel, 'osx-64', '3.10', REQUIRES)) + + def test_universal2_installs_on_either_mac_and_not_on_linux(self) -> None: + installs = vendor().installs + wheel = 'cryptography-48.0.1-cp311-abi3-macosx_10_9_universal2.whl' + self.assertTrue(installs(wheel, 'osx-64', '3.12')) + self.assertTrue(installs(wheel, 'osx-arm64', '3.12')) + self.assertFalse(installs(wheel, 'linux-64', '3.12')) + + def test_an_arm64_wheel_does_not_install_on_intel(self) -> None: + installs = vendor().installs + wheel = 'cryptography-49.0.0-cp311-abi3-macosx_11_0_arm64.whl' + self.assertFalse(installs(wheel, 'osx-64', '3.12')) + self.assertTrue(installs(wheel, 'osx-arm64', '3.12')) + + def test_an_own_abi_wheel_installs_on_its_own_python_alone(self) -> None: + installs = vendor().installs + wheel = 'cryptography-51.0.0-cp312-cp312-macosx_10_13_x86_64.whl' + self.assertEqual([python for python in PYTHONS if installs(wheel, 'osx-64', python)], ['3.12']) + self.assertFalse(installs('cryptography-48.0.1-cp314-cp314t-macosx_10_9_universal2.whl', 'osx-64', '3.14')) + + def test_manylinux_installs_on_linux_and_musllinux_nowhere(self) -> None: + installs = vendor().installs + for wheel, subdir, expected in ( + ('cryptography-48.0.1-cp311-abi3-manylinux_2_34_x86_64.whl', 'linux-64', True), + ('cryptography-50.0.2-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl', 'linux-64', True), + ('cryptography-48.0.1-cp311-abi3-musllinux_1_2_x86_64.whl', 'linux-64', False), + ('cryptography-48.0.1-cp311-abi3-manylinux_2_34_x86_64.whl', 'osx-64', False), + ('cryptography-48.0.1-cp311-abi3-win_amd64.whl', 'osx-64', False), + ('cryptography-51.0.0-py3-none-any.whl', 'linux-64', True), + ('cryptography-51.0.0-py3-none-any.whl', 'osx-64', True)): + with self.subTest(wheel=wheel, subdir=subdir): + self.assertEqual(installs(wheel, subdir, '3.12'), expected) + + def test_an_sdist_and_an_excluded_python_install_nowhere(self) -> None: + installs = vendor().installs + self.assertFalse(installs('cryptography-48.0.1.tar.gz', 'osx-64', '3.12')) + wheel = 'cryptography-51.0.0-cp311-abi3-macosx_10_9_universal2.whl' + self.assertFalse(installs(wheel, 'osx-64', '3.11', '>=3.12')) + self.assertTrue(installs(wheel, 'osx-64', '3.12', '>=3.12')) + + def test_the_recording_has_osx_64_wheels_up_to_48_0_1(self) -> None: + newest = vendor().newest_wheel + self.assertEqual(newest(recorded(), 'osx-64', PYTHONS), '48.0.1') + self.assertEqual(newest(recorded(), 'osx-arm64', PYTHONS), '50.0.2') + self.assertEqual(newest(recorded(), 'linux-64', PYTHONS), '50.0.2') + self.assertEqual(newest(recorded(), 'linux-64', PYTHONS, ceiling='49.0.0'), '49.0.0') + + def test_no_matching_wheel_is_none(self) -> None: + newest = vendor().newest_wheel + # 45.0.0's universal2 wheels are yanked, so nothing up to 47 has one. + self.assertIsNone(newest(recorded(), 'osx-64', PYTHONS, ceiling='47.0.0')) + self.assertIsNone(newest({'releases': {}}, 'osx-64', PYTHONS)) + + def test_every_python_needs_a_wheel(self) -> None: + newest = vendor().newest_wheel + project = recorded(**{'51.0.0': files('51.0.0', 'cp312-cp312-macosx_10_13_x86_64')}) + self.assertEqual(newest(project, 'osx-64', PYTHONS), '48.0.1') + self.assertEqual(newest(project, 'osx-64', ['3.12']), '51.0.0') + + def test_a_pre_release_and_a_yanked_wheel_are_passed_over(self) -> None: + newest = vendor().newest_wheel + project = recorded(**{'52.0.0rc1': files('52.0.0rc1', 'cp311-abi3-macosx_10_9_universal2'), + '51.0.0': files('51.0.0', 'cp311-abi3-macosx_10_9_universal2', yanked=True)}) + self.assertEqual(newest(project, 'osx-64', PYTHONS), '48.0.1') + self.assertEqual(newest(recorded(**{'51.0.0': INTEL_AGAIN}), 'osx-64', PYTHONS), '51.0.0') + + def test_json_without_releases_is_refused(self) -> None: + with self.assertRaises(ValueError): + vendor().newest_wheel({'info': {}}, 'osx-64', PYTHONS) + + +class BumpTests(unittest.TestCase): + """:func:`refresh` on stand-in pins, the recording and stand-in installed versions.""" + + def refresh(self, project: 'Any', installed: 'Any' = '52.0.0', pins: 'Any' = None) -> 'Any': + def fetch(name: 'str') -> 'Any': + if isinstance(project, BaseException): + raise project + return project + return vendor().refresh(table() if pins is None else pins, fetch, + lambda name: installed, PYTHONS, TODAY) + + def test_a_newer_wheel_moves_the_pin_and_renews_its_date(self) -> None: + module = vendor() + result, notes = self.refresh(recorded(**{'51.0.0': INTEL_AGAIN})) + self.assertEqual(result, {'cryptography': {'osx-64': module.Pin( + '51.0.0', TODAY + module.REVIEW_HORIZON, 'no Intel wheel (#1827)')}}) + self.assertEqual(TODAY + module.REVIEW_HORIZON, datetime.date(2027, 1, 16)) + self.assertEqual(notes, [('notice', 'cryptography on osx-64 moves from 48.0.1 to 51.0.0, the newest ' + 'release with a wheel there; due for review by 2027-01-16')]) + + def test_the_installed_release_with_a_wheel_drops_the_pin(self) -> None: + result, notes = self.refresh(recorded(**{'51.0.0': INTEL_AGAIN}), installed='51.0.0') + self.assertEqual(result, {}) + self.assertEqual(notes, [('notice', 'cryptography 51.0.0, the installed release, has a wheel for ' + 'osx-64, so its 48.0.1 pin there is dropped')]) + + def test_a_pin_already_at_the_installed_release_is_dropped(self) -> None: + for project, version in ((recorded(), '50.0.2'), (recorded(**{'51.0.0': INTEL_AGAIN}), '51.0.0')): + with self.subTest(pin=version, wheel=version == '51.0.0'): + result, notes = self.refresh(project, installed=version, pins=table(version)) + self.assertEqual(result, {}) + self.assertEqual(notes, [('notice', f'cryptography {version} on osx-64 is the installed ' + 'release itself, so the pin is dropped')]) + + def test_a_pin_with_no_newer_wheel_keeps_its_version_and_date(self) -> None: + self.assertEqual(self.refresh(recorded(), installed='50.0.2'), (table(), [])) + + def test_a_pin_never_passes_the_installed_release(self) -> None: + self.assertEqual(self.refresh(recorded(**{'51.0.0': INTEL_AGAIN}), installed='50.0.2'), (table(), [])) + + def test_a_pin_on_a_platform_no_leg_builds_is_left_alone(self) -> None: + pins = {'cryptography': {'win-64': vendor().Pin('1.0', TODAY, 'a stand-in')}} + result, notes = self.refresh(recorded(**{'51.0.0': INTEL_AGAIN}), pins=pins) + self.assertEqual(result, pins) + self.assertEqual([level for level, _ in notes], ['warning']) + + def test_pypi_failing_leaves_the_pin_and_warns(self) -> None: + # What _fetch raises: OSError for a failed request (FetchTests), and + # ValueError for a reply an older requests cannot decode. + for error in (OSError('503 Server Error: Service Unavailable'), TimeoutError('timed out'), + ValueError('Expecting value: line 1 column 1')): + with self.subTest(error=type(error).__name__): + result, notes = self.refresh(error) + self.assertEqual(result, table()) + self.assertEqual(len(notes), 1, notes) + self.assertEqual(notes[0][0], 'warning') + self.assertTrue(notes[0][1].startswith('PyPI could not be read for cryptography ('), notes) + self.assertTrue(notes[0][1].endswith('), so its pins stay as they are'), notes) + + def test_json_without_releases_leaves_the_pin_and_warns(self) -> None: + result, notes = self.refresh({'message': 'Not Found'}) + self.assertEqual(result, table()) + self.assertEqual(notes, [('warning', "cryptography 48.0.1 on osx-64 stays as it is: " + "PyPI's JSON has no releases")]) + + def test_a_package_not_installed_leaves_the_pin_and_warns(self) -> None: + result, notes = self.refresh(recorded(**{'51.0.0': INTEL_AGAIN}), installed=None) + self.assertEqual(result, table()) + self.assertEqual(notes, [('warning', 'cryptography is not installed here, so its pins stay as they are')]) + + +class FetchTests(unittest.TestCase): + """``_fetch`` reads PyPI through :mod:`requests`, imported only then, and raises its errors as OSError.""" + + HEADERS = {'User-Agent': 'PyPCAPKit conda-update (#1851)', 'Accept': 'application/json'} + + def fetch(self, fake: 'types.ModuleType') -> 'Any': + with no_urllib(), mock.patch.dict(sys.modules, {'requests': fake}): + return vendor()._fetch('cryptography') + + def test_pypi_is_read_through_requests(self) -> None: + fake, calls = stand_in_requests(recorded()) + self.assertEqual(self.fetch(fake), recorded()) + self.assertEqual(calls, [('https://pypi.org/pypi/cryptography/json', + {'headers': self.HEADERS, 'timeout': 60})]) + + def test_each_requests_error_is_raised_as_one_refresh_catches(self) -> None: + module = vendor() + for failure, message in (({'error': 'Name or service not known'}, 'Name or service not known'), + ({'status': '503 Server Error: Service Unavailable'}, + '503 Server Error: Service Unavailable'), + ({'undecodable': True}, 'Expecting value: line 1 column 1 (char 0)')): + with self.subTest(**failure): + fake, _ = stand_in_requests(recorded(), **failure) + self.assertFalse(issubclass(fake.RequestException, OSError)) # type: ignore[attr-defined] + with self.assertRaises(OSError) as raised: + self.fetch(fake) + self.assertIsInstance(raised.exception, module.UNREADABLE) + self.assertEqual(str(raised.exception), message) + self.assertIsInstance(raised.exception.__cause__, fake.RequestException) # type: ignore[attr-defined] + + def test_without_requests_only_the_request_fails(self) -> None: + """The script loads, and ``--smoke`` passes, with :mod:`requests` unimportable. + + ``conda-dist`` and Create Release's ``conda`` legs run ``--smoke`` with + pip, setuptools and wheel alone installed (#1854). + + """ + with no_urllib(), mock.patch.dict(sys.modules, {'requests': None}): + spec = importlib.util.spec_from_file_location('conda_vendor_without_requests', + ROOT / 'util' / 'conda_vendor.py') + assert spec is not None and spec.loader is not None + fresh = importlib.util.module_from_spec(spec) + spec.loader.exec_module(fresh) # type: ignore[union-attr] + with self.assertRaises(ImportError): + fresh._fetch('cryptography') + with tempfile.TemporaryDirectory() as temp: + package = smoke_package(fresh, pathlib.Path(temp)) + self.assertEqual(fresh.main(['--smoke', str(package)]), 0) + + +class Scratch: + """``util/conda_vendor.py``'s files and hooks pointed into a temporary tree, for :func:`main`.""" + + def __init__(self, pins: 'dict[str, dict[str, Any]]') -> None: + self.pins = pins + + def __enter__(self) -> 'Scratch': + module = vendor() + self.temp = tempfile.TemporaryDirectory() + root = pathlib.Path(self.temp.name) + (root / 'conda').mkdir() + self.file, self.table = root / 'conda' / 'requirements.txt', root / 'conda' / 'platform-pins.toml' + self.table.write_text(module.render_pins(self.pins), encoding='utf-8') + self.saved = {name: getattr(module, name) for name in ( + 'ROOT', 'FILE', 'PINS', 'PLATFORM_PINS', '_requires', '_version', '_installed', '_fetch', '_today')} + module.ROOT, module.FILE, module.PINS, module.PLATFORM_PINS = root, self.file, self.table, self.pins + module._requires = lambda name: [] + module._version = module._installed = lambda name: '52.0.0' + module._today = lambda: TODAY + module._fetch = self.offline + return self + + @staticmethod + def offline(name: 'str') -> 'Any': + raise AssertionError(f'PyPI was asked for {name}') + + def __exit__(self, *exc_info: 'Any') -> None: + module = vendor() + for name, value in self.saved.items(): + setattr(module, name, value) + self.temp.cleanup() + + def main(self, *argv: 'str') -> 'tuple[int, str]': + with contextlib.redirect_stdout(io.StringIO()) as output: + status = vendor().main(list(argv)) + return status, output.getvalue() + + +class RefreshCommandTests(unittest.TestCase): + """``--refresh-pins`` end to end, on a scratch table and a stand-in PyPI.""" + + def test_a_moved_pin_is_written_with_its_new_date(self) -> None: + module = vendor() + with Scratch(table()) as scratch: + module._fetch = lambda name: recorded(**{'51.0.0': INTEL_AGAIN}) + status, output = scratch.main('--refresh-pins') + written = scratch.table.read_text(encoding='utf-8') + self.assertEqual(status, 0) + self.assertEqual(written, module.render_pins(table('51.0.0', datetime.date(2027, 1, 16)))) + self.assertIn('review-by = 2027-01-16\n', written) + self.assertEqual(output, '::notice title=Platform pins::cryptography on osx-64 moves from 48.0.1 to ' + '51.0.0, the newest release with a wheel there; due for review by 2027-01-16\n') + + def test_a_pin_at_the_installed_release_is_written_out(self) -> None: + module = vendor() + with Scratch(table('52.0.0')) as scratch: + module._fetch = lambda name: recorded() + status, output = scratch.main('--refresh-pins') + written = scratch.table.read_text(encoding='utf-8') + self.assertEqual(status, 0) + self.assertEqual(written, module.render_pins({})) + self.assertEqual(output, '::notice title=Platform pins::cryptography 52.0.0 on osx-64 is the installed ' + 'release itself, so the pin is dropped\n') + + def test_a_network_failure_writes_nothing_and_passes(self) -> None: + module = vendor() + fake, calls = stand_in_requests(error='Name or service not known') + with Scratch(table()) as scratch, no_urllib(), mock.patch.dict(sys.modules, {'requests': fake}): + before = scratch.table.read_bytes() + module._fetch = scratch.saved['_fetch'] + status, output = scratch.main('--refresh-pins') + after = scratch.table.read_bytes() + self.assertEqual(status, 0) + self.assertEqual(after, before) + self.assertEqual([url for url, _ in calls], ['https://pypi.org/pypi/cryptography/json']) + self.assertEqual(output, '::warning title=Platform pins::PyPI could not be read for cryptography ' + '(Name or service not known), so its pins stay as they are\n') + + def test_an_overdue_pin_left_where_it_is_is_warned_about(self) -> None: + module = vendor() + with Scratch(table(review_by=TODAY - datetime.timedelta(days=1))) as scratch: + module._fetch = lambda name: recorded() + status, output = scratch.main('--refresh-pins') + self.assertEqual(status, 0) + self.assertEqual(output.splitlines(), [ + '::warning title=Platform pins::overdue: cryptography is pinned to 48.0.1 on osx-64, and was due for ' + 'review by 2026-10-16; check the pin still holds, then set a new review-by date, such as 2027-01-16']) + + +class CheckTests(unittest.TestCase): + """``--check`` on a pin in date and past it, never reading the network.""" + + def test_review_reports_a_pin_past_its_date_only(self) -> None: + review = vendor().review + due = datetime.date(2026, 12, 1) + self.assertEqual(review(table(review_by=due), due - datetime.timedelta(days=1)), []) + self.assertEqual(review(table(review_by=due), due), []) + self.assertEqual(review(table(review_by=due), due + datetime.timedelta(days=1)), [ + 'overdue: cryptography is pinned to 48.0.1 on osx-64, and was due for review by 2026-12-01; ' + 'check the pin still holds, then set a new review-by date, such as 2027-03-03']) + + def test_check_passes_on_a_pin_in_date(self) -> None: + with Scratch(table(review_by=TODAY)) as scratch, mock.patch.dict(sys.modules, {'requests': None}): + self.assertEqual(scratch.main(), (0, '')) + self.assertEqual(scratch.main('--check'), (0, '')) + + def test_check_fails_on_a_pin_past_its_date(self) -> None: + with Scratch(table(review_by=TODAY - datetime.timedelta(days=1))) as scratch, \ + mock.patch.dict(sys.modules, {'requests': None}): + self.assertEqual(scratch.main(), (0, '')) + status, output = scratch.main('--check') + self.assertEqual(status, 1) + self.assertEqual(output, 'conda/platform-pins.toml: overdue: cryptography is pinned to 48.0.1 on osx-64, ' + 'and was due for review by 2026-10-16; check the pin still holds, then set a new ' + 'review-by date, such as 2027-01-16\n') + + +class TableFileTests(unittest.TestCase): + """:file:`conda/platform-pins.toml` as committed, and as :func:`render_pins` writes it.""" + + def test_the_committed_table_is_in_the_form_the_job_writes(self) -> None: + module = vendor() + text = module.PINS.read_text(encoding='utf-8') + self.assertEqual(module.load_pins(text), module.PLATFORM_PINS) + self.assertEqual(module.render_pins(module.PLATFORM_PINS), text) + + def test_every_committed_pin_says_why(self) -> None: + for name, platforms in vendor().PLATFORM_PINS.items(): + for subdir, pin in platforms.items(): + with self.subTest(package=name, subdir=subdir): + self.assertTrue(pin.reason.strip()) + + def test_a_table_reads_back_as_written(self) -> None: + module = vendor() + pins = {'cryptography': {'osx-64': module.Pin('48.0.1', TODAY, 'say "why" \\ here\n\x7f, é')}, + 'lxml': {'linux-64': module.Pin('6.0', TODAY, ''), 'osx-arm64': module.Pin('6.1', TODAY, 'b')}} + self.assertEqual(module.load_pins(module.render_pins(pins)), pins) + self.assertEqual(module.load_pins(module.render_pins({})), {}) + + def test_a_malformed_pin_is_refused(self) -> None: + load = vendor().load_pins + for text in ('[cryptography.osx-64]\nversion = "48.0.1"\nreason = "r"\n', + '[cryptography.osx-64]\nversion = "48.0.1"\nreview-by = "2027-01-15"\nreason = "r"\n', + '[cryptography.osx-64]\nversion = "48.0.1"\nreview-by = 2027-01-15T00:00:00\nreason = "r"\n', + '[cryptography.osx-64]\nversion = 48\nreview-by = 2027-01-15\nreason = "r"\n', + '[cryptography.osx-64]\nversion = "48.0.1"\nreview-by = 2027-01-15\nreason = "r"\nnote = 1\n', + 'cryptography = "48.0.1"\n'): + with self.subTest(text=text), self.assertRaises(ValueError): + load(text) + + +class VerdictTests(unittest.TestCase): + """A commit moving a pin is not built under its parent's ``Unit Tests`` verdict (#1643).""" + + def test_the_table_is_not_a_bot_managed_path(self) -> None: + """:file:`util/verdict_covers_checkout.py` must not wave the table's commits through. + + :file:`util/conda_vendor.py` reads the table on import, so every test of + that script does, and a moved pin changes what those tests expect of + :file:`conda/requirements.txt`. A bot commit to the table therefore waits + for its own ``Unit Tests`` run instead of being covered by its parent's. + + """ + spec = importlib.util.spec_from_file_location('verdict_covers_checkout', + ROOT / 'util' / 'verdict_covers_checkout.py') + assert spec is not None and spec.loader is not None + verdict = importlib.util.module_from_spec(spec) + spec.loader.exec_module(verdict) # type: ignore[union-attr] + self.assertEqual(vendor().PINS, ROOT / 'conda' / 'platform-pins.toml') + self.assertNotIn('conda/platform-pins.toml', verdict.BOT_MANAGED_PATHS) + + +@unittest.skipIf(yaml is None, 'PyYAML, from the test extra, is not installed') +class WorkflowTests(unittest.TestCase): + """``conda-update`` refreshes the pins after installing ``.[all]`` and before writing the requirements.""" + + def setUp(self) -> None: + self.jobs = yaml.safe_load(WORKFLOW.read_text(encoding='utf-8'))['jobs'] + self.steps = self.jobs['conda-update']['steps'] + + def index(self, found: 'Any') -> 'int': + """The index of the one ``conda-update`` step ``found`` holds for.""" + hits = [index for index, step in enumerate(self.steps) if found(step)] + self.assertEqual(len(hits), 1, hits) + return hits[0] + + @staticmethod + def runs(step: 'dict[str, Any]') -> 'list[str]': + return [line.strip() for line in str(step.get('run', '')).splitlines() + if line.strip() and not line.strip().startswith('#')] + + def test_the_refresh_runs_between_the_install_and_the_writer(self) -> None: + refresh = self.index(lambda step: self.runs(step) == ['python util/conda_vendor.py --refresh-pins']) + install = self.index(lambda step: 'python -m pip install -e .[all]' in self.runs(step)) + writer = self.index(lambda step: 'python util/conda_vendor.py' in self.runs(step)) + changed = self.index(lambda step: str(step.get('uses', '')).startswith('tj-actions/verify-changed-files@')) + self.assertLess(install, refresh) + self.assertLess(refresh, writer) + self.assertLess(writer, changed) + for key in ('if', 'continue-on-error'): + with self.subTest(key=key): + self.assertNotIn(key, self.steps[refresh]) + + def test_the_job_installs_requests_before_the_refresh(self) -> None: + """``.[all]`` does not bring :mod:`requests`, which ``--refresh-pins`` reads PyPI with.""" + refresh = self.index(lambda step: self.runs(step) == ['python util/conda_vendor.py --refresh-pins']) + # Only the install lines are split: another step's run carries a quote across lines. + installs = [index for index, step in enumerate(self.steps) for line in self.runs(step) + if line.startswith('python -m pip install ') and 'requests' in shlex.split(line)] + self.assertTrue(installs, 'no step of conda-update installs requests') + self.assertLess(min(installs), refresh) + + def test_the_commit_takes_the_rewritten_table(self) -> None: + commit = self.index(lambda step: step.get('name') == 'Commit Changes') + self.assertIn('git commit -am', str(self.steps[commit]['run'])) + + def test_no_other_job_reads_pypi_for_the_pins(self) -> None: + for job_id, job in self.jobs.items(): + if job_id == 'conda-update': + continue + for step in job.get('steps', ()): + with self.subTest(job=job_id, step=step.get('name')): + self.assertNotIn('--refresh-pins', str(step.get('run', ''))) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/project/test_conda_vendor_1814.py b/tests/project/test_conda_vendor_1814.py index 799be6a5b..5252e3455 100644 --- a/tests/project/test_conda_vendor_1814.py +++ b/tests/project/test_conda_vendor_1814.py @@ -25,6 +25,7 @@ import bz2 import contextlib +import datetime import importlib.util import io import json @@ -252,10 +253,12 @@ def test_check_passes_on_its_own_output_and_fails_on_a_stale_file(self) -> None: metadata = dict.fromkeys((req.name for req in module.direct()), []) with tempfile.TemporaryDirectory() as temp: target = pathlib.Path(temp) / 'requirements.txt' - saved = module.FILE, module.ROOT, module._requires, module._version + saved = module.FILE, module.ROOT, module._requires, module._version, module._today module.FILE, module.ROOT = target, pathlib.Path(temp) module._requires = lambda name: metadata.get(name, []) module._version = lambda name: '1.0' + # No platform pin is overdue then; their review-by dates are #1851's tests'. + module._today = lambda: datetime.date.min try: self.assertEqual(module.main([]), 0) with contextlib.redirect_stdout(io.StringIO()) as quiet: @@ -266,7 +269,7 @@ def test_check_passes_on_its_own_output_and_fails_on_a_stale_file(self) -> None: self.assertEqual(module.main(['--check']), 1) self.assertIn('stale: scapy', report.getvalue()) finally: - module.FILE, module.ROOT, module._requires, module._version = saved + module.FILE, module.ROOT, module._requires, module._version, module._today = saved class DuplicatePinTests(unittest.TestCase): @@ -320,10 +323,12 @@ def test_check_exits_1_on_a_conflicting_duplicate_of_its_own_output(self) -> Non module = self.module with tempfile.TemporaryDirectory() as temp: target = pathlib.Path(temp) / 'requirements.txt' - saved = module.FILE, module.ROOT, module._requires, module._version + saved = module.FILE, module.ROOT, module._requires, module._version, module._today module.FILE, module.ROOT = target, pathlib.Path(temp) module._requires = lambda name: [] module._version = lambda name: '1.0' + # No platform pin is overdue then; their review-by dates are #1851's tests'. + module._today = lambda: datetime.date.min try: self.assertEqual(module.main([]), 0) written = target.read_text(encoding='utf-8') @@ -337,7 +342,7 @@ def test_check_exits_1_on_a_conflicting_duplicate_of_its_own_output(self) -> Non self.assertEqual(report.getvalue(), "requirements.txt: conflict: aenum pins '==1.0' and " "'==0.9' under the marker None\n") finally: - module.FILE, module.ROOT, module._requires, module._version = saved + module.FILE, module.ROOT, module._requires, module._version, module._today = saved #: A ``pcapkit/__init__.py`` reduced to its ``_extern`` shim. diff --git a/util/conda_vendor.py b/util/conda_vendor.py index b9d8cba2c..4240d6891 100644 --- a/util/conda_vendor.py +++ b/util/conda_vendor.py @@ -30,20 +30,31 @@ :file:`.github/workflows/cron-conda.yml` installs ``.[all]`` before running this script. Resolving the newest releases is that install step's job, and needs the network; this script reads only what is installed, so it works offline. The one -exception is :data:`PLATFORM_PINS`: a package whose newest release has no wheel -for some platform keeps an older pin for it, on a line of its own whose marker -names that platform, beside the installed pin for the rest (#1827). The conda -legs build natively, so each leg's ``pip download`` evaluates those markers on -its own runner. +exception is :data:`PLATFORM_PINS`, read from :file:`conda/platform-pins.toml`: +a package whose newest release has no wheel for some platform keeps an older pin +for it, on a line of its own whose marker names that platform, beside the +installed pin for the rest (#1827). The conda legs build natively, so each leg's +``pip download`` evaluates those markers on its own runner. + +Each of those pins carries a review-by date (#1851). With ``--refresh-pins``, +the one mode that reads the network, it asks PyPI for the newest release, up to +the installed one, with a wheel every conda Python installs on the pinned +platform (:func:`newest_wheel`). That request is the one thing here that needs +:mod:`requests`, which is imported only to make it, so no other mode needs it +installed. It moves the pin up to that release, due for +review :data:`REVIEW_HORIZON` from today, drops a pin that reaches the installed +release or already pins it, and rewrites the table. A package PyPI cannot be +read for keeps its pins, with a warning, and the run still exits 0. +``conda-update`` runs it before writing the requirements. Run without arguments it rewrites :file:`conda/requirements.txt`. With ``--check`` it writes nothing, and exits 1 when the file names a different set of packages or markers than it derives, a platform pin other than -:data:`PLATFORM_PINS`, or when that table names a package no build vendors -- -the other pins aside, since they move with every release and the weekly run -refreshes them. Lines pinning one package differently under one marker fail it -too, since pip cannot install them together when the pins disagree (#1852). -With ``--smoke PACKAGE`` it unpacks a +:data:`PLATFORM_PINS`, or when that table names a package no build vendors or +holds a pin past its review-by date -- the other pins aside, since they move with +every release and the weekly run refreshes them. Lines pinning one package +differently under one marker fail it too, since pip cannot install them together +when the pins disagree (#1852). With ``--smoke PACKAGE`` it unpacks a built ``.tar.bz2`` conda package and checks that every distribution vendored in it shipped every file its ``RECORD`` lists, and that each of its modules imports from :file:`pcapkit/_extern` with nothing else on the path. It also checks that @@ -58,21 +69,24 @@ from __future__ import annotations import argparse +import datetime import importlib.metadata import importlib.util import json import pathlib import platform +import re import subprocess # nosec B404 import sys import tarfile import tempfile import textwrap import tomllib -from typing import TYPE_CHECKING +import urllib.parse +from typing import TYPE_CHECKING, NamedTuple if TYPE_CHECKING: - from typing import Callable, Iterable, Literal, Optional + from typing import Any, Callable, Iterable, Literal, Optional from packaging.markers import Marker from packaging.requirements import Requirement @@ -94,11 +108,22 @@ 'osx-64': ('darwin', 'Darwin', 'x86_64'), 'osx-arm64': ('darwin', 'Darwin', 'arm64'), } -#: The packages pinned apart on some of :data:`PLATFORMS`, as {package: {conda -#: subdir: version}} (#1827). cryptography stopped shipping Intel macOS wheels at -#: 49.0.0, so osx-64 vendors 48.0.1, the newest release with one (universal2); -#: otherwise its build would compile cryptography from source. -PLATFORM_PINS = {'cryptography': {'osx-64': '48.0.1'}} +#: The file :data:`PLATFORM_PINS` is read from, and ``--refresh-pins`` rewrites (#1827, #1851). +PINS = ROOT / 'conda' / 'platform-pins.toml' +#: How long a pin stands before it is due for review, a quarter of a year: +#: ``--refresh-pins`` dates a pin it moves this far ahead (#1851). +REVIEW_HORIZON = datetime.timedelta(weeks=13) +#: The wheel platform tags a build on each of :data:`PLATFORMS` installs, besides +#: ``any``, for :func:`installs` (#1851): glibc's manylinux on linux-64, and on +#: macOS the runner's own architecture or universal2, for any macOS version. A +#: tag left out here only leaves a pin where it is. +WHEEL_PLATFORMS = { + 'linux-64': re.compile(r'manylinux(?:1|2010|2014|_\d+_\d+)_x86_64'), + 'osx-64': re.compile(r'macosx_\d+_\d+_(?:x86_64|universal2)'), + 'osx-arm64': re.compile(r'macosx_\d+_\d+_(?:arm64|universal2)'), +} +#: Where ``--refresh-pins`` reads a package's releases: PyPI's JSON API. +PYPI = 'https://pypi.org/pypi/{}/json' #: The object format and the machine of every native binary a package for each #: of :data:`PLATFORMS` may carry (#1822). A universal Mach-O passes when one of #: its slices is the machine. @@ -110,6 +135,88 @@ #: File names a native extension or library goes by, so one whose header this #: cannot read is reported rather than passed over. NATIVE_SUFFIXES = ('.so', '.dylib', '.pyd', '.dll') +#: What :func:`render_pins` writes above the pins, so that a rewrite keeps it. +PINS_HEADER = '''\ +# The packages conda/requirements.txt pins apart on some conda platforms, as a +# table per package and conda subdir (#1827): the version that platform vendors +# in place of the installed release, the date the pin is due for review by, and +# why the platform cannot take the installed release. +# +# util/conda_vendor.py reads this file. Conda Update's `conda-update` job runs +# its `--refresh-pins` first, which asks PyPI for the newest release, up to the +# installed one, with a wheel each conda Python installs on the pinned platform. +# It moves the pin up to that release with a fresh review-by date, and drops a +# pin that reaches the installed release or already pins it (#1851). `--check` +# fails on a pin past its review-by date: check by hand that the pin still +# holds, then set a new date. The job writes this whole file, this comment +# included, so anything a pin needs said goes in its `reason`. +''' + + +class Pin(NamedTuple): + """A version one of :data:`PLATFORMS` vendors in place of the installed release (#1827).""" + + #: The version that platform vendors. + version: str + #: The date the pin is due for review by; ``--check`` fails after it (#1851). + review_by: datetime.date + #: Why that platform cannot take the installed release. + reason: str + + +def _toml(text: 'str') -> 'str': + """``text`` as a TOML key: bare when it can be, a basic string otherwise.""" + if re.fullmatch(r'[A-Za-z0-9_-]+', text): + return text + return _toml_string(text) + + +def _toml_string(text: 'str') -> 'str': + """``text`` as a TOML basic string, with the characters TOML forbids there escaped.""" + escaped = ''.join(f'\\{char}' if char in '"\\' else + f'\\u{ord(char):04x}' if ord(char) < 0x20 or ord(char) == 0x7f else char + for char in text) + return f'"{escaped}"' + + +def load_pins(text: 'str') -> 'dict[str, dict[str, Pin]]': + """:file:`conda/platform-pins.toml`'s pins, as {package: {conda subdir: :class:`Pin`}}. + + Each pin is a table of ``version`` and ``reason``, strings, and ``review-by``, + a TOML date. Anything else raises :class:`ValueError`. + + """ + table = {} # type: dict[str, dict[str, Pin]] + for name, platforms in tomllib.loads(text).items(): + if not isinstance(platforms, dict): + raise ValueError(f'{name}: a package is a table of conda subdirs, not {platforms!r}') + for subdir, entry in platforms.items(): + fields = entry if isinstance(entry, dict) else {} + version, review_by, reason = fields.get('version'), fields.get('review-by'), fields.get('reason') + if (set(fields) != {'version', 'review-by', 'reason'} or not isinstance(version, str) + or not isinstance(reason, str) or not isinstance(review_by, datetime.date) + or isinstance(review_by, datetime.datetime)): + raise ValueError(f'{name}.{subdir}: a pin is a table of version and reason, strings, ' + f'and review-by, a date; not {entry!r}') + table.setdefault(name, {})[subdir] = Pin(version, review_by, reason) + return table + + +def render_pins(table: 'dict[str, dict[str, Pin]]') -> 'str': + """:file:`conda/platform-pins.toml` holding ``table``, which :func:`load_pins` reads back.""" + parts = [PINS_HEADER] + for name, platforms in sorted(table.items()): + for subdir, pin in sorted(platforms.items()): + parts.append(f'\n[{_toml(name)}.{_toml(subdir)}]\n' + f'version = {_toml_string(pin.version)}\n' + f'review-by = {pin.review_by.isoformat()}\n' + f'reason = {_toml_string(pin.reason)}\n') + return ''.join(parts) + + +#: The packages pinned apart on some of :data:`PLATFORMS`, as {package: {conda +#: subdir: :class:`Pin`}}, from :data:`PINS` (#1827, #1851). +PLATFORM_PINS = load_pins(PINS.read_text(encoding='utf-8')) def _load_python_versions() -> 'object': @@ -275,7 +382,7 @@ def pins(name: 'str', entry: 'dict[str, object]', """Each line ``name`` gets: its marker, and the version :data:`PLATFORM_PINS` fixes or :data:`None`.""" python = render_marker(entry['builds'], envs) # type: ignore[arg-type] # One for a platform no leg builds is unpinnable()'s to report. - fixed = {subdir: version for subdir, version in PLATFORM_PINS.get(name, {}).items() if subdir in PLATFORMS} + fixed = {subdir: pin.version for subdir, pin in PLATFORM_PINS.get(name, {}).items() if subdir in PLATFORMS} if not fixed: return [(python, None)] groups = {} # type: dict[str, set[str]] @@ -396,6 +503,154 @@ def compare(text: 'str', found: 'dict[str, dict[str, object]]', return problems +def _wheel_abis(python: 'str') -> 'frozenset[tuple[str, str]]': + """The interpreter and ABI tags CPython ``python`` installs a wheel for, as :mod:`packaging.tags` has them. + + Its own ABI, the stable ``abi3`` of its own and every older CPython, and + ``none``. The free-threaded ABI is not among them, since no conda build is. + + """ + from packaging import tags # pylint: disable=import-outside-toplevel + major, minor = map(int, python.split('.')) + interpreter = f'cp{major}{minor}' + found = [*tags.cpython_tags((major, minor), abis=[interpreter], platforms=['any']), + *tags.compatible_tags((major, minor), interpreter, platforms=['any'])] + return frozenset((tag.interpreter, tag.abi) for tag in found) + + +def installs(filename: 'str', subdir: 'str', python: 'str', requires_python: 'Optional[str]' = None) -> 'bool': + """Whether CPython ``python`` on ``subdir`` of :data:`PLATFORMS` installs the wheel ``filename`` (#1851). + + One of the wheel's tags has to carry an interpreter and ABI of + :func:`_wheel_abis` and a platform of :data:`WHEEL_PLATFORMS`, or ``any``. A + file that is not a wheel installs nowhere, nor does one whose + ``requires_python`` leaves ``python`` out, as PyPI's JSON gives it per file. + + """ + # pylint: disable-next=import-outside-toplevel + from packaging.specifiers import InvalidSpecifier, SpecifierSet + # pylint: disable-next=import-outside-toplevel + from packaging.utils import InvalidWheelFilename, parse_wheel_filename + try: + tags = parse_wheel_filename(filename)[3] + except InvalidWheelFilename: + return False + if requires_python: + try: + if f'{python}.0' not in SpecifierSet(requires_python): + return False + except InvalidSpecifier: + return False + abis, platforms = _wheel_abis(python), WHEEL_PLATFORMS[subdir] + return any((tag.interpreter, tag.abi) in abis and (tag.platform == 'any' or platforms.fullmatch(tag.platform)) + for tag in tags) + + +def newest_wheel(project: 'dict[str, Any]', subdir: 'str', pythons: 'Iterable[str]', + ceiling: 'Optional[str]' = None) -> 'Optional[str]': + """The newest release in ``project`` with a wheel each of ``pythons`` installs on ``subdir`` (#1851). + + ``project`` is PyPI's JSON for a package, whose ``releases`` map each version + to its files. A yanked file is passed over, and so are a pre-release and, + given a ``ceiling``, a release above it. :data:`None` when no release has such + a wheel; :class:`ValueError` when ``project`` lists no releases at all. + + """ + from packaging.version import InvalidVersion, Version # pylint: disable=import-outside-toplevel + releases = project.get('releases') if isinstance(project, dict) else None + if not isinstance(releases, dict): + raise ValueError("PyPI's JSON has no releases") + pythons = list(pythons) + top = None if ceiling is None else Version(ceiling) + best, found = None, None # type: Optional[Version], Optional[str] + for text, files in releases.items(): + try: + version = Version(text) + except InvalidVersion: + continue + if version.is_prerelease or (top is not None and version > top) or (best is not None and version <= best): + continue + wheels = [file for file in files or () if isinstance(file, dict) and not file.get('yanked') + and str(file.get('filename', '')).endswith('.whl')] + if pythons and all(any(installs(file['filename'], subdir, python, file.get('requires_python')) + for file in wheels) for python in pythons): + best, found = version, text + return found + + +#: What :func:`_fetch` raises when PyPI cannot be read: :class:`OSError` for a +#: failed request, and :class:`ValueError` for a reply that is not JSON, as +#: requests releases before its own ``JSONDecodeError`` raise it. +UNREADABLE = (OSError, ValueError) + + +def refresh(table: 'dict[str, dict[str, Pin]]', fetch: 'Callable[[str], dict[str, Any]]', + installed: 'Callable[[str], Optional[str]]', pythons: 'Iterable[str]', + today: 'datetime.date') -> 'tuple[dict[str, dict[str, Pin]], list[tuple[str, str]]]': + """``table`` with each pin moved up as far as PyPI has a wheel for its platform, and notes on it (#1851). + + ``fetch`` returns a package's PyPI JSON, raising one of :data:`UNREADABLE` + when it cannot, and ``installed`` the version installed here, or + :data:`None`. A pin moves to :func:`newest_wheel` up to the installed + version, due for review on ``today`` plus :data:`REVIEW_HORIZON`. A pin that + reaches the installed version that way, or already pins it, is dropped, as + the other platforms vendor that version. A package or pin it cannot settle + stays as it is. Each note is a + workflow command, ``notice`` for a change and ``warning`` for a pin left + alone, and its text. + + """ + from packaging.version import Version # pylint: disable=import-outside-toplevel + pythons = list(pythons) + result = {} # type: dict[str, dict[str, Pin]] + notes = [] # type: list[tuple[str, str]] + for name, platforms in sorted(table.items()): + kept = result[name] = dict(platforms) + have = installed(name) + if have is None: + notes.append(('warning', f'{name} is not installed here, so its pins stay as they are')) + continue + try: + project = fetch(name) + except UNREADABLE as error: + notes.append(('warning', f'PyPI could not be read for {name} ({error}), so its pins stay as they are')) + continue + for subdir, pin in sorted(platforms.items()): + if subdir not in WHEEL_PLATFORMS: + notes.append(('warning', f'{name} is pinned for {subdir}, which no conda leg builds, ' + 'so that pin stays as it is')) + continue + try: + newest = newest_wheel(project, subdir, pythons, ceiling=have) + current, ceiling = Version(pin.version), Version(have) + target = None if newest is None or Version(newest) <= current else Version(newest) + except ValueError as error: + notes.append(('warning', f'{name} {pin.version} on {subdir} stays as it is: {error}')) + continue + if current == ceiling: + del kept[subdir] + notes.append(('notice', f'{name} {pin.version} on {subdir} is the installed release itself, ' + 'so the pin is dropped')) + elif newest is not None and target == ceiling: + del kept[subdir] + notes.append(('notice', f'{name} {have}, the installed release, has a wheel for {subdir}, ' + f'so its {pin.version} pin there is dropped')) + elif newest is not None and target is not None: + kept[subdir] = pin._replace(version=newest, review_by=today + REVIEW_HORIZON) + notes.append(('notice', f'{name} on {subdir} moves from {pin.version} to {newest}, the newest ' + f'release with a wheel there; due for review by {kept[subdir].review_by}')) + return {name: platforms for name, platforms in result.items() if platforms}, notes + + +def review(table: 'dict[str, dict[str, Pin]]', today: 'datetime.date') -> 'list[str]': + """Each pin of ``table`` past its review-by date on ``today``, one line apiece (#1851).""" + return [f'overdue: {name} is pinned to {pin.version} on {subdir}, and was due for review by ' + f'{pin.review_by}; check the pin still holds, then set a new review-by date, ' + f'such as {today + REVIEW_HORIZON}' + for name, platforms in sorted(table.items()) for subdir, pin in sorted(platforms.items()) + if pin.review_by < today] + + def _requires(name: 'str') -> 'Optional[list[str]]': return importlib.metadata.requires(name) @@ -404,6 +659,55 @@ def _version(name: 'str') -> 'str': return importlib.metadata.version(name) +def _installed(name: 'str') -> 'Optional[str]': + try: + return importlib.metadata.version(name) + except importlib.metadata.PackageNotFoundError: + return None + + +def _fetch(name: 'str') -> 'dict[str, Any]': + """``name``'s PyPI JSON, the one request this script makes, for ``--refresh-pins``. + + :mod:`requests` is imported here, not with the module, so that the other + modes run on the standard library alone. Any :class:`requests.RequestException`, + an HTTP error status or a reply that is not JSON included, is raised as + :class:`OSError`, one of :data:`UNREADABLE`, whatever requests derives it from. + + """ + import requests # pylint: disable=import-outside-toplevel + try: + response = requests.get(PYPI.format(urllib.parse.quote(name)), timeout=60, + headers={'User-Agent': 'PyPCAPKit conda-update (#1851)', + 'Accept': 'application/json'}) + response.raise_for_status() + return response.json() + except requests.RequestException as error: + raise OSError(str(error)) from error + + +def _today() -> 'datetime.date': + """Today's date in UTC, the runners' time zone.""" + return datetime.datetime.now(datetime.timezone.utc).date() + + +def _escape(text: 'str') -> 'str': + """``text`` as a workflow command's data.""" + return text.replace('%', '%25').replace('\r', '%0D').replace('\n', '%0A') + + +def refresh_pins() -> 'int': + """``--refresh-pins``: :func:`refresh` :data:`PLATFORM_PINS`, report it, and rewrite :data:`PINS` on a change.""" + today = _today() + table, notes = refresh(PLATFORM_PINS, _fetch, _installed, conda_pythons(), today) + notes.extend(('warning', problem) for problem in review(table, today)) + for level, text in notes: + print(f'::{level} title=Platform pins::{_escape(text)}') + if table != PLATFORM_PINS: + PINS.write_text(render_pins(table), encoding='utf-8') + return 0 + + #: Run with ``-I -S`` and the unpacked ``site-packages`` as its argument, so that #: nothing outside the package can satisfy an import. PROBE = textwrap.dedent('''\ @@ -565,10 +869,15 @@ def main(argv: 'Optional[list[str]]' = None) -> 'int': help='write nothing; exit 1 when conda/requirements.txt is stale') group.add_argument('--smoke', metavar='PACKAGE', type=pathlib.Path, help='check the vendored packages of a built .tar.bz2 conda package') + group.add_argument('--refresh-pins', action='store_true', + help='move each platform pin up as far as PyPI has a wheel for its platform, ' + 'and rewrite conda/platform-pins.toml; reads the network') args = parser.parse_args(argv) if args.smoke is not None: return smoke(args.smoke) + if args.refresh_pins: + return refresh_pins() envs = environments(conda_pythons()) try: @@ -578,9 +887,11 @@ def main(argv: 'Optional[list[str]]' = None) -> 'int': return 2 if args.check: - problems = compare(FILE.read_text(encoding='utf-8'), found, envs) + problems = [f'{FILE.relative_to(ROOT)}: {problem}' + for problem in compare(FILE.read_text(encoding='utf-8'), found, envs)] + problems.extend(f'{PINS.relative_to(ROOT)}: {problem}' for problem in review(PLATFORM_PINS, _today())) for problem in problems: - print(f'{FILE.relative_to(ROOT)}: {problem}') + print(problem) return 1 if problems else 0 problems = unpinnable(found) diff --git a/util/verdict_covers_checkout.py b/util/verdict_covers_checkout.py index 94f472319..6011e8db0 100644 --- a/util/verdict_covers_checkout.py +++ b/util/verdict_covers_checkout.py @@ -56,11 +56,17 @@ #: Files a bot commit may touch and still be built under its parent's verdict. #: These are what Conda Update ("Bumped build to N") and Create Release's -#: ``tag`` job ("Reset conda build number") commit, and no test reads either. +#: ``tag`` job ("Reset conda build number") commit. No test reads the committed +#: ``conda/build``; the #1814 and #1827 tests do read ``conda/requirements.txt``, +#: and a bot commit to it is still built under its parent's verdict. #: Vendor Update's "Bumped version to N" commits are left out on purpose: every #: one of them rewrites ``pcapkit/``, and e58618bdf's Unit Tests run #: 36355214959 reported "12 failed" on each Python leg, where its parent -#: 70fa92010's run 36354135989 passed every job. +#: 70fa92010's run 36354135989 passed every job. ``conda/platform-pins.toml`` is +#: left out too, though a Conda Update commit that moves a platform pin touches +#: it (#1851): util/conda_vendor.py reads it on import, so every test of that +#: script does, and a moved pin changes what those tests expect of +#: ``conda/requirements.txt``. BOT_MANAGED_PATHS = frozenset({'conda/build', 'conda/requirements.txt'}) #: A full commit sha, which is all ``head_sha`` ever holds for a push run.