diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index 144887ea36..ddd0f3497c 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -1221,10 +1221,10 @@ jobs: - ubuntu-latest # - windows-latest # skip for now # #1633: the supported tier only, from .github/python-versions.toml - # through ``version_check`` (#1660). There is no KTLO 3.10 leg because it - # can only fail: conda/requirements.txt pins soupsieve==3.0.3 - # (Requires-Python >=3.11.5), and util/conda-dist.py pins the newest - # release of every dependency, so that floor only rises. + # through ``version_check`` (#1660). There is no KTLO 3.10 leg: the list + # util/conda_vendor.py derives is for the supported tier, so it leaves + # out typing-extensions, which pcapkit/utilities/compat.py imports below + # 3.11, and pins the newest release of each dependency, a rising floor. python-version: ${{ fromJSON(needs.version_check.outputs.PYTHON_CONDA) }} steps: # #1634: this run's commit, not ``ref: conda-+0``, then the check diff --git a/.github/workflows/cron-conda.yml b/.github/workflows/cron-conda.yml index e2625b7215..1cfa231b04 100644 --- a/.github/workflows/cron-conda.yml +++ b/.github/workflows/cron-conda.yml @@ -156,28 +156,25 @@ jobs: with: python-version: ${{ steps.python-versions.outputs.latest }} - # `.[all,dev]`, not `.[all]` alone: `util/conda-dist.py` below only - # *refreshes* the version pin of every package already named in - # `conda/requirements.txt` -- via `importlib.metadata.version(name)`, - # which raises unless that distribution is installed here -- and that - # file names `dpkt`, `scapy`, `pyshark`, `requests` and `beautifulsoup4` - # with no marker. #910 narrowed `all` to core addons only, so those five - # come from `dev` instead; without it this step would install - # cleanly and the next one would crash on the first missing name. + # `.[all]`, which is what `util/conda_vendor.py` below vendors (#1814): + # it derives `conda/requirements.txt` from `dependencies` and the `all` + # extra, walks what they require through the installed metadata, and + # pins each package at the version installed here. So this install is + # the step that resolves the newest releases, and the script itself + # reads nothing from the network. - name: Install and Setup run: | set -x python -m pip install -U pip setuptools wheel packaging - python -m pip install -e .[all,dev] - #python -m pip install pathlib2 typing_extensions + python -m pip install -e .[all] - name: Setup Conda Distribution run: | export PCAPKIT_DEVMODE=1 export PCAPKIT_VERBOSE=1 - python util/conda-dist.py + python util/conda_vendor.py - name: Verify Changed files uses: tj-actions/verify-changed-files@v17 @@ -452,14 +449,12 @@ jobs: python -m pip install -U pip setuptools wheel python -m pip install -e .[all] - python -m pip install pathlib2 typing_extensions - name: Install and Setup (Windows) if: ${{ matrix.os == 'windows-latest' }} run: | python -m pip install -U pip setuptools wheel python -m pip install -e .[all] - python -m pip install pathlib2 typing_extensions - name: Setup Conda Distribution if: ${{ matrix.os != 'windows-latest' }} @@ -530,6 +525,8 @@ jobs: ;; esac + echo "host_package=$host_package" >> "$GITHUB_OUTPUT" + host_platform="$(basename "$(dirname "$host_package")")" if [ -n "$target_platform" ] && [ "$target_platform" != "$host_platform" ]; then conda convert "$host_package" -o "$out_dir" -p "$target_platform" @@ -550,6 +547,20 @@ jobs: echo EOF } >> "$GITHUB_OUTPUT" + # #1814: the package declares no run requirements, so the packages it + # vendors in `pcapkit/_extern` are all it has, and they once shipped as + # modules alone. This unpacks the package built for this runner and + # checks that every vendored distribution has every file its RECORD lists + # and imports from there, with no site-packages on the path, before + # anything is uploaded. setup-python's interpreter, not conda's: it is + # the matrix version the package was built for, which conda's base is + # not on 3.15. + - name: Smoke-test the vendored packages + env: + PCAPKIT_CONDA_PACKAGE: ${{ steps.conda_build.outputs.host_package }} + run: | + "${pythonLocation}/bin/python" util/conda_vendor.py --smoke "$PCAPKIT_CONDA_PACKAGE" + - name: Upload conda packages to Anaconda uses: anaconda/actions/upload-package@v0.3.1 with: diff --git a/MANIFEST.in b/MANIFEST.in index efb018c1b8..96a45939be 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -87,6 +87,7 @@ recursive-include docs/source/changelog *.rst # Nothing here names ``pcapkit/_extern``. ``conda/pypcapkit/build.sh`` and # ``bld.bat`` create it and then run ``pip install .``, which builds the wheel # straight from the tree, so the conda recipe builds no sdist; its modules -# reach that wheel through package discovery. A line naming it would warn in -# every other build, and ``include`` of a directory matched nothing even in the -# conda one (#1794). +# reach that wheel through package discovery, and every other file under it +# through the ``package-data`` entry in ``pyproject.toml`` (#1814). A line naming +# it would warn in every other build, and ``include`` of a directory matched +# nothing even in the conda one (#1794). diff --git a/conda/pypcapkit/meta.yaml b/conda/pypcapkit/meta.yaml index 94fd633317..8bdf8f441f 100644 --- a/conda/pypcapkit/meta.yaml +++ b/conda/pypcapkit/meta.yaml @@ -24,7 +24,7 @@ requirements: - git - python - pip - - setuptools>=61.0.0 + - setuptools>=62.3.0 # # version compatibility # - bpc-f2format @@ -49,7 +49,7 @@ requirements: host: - python - pip - - setuptools>=61.0.0 + - setuptools>=62.3.0 # # version compatibility # - bpc-f2format diff --git a/conda/requirements.txt b/conda/requirements.txt index d8edec9359..54c58ae182 100644 --- a/conda/requirements.txt +++ b/conda/requirements.txt @@ -1,25 +1,9 @@ aenum==3.1.17 -appdirs==1.4.4 -beautifulsoup4==4.15.0 -certifi==2026.7.22 +cffi==2.1.1 chardet==7.6.0 -charset-normalizer==3.5.2 +cryptography==50.0.2 dictdumper==0.8.4.post6 -dpkt==1.9.8 emoji==2.16.0 -html5lib==1.1 -idna==3.20 -lxml==6.1.3 -packaging==26.3 -pathlib2; python_version < "3.5" -pyshark==0.6 -pysocks==1.7.1 -requests==2.34.2 -scapy==2.8.0 -six==1.17.0 -soupsieve==3.0.3 +pycparser==3.11 +pycrate==0.8.1 tbtrim==0.3.1 -termcolor==3.3.0 -typing-extensions==4.16.0; python_version < "3.11" -urllib3==2.8.0 -webencodings==0.6.1 diff --git a/pyproject.toml b/pyproject.toml index 6a20fc2d09..b05057359f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,10 @@ [build-system] requires = [ - "setuptools>=61.0.0", + # 62.3.0 is the first release whose ``package-data`` globs read ``**`` as any + # depth, which the ``pcapkit._extern`` entry below needs (#1814). Measured: + # 61.0.0 and 62.2.0 read it as ``*``, and ship only the files one directory + # down from it. + "setuptools>=62.3.0", # version compatibility "bpc-f2format; python_version < '3.6'", @@ -50,7 +54,7 @@ license = { text="BSD 3-Clause License" } # blocking, and a failure there is a warning on a green leg (#1672). Each 3.7 leg # fails a step by design: the import fails as above, and the ``test`` extra's # ``pytest>=8`` needs 3.8. 3.6 is not run at all: setup-python -# builds it only for retired Ubuntu images, and the build needs ``setuptools>=61`` +# builds it only for retired Ubuntu images, and the build needs ``setuptools>=62.3`` # besides. The ``PCAP_CT`` extra needs 3.10 regardless. requires-python = ">=3.6, <4" description = "PyPCAPKit: comprehensive network packet analysis library" @@ -368,6 +372,21 @@ exclude = [ "sample*", ] +# Only the conda recipe creates ``pcapkit/_extern``: ``conda/pypcapkit/build.sh`` +# and ``bld.bat`` install the packages ``conda/requirements.txt`` names there, +# then run ``pip install .``, which builds the wheel from the tree. Package +# discovery reaches their modules alone, so every other file -- ``emoji``'s JSON, +# ``chardet``'s models, compiled extensions, each ``dist-info`` -- stays behind +# without this entry (#1814). Elsewhere ``pcapkit._extern`` is no package at all, +# and setuptools ignores these entries without a warning; ``MANIFEST.in`` could not +# do this, since a line naming a missing directory warns in every such build. The +# bytecode ``pip install --target`` compiles stays out, as from any other wheel. +[tool.setuptools.package-data] +"pcapkit._extern" = [ "**/*" ] + +[tool.setuptools.exclude-package-data] +"pcapkit._extern" = [ "*.pyc" ] + [tool.setuptools.dynamic] version = { attr="pcapkit.__version__" } diff --git a/tests/project/test_conda_vendor_1814.py b/tests/project/test_conda_vendor_1814.py new file mode 100644 index 0000000000..f60e1549e2 --- /dev/null +++ b/tests/project/test_conda_vendor_1814.py @@ -0,0 +1,326 @@ +# -*- coding: utf-8 -*- +"""Pins for the conda distribution's vendored packages (#1814). + +* Every file under :file:`pcapkit/_extern` reaches the wheel, not just the + modules package discovery finds, with no bytecode; a tree without the + directory builds with no warning about it. Measured by building a wheel of a + stub package carrying :file:`pyproject.toml`'s own ``[tool.setuptools]`` tables. +* :file:`util/conda_vendor.py` derives :file:`conda/requirements.txt` from + ``dependencies`` and the ``all`` extra, walks what they require, drops a + requirement no conda build can need, and its ``--check`` passes on the list it + writes and fails on a stale one. These run on stand-in metadata, so they need + neither the network nor any vendored package installed. +* The committed list holds every requirement :file:`pyproject.toml` makes live on + a conda build, with its marker, and none that is dead there. +* ``--smoke`` passes on a conda package whose vendored distributions are complete + and fails on one missing a data file. + +""" + +from __future__ import annotations + +import bz2 +import contextlib +import importlib.util +import io +import json +import pathlib +import subprocess # nosec: B404 +import sys +import tarfile +import tempfile +import textwrap +import unittest +import zipfile +from typing import TYPE_CHECKING + +try: + import tomllib +except ImportError: # pragma: no cover + tomllib = None # type: ignore[assignment] + +if TYPE_CHECKING: + from typing import Any, Optional + +ROOT = pathlib.Path(__file__).resolve().parents[2] +SCRIPT = ROOT / 'util' / 'conda_vendor.py' +NO_TOMLLIB = 'tomllib, which reads pyproject.toml and .github/python-versions.toml, needs Python 3.11+' + +_MODULE = [] # type: list[Any] + + +def vendor() -> 'Any': + """:file:`util/conda_vendor.py`, loaded once.""" + if tomllib is None: # pragma: no cover + raise unittest.SkipTest(NO_TOMLLIB) + if not _MODULE: + spec = importlib.util.spec_from_file_location('conda_vendor', SCRIPT) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) # type: ignore[union-attr] + _MODULE.append(module) + return _MODULE[0] + + +def toml_tables(prefix: 'str', table: 'dict[str, Any]') -> 'str': + """``table`` as TOML: strings, booleans and lists of strings, nested tables last.""" + plain = {key: value for key, value in table.items() if not isinstance(value, dict)} + out = [f'[{prefix}]'] + [f'{json.dumps(key)} = {json.dumps(value)}' for key, value in plain.items()] + for key, value in table.items(): + if isinstance(value, dict): + out.append(toml_tables(f'{prefix}.{json.dumps(key)}', value)) + return '\n'.join(out) + '\n' + + +#: Builds a wheel through setuptools' PEP 517 hook, in the stub's directory, and +#: prints its name. +BUILD = 'import sys; from setuptools import build_meta; print(build_meta.build_wheel(sys.argv[1]))' + + +class PackageDataTests(unittest.TestCase): + """``pip install .`` in the conda recipe ships the whole of :file:`pcapkit/_extern`.""" + + def build(self, extern: 'Optional[dict[str, bytes]]') -> 'tuple[list[str], str]': + """The stub's wheel listing and build log, with ``extern`` under ``pcapkit/_extern``.""" + if tomllib is None: # pragma: no cover + raise unittest.SkipTest(NO_TOMLLIB) + config = tomllib.loads((ROOT / 'pyproject.toml').read_text(encoding='utf-8')) + setuptools = {key: value for key, value in config['tool']['setuptools'].items() + if key != 'dynamic'} + with tempfile.TemporaryDirectory() as temp: + stub = pathlib.Path(temp) / 'stub' + (stub / 'pcapkit').mkdir(parents=True) + (stub / 'pcapkit' / '__init__.py').write_text('', encoding='utf-8') + (stub / 'pyproject.toml').write_text( + '[project]\nname = "stub"\nversion = "0"\n\n' + toml_tables('tool.setuptools', setuptools), + encoding='utf-8') + for name, data in (extern or {}).items(): + path = stub / 'pcapkit' / '_extern' / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + out = pathlib.Path(temp) / 'dist' + out.mkdir() + result = subprocess.run([sys.executable, '-c', BUILD, str(out)], cwd=stub, # nosec + capture_output=True, text=True, check=False, timeout=300) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + wheel = out / result.stdout.strip().splitlines()[-1] + with zipfile.ZipFile(wheel) as archive: + names = archive.namelist() + return names, result.stdout + result.stderr + + def test_every_vendored_file_ships(self) -> None: + extern = { + 'certifi/__init__.py': b'', + 'certifi/cacert.pem': b'-----BEGIN CERTIFICATE-----\n', + 'certifi/__pycache__/core.cpython-314.pyc': b'\0', + 'certifi-1.0.dist-info/METADATA': b'Name: certifi\n', + 'certifi-1.0.dist-info/RECORD': b'certifi/cacert.pem,,\n', + 'emoji/unicode_codes/emoji.json': b'{}', + '7cf47097__mypyc.cpython-314-x86_64-linux-gnu.so': b'\0', + 'six.py': b'', + '__pycache__/six.cpython-314.pyc': b'\0', + } + names, _ = self.build(extern) + shipped = sorted(name[len('pcapkit/_extern/'):] for name in names + if name.startswith('pcapkit/_extern/')) + self.assertEqual(shipped, sorted(name for name in extern if not name.endswith('.pyc'))) + + def test_a_tree_without_extern_ships_none_of_it_and_warns_nothing(self) -> None: + names, log = self.build(None) + self.assertEqual(sorted(name for name in names if not name.startswith('stub-0.dist-info/')), + ['pcapkit/__init__.py']) + self.assertNotRegex(log, r'(?i)warn.*_extern|_extern.*warn') + + +def requirement_lines(*lines: 'str') -> 'str': + return ''.join(f'{line}\n' for line in lines) + + +#: Stand-in ``Requires-Dist`` metadata for :func:`conda_vendor.derive`. +METADATA = { + 'core': ['helper>=1', 'oldshim; python_version < "3.8"', 'docsy; extra == "docs"'], + 'helper': None, + 'crypt': ['ffi; platform_python_implementation != "PyPy"', 'socksy; extra == "socks"'], + 'ffi': ['parser; implementation_name != "PyPy"'], + 'parser': [], + 'socksy': [], + 'newer': [], +} +VERSIONS = {name: '1.0' for name in METADATA} + + +def requires(name: 'str') -> 'Optional[list[str]]': + if name not in METADATA: + raise KeyError(name) + return METADATA[name] + + +class DeriveTests(unittest.TestCase): + """:mod:`conda_vendor` classifies and writes the vendored set.""" + + PYTHONS = ['3.11', '3.12', '3.13'] + + def found(self, *direct: 'str') -> 'dict[str, dict[str, Any]]': + module = vendor() + from packaging.requirements import Requirement + return module.derive([Requirement(line) for line in direct], + module.environments(self.PYTHONS), requires) + + def test_classes(self) -> None: + found = self.found('core', 'crypt[socks]', 'backport; python_version < "3.11"', + 'newer; python_version >= "3.12"') + self.assertEqual({name: entry['class'] for name, entry in found.items()}, + {'core': 'needed', 'helper': 'transitive', 'crypt': 'needed', + 'ffi': 'transitive', 'parser': 'transitive', 'socksy': 'transitive', + 'backport': 'dead', 'newer': 'needed'}) + + def test_lines_carry_a_marker_only_where_some_builds_need_none(self) -> None: + module = vendor() + envs = module.environments(self.PYTHONS) + found = self.found('core', 'backport; python_version < "3.11"', 'newer; python_version >= "3.12"') + self.assertEqual(module.lines(found, envs, VERSIONS.__getitem__), + ['core==1.0', 'helper==1.0', + 'newer==1.0; python_version == "3.12" or python_version == "3.13"']) + + def test_a_live_package_not_installed_here_is_named_with_its_builds(self) -> None: + module = vendor() + import importlib.metadata # 3.8+, and vendor() has skipped below 3.11 + from packaging.requirements import Requirement + here = f'{sys.version_info.major}.{sys.version_info.minor}' + + def lacking(name: 'str') -> 'list[str]': + if name == 'shim': + raise importlib.metadata.PackageNotFoundError(name) + return {'core': ['shim; python_version < "3.2"']}.get(name, []) + + # Python 3.1 stands for a build this interpreter is not, as 3.11 is for + # Conda Update's newest Python. + with self.assertRaises(module.NotInstalled) as elsewhere: + module.derive([Requirement('core')], module.environments(['3.1', '3.2']), lacking) + self.assertIn('shim is needed on Python 3.1 on darwin, linux (core requires shim; ' + 'python_version < "3.2")', str(elsewhere.exception)) + self.assertIn(f'and Python {here} on {sys.platform}, which runs this, does not have it: ' + 'its marker leaves it out here', str(elsewhere.exception)) + with self.assertRaises(module.NotInstalled) as local: + module.derive([Requirement('shim')], module.environments([here]), lacking) + self.assertIn('(pyproject.toml asks for shim)', str(local.exception)) + self.assertTrue(str(local.exception).endswith('install .[all] first'), str(local.exception)) + + def test_compare_passes_on_written_lines_and_names_each_stale_one(self) -> None: + module = vendor() + envs = module.environments(self.PYTHONS) + found = self.found('core', 'crypt', 'backport; python_version < "3.11"') + written = requirement_lines(*module.lines(found, envs, VERSIONS.__getitem__)) + self.assertEqual(module.compare(written, found, envs), []) + repinned = written.replace('core==1.0', 'core==9.9') + self.assertEqual(module.compare(repinned, found, envs), []) + stale = requirement_lines('backport==1.0; python_version < "3.11"', 'core==1.0', + 'crypt==1.0', 'ffi==1.0', 'helper==1.0', 'lxml==6.1.3') + problems = module.compare(stale, found, envs) + self.assertEqual([problem.split(':')[0] + ':' + problem.split()[1] for problem in problems], + ['dead:backport', 'stale:lxml', 'missing:parser']) + + +class CommittedListTests(unittest.TestCase): + """:file:`conda/requirements.txt` against :file:`pyproject.toml`, offline.""" + + def test_every_live_direct_requirement_is_listed_and_no_dead_one(self) -> None: + module = vendor() + from packaging.markers import Marker + envs = module.environments(module.conda_pythons()) + have = module.parse((ROOT / 'conda' / 'requirements.txt').read_text(encoding='utf-8')) + for found in module.derive(module.direct(), envs, lambda name: []).items(): + name, entry = found + with self.subTest(requirement=name): + if entry['class'] == 'dead': + self.assertNotIn(name, have) + continue + self.assertIn(name, have) + marker = module.render_marker(entry['builds'], envs) + self.assertEqual(have[name], None if marker is None else str(Marker(marker))) + + def test_check_passes_on_its_own_output_and_fails_on_a_stale_file(self) -> None: + module = vendor() + metadata = dict.fromkeys((req.name for req in module.direct()), []) + with tempfile.TemporaryDirectory() as temp: + target = pathlib.Path(temp) / 'requirements.txt' + saved = module.FILE, module.ROOT, module._requires, module._version + module.FILE, module.ROOT = target, pathlib.Path(temp) + module._requires = lambda name: metadata.get(name, []) + module._version = lambda name: '1.0' + try: + self.assertEqual(module.main([]), 0) + with contextlib.redirect_stdout(io.StringIO()) as quiet: + self.assertEqual(module.main(['--check']), 0) + self.assertEqual(quiet.getvalue(), '') + target.write_text(target.read_text(encoding='utf-8') + 'scapy==2.8.0\n', encoding='utf-8') + with contextlib.redirect_stdout(io.StringIO()) as report: + self.assertEqual(module.main(['--check']), 1) + self.assertIn('stale: scapy', report.getvalue()) + finally: + module.FILE, module.ROOT, module._requires, module._version = saved + + +#: A ``pcapkit/__init__.py`` reduced to its ``_extern`` shim. +SHIM = textwrap.dedent('''\ + import os, sys + _extern = os.path.join(os.path.dirname(os.path.realpath(__file__)), '_extern') + if os.path.exists(_extern): + sys.path.append(_extern) +''') + + +class SmokeTests(unittest.TestCase): + """``--smoke`` reads a built conda package's vendored distributions.""" + + def package(self, temp: 'str', files: 'dict[str, str]') -> 'pathlib.Path': + site = f'lib/python{sys.version_info.major}.{sys.version_info.minor}/site-packages/' + path = pathlib.Path(temp) / 'pypcapkit-0-py_0.tar.bz2' + buffer = io.BytesIO() + with tarfile.open(fileobj=buffer, mode='w') as archive: + for name, text in dict({'pcapkit/__init__.py': SHIM}, **files).items(): + data = text.encode('utf-8') + info = tarfile.TarInfo(site + name) + info.size = len(data) + archive.addfile(info, io.BytesIO(data)) + path.write_bytes(bz2.compress(buffer.getvalue())) + return path + + def smoke(self, files: 'dict[str, str]') -> 'tuple[int, str]': + with tempfile.TemporaryDirectory() as temp: + package = self.package(temp, files) + result = subprocess.run([sys.executable, str(SCRIPT), '--smoke', str(package)], # nosec + capture_output=True, text=True, check=False, timeout=120) + return result.returncode, result.stdout + result.stderr + + #: ``bundle`` reads its data on first use, as chardet reads its models, so a + #: missing data file leaves the import working and only ``RECORD`` notices. + FILES = { + 'pcapkit/_extern/bundle/__init__.py': + "import os\n\ndef where():\n return open(os.path.join(os.path.dirname(__file__), 'data.pem')).read()\n", + 'pcapkit/_extern/bundle/data.pem': 'pem\n', + 'pcapkit/_extern/bundle-1.0.dist-info/METADATA': 'Name: bundle\n', + 'pcapkit/_extern/bundle-1.0.dist-info/RECORD': + 'bundle/__init__.py,,\nbundle/data.pem,,\nbundle-1.0.dist-info/METADATA,,\n' + 'bundle-1.0.dist-info/RECORD,,\n../../bin/bundle,,\n', + } + + def test_a_complete_package_passes(self) -> None: + if tomllib is None: # pragma: no cover + raise unittest.SkipTest(NO_TOMLLIB) + status, log = self.smoke(self.FILES) + self.assertEqual(status, 0, log) + self.assertIn('bundle: imported from pcapkit/_extern', log) + + def test_a_missing_data_file_fails(self) -> None: + if tomllib is None: # pragma: no cover + raise unittest.SkipTest(NO_TOMLLIB) + files = {name: text for name, text in self.FILES.items() if not name.endswith('data.pem')} + status, log = self.smoke(files) + self.assertEqual(status, 1, log) + self.assertIn('MISSING bundle/data.pem', log) + self.assertIn('bundle: imported from pcapkit/_extern', log) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/project/test_python_matrix_tiers_1633.py b/tests/project/test_python_matrix_tiers_1633.py index 2f9fed0760..ed635730ba 100644 --- a/tests/project/test_python_matrix_tiers_1633.py +++ b/tests/project/test_python_matrix_tiers_1633.py @@ -439,7 +439,7 @@ def test_release_status_reports_a_failed_ktlo_leg_without_warning(self) -> None: self.assertNotIn('KTLO', clean.stdout) def test_no_conda_leg_is_ktlo_and_315_leaves_base_python_alone(self) -> None: - """A KTLO 3.10 conda leg could only fail (soupsieve 3 needs >=3.11.5); 3.15 needs base left alone.""" + """No KTLO 3.10 conda leg: the vendored set is derived for the supported tier; 3.15 needs base left alone.""" for name, job_id in (('create-release.yml', 'conda'), ('cron-conda.yml', 'conda-dist')): spec = job(name, job_id) with self.subTest(workflow=name, job=job_id): diff --git a/util/conda-dist.py b/util/conda-dist.py deleted file mode 100644 index 2c1d1126f7..0000000000 --- a/util/conda-dist.py +++ /dev/null @@ -1,41 +0,0 @@ -# -*- coding: utf-8 -*- - -import importlib.metadata as imp_meta -import os.path as os_path -import re - -import packaging.requirements as pkg_req - -req_file = os_path.join('conda', 'requirements.txt') -version_spec_re = re.compile(r'\s*(===|~=|!=|==|<=|>=|<|>)\s*') - - -def normalise_requirement(line: str) -> str: - requirement, sep, marker = line.strip().partition(';') - requirement = version_spec_re.sub(r'\1', requirement) - if sep: - return f'{requirement}; {marker.strip()}' - return requirement - - -data = {} # dict[str, str] -with open(req_file) as file: - for line in file: - if not line.strip(): - continue - req = pkg_req.Requirement(normalise_requirement(line)) - - try: - ver = imp_meta.version(req.name) - if req.marker is None: - data[req.name] = f'{req.name}=={ver}' - else: - data[req.name] = f'{req.name}=={ver}; {req.marker}' - except imp_meta.PackageNotFoundError: - if req.marker is None: - raise - data[req.name] = f'{req.name}; {req.marker}' - -with open(req_file, 'w') as file: - for key, line in sorted(data.items()): - print(line, file=file) diff --git a/util/conda_vendor.py b/util/conda_vendor.py new file mode 100644 index 0000000000..f292b9f4e7 --- /dev/null +++ b/util/conda_vendor.py @@ -0,0 +1,389 @@ +# -*- coding: utf-8 -*- +"""The third-party packages the conda recipe vendors into :file:`pcapkit/_extern` (#1814). + +The conda package declares no run requirements. ``conda/pypcapkit/build.sh`` and +``bld.bat`` install every wheel that ``pip download -r conda/requirements.txt`` +fetched into :file:`pcapkit/_extern`, which :file:`pcapkit/__init__.py` appends +to :data:`sys.path`. So :file:`conda/requirements.txt` is the conda +distribution's whole dependency list, and this script derives it from +:file:`pyproject.toml` instead of leaving it to be edited by hand. Each package +falls in exactly one of these classes: + +``needed`` + ``[project] dependencies`` and the ``all`` extra. A conda user has no + extras to ask for, and ``all`` is the set the owner ruled on #910 to be what + the library needs to work at full functionality. The capture engines and the + crawlers' ``vendor`` extra are installed on demand there, so a conda user + installs them into the environment, where they take precedence over + anything vendored. +``transitive`` + What a needed package requires in turn, with the extras it was asked for, + as the installed distributions' metadata declares it. +``dead`` + A requirement whose marker is false on every interpreter a conda package is + built for: the supported CPython tier of :file:`.github/python-versions.toml`, + on the platforms in :data:`PLATFORMS`. It is left out. +``stale`` + Anything else the file names. + +Each line pins the version installed here, as ``conda-update`` in +:file:`.github/workflows/cron-conda.yml` installs ``.[all]`` before running this +script. Resolving the newest releases is that install step's job, and needs the +network; this script reads only what is installed, so it works offline. + +Run without arguments it rewrites :file:`conda/requirements.txt`. With +``--check`` it writes nothing, and exits 1 when the file names a different set of +packages or markers than it derives -- the pins aside, since they move with every +release and the weekly run refreshes them. With ``--smoke PACKAGE`` it unpacks a +built ``.tar.bz2`` conda package and checks that every distribution vendored in +it shipped every file its ``RECORD`` lists, and that each of its modules imports +from :file:`pcapkit/_extern` with nothing else on the path; that mode needs only +the standard library. + +""" + +from __future__ import annotations + +import argparse +import importlib.metadata +import importlib.util +import pathlib +import subprocess # nosec B404 +import sys +import tarfile +import tempfile +import textwrap +import tomllib +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Callable, Iterable, Optional + + from packaging.markers import Marker + from packaging.requirements import Requirement + +ROOT = pathlib.Path(__file__).resolve().parents[1] +PYPROJECT = ROOT / 'pyproject.toml' +FILE = ROOT / 'conda' / 'requirements.txt' + +#: The extras the conda distribution carries, besides ``dependencies``. +EXTRAS = ('all',) +#: The platforms the conda workflows build on -- ``cron-conda.yml`` and +#: ``create-release.yml`` -- each as ``platform.system()`` and +#: ``platform.machine()`` read on its runner: ``ubuntu-latest`` and the arm64 +#: ``macos-latest``. Their Windows legs are commented out, and with them the +#: recipe's ``bld.bat``. +PLATFORMS = {'linux': ('Linux', 'x86_64'), 'darwin': ('Darwin', 'arm64')} + + +def _load_python_versions() -> 'object': + """:file:`util/python_versions.py`, the tier file's reader, beside this script.""" + spec = importlib.util.spec_from_file_location('python_versions', + pathlib.Path(__file__).with_name('python_versions.py')) + assert spec is not None and spec.loader is not None # nosec B101 + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) # type: ignore[union-attr] + return module + + +def conda_pythons() -> 'list[str]': + """The CPython versions a conda package is built for, as ``conda`` in python_versions.py.""" + versions = _load_python_versions() + return list(versions.tier(versions.load(), 'cpython', 'supported')) # type: ignore[attr-defined] + + +def environments(pythons: 'Iterable[str]') -> 'list[dict[str, str]]': + """A marker environment per conda build: each Python on each of :data:`PLATFORMS`.""" + found = [] + for python in pythons: + for platform, (system, machine) in PLATFORMS.items(): + found.append({ + 'python_version': python, 'python_full_version': f'{python}.0', + 'implementation_name': 'cpython', 'platform_python_implementation': 'CPython', + 'implementation_version': f'{python}.0', 'sys_platform': platform, + 'platform_system': system, 'os_name': 'posix', + 'platform_machine': machine, 'platform_release': '', 'platform_version': '', + }) + return found + + +def direct(pyproject: 'pathlib.Path' = PYPROJECT) -> 'list[Requirement]': + """``[project] dependencies`` and the :data:`EXTRAS`, as written in ``pyproject``.""" + from packaging.requirements import Requirement # pylint: disable=import-outside-toplevel + project = tomllib.loads(pathlib.Path(pyproject).read_text(encoding='utf-8'))['project'] + lines = list(project.get('dependencies', [])) + for extra in EXTRAS: + lines.extend(project.get('optional-dependencies', {})[extra]) + return [Requirement(line) for line in lines] + + +def _live(marker: 'Optional[Marker]', env: 'dict[str, str]', extras: 'Iterable[str]') -> 'bool': + if marker is None: + return True + return any(marker.evaluate(dict(env, extra=extra)) for extra in (tuple(extras) or ('',))) + + +class NotInstalled(Exception): + """Packages some conda build needs that this interpreter has no metadata for.""" + + +def describe(builds: 'Iterable[int]', envs: 'list[dict[str, str]]') -> 'str': + """``builds`` in words: their Pythons, then their platforms.""" + chosen = [envs[index] for index in builds] + pythons = sorted({env['python_version'] for env in chosen}, + key=lambda version: tuple(map(int, version.split('.')))) + platforms = sorted({env['sys_platform'] for env in chosen}) + return f'Python {", ".join(pythons)} on {", ".join(platforms)}' + + +def _not_installed(missing: 'dict[str, set[str]]', found: 'dict[str, dict[str, object]]', + envs: 'list[dict[str, str]]') -> 'NotInstalled': + host = (f'{sys.version_info.major}.{sys.version_info.minor}', sys.platform) + reasons = [] + for name, via in sorted(missing.items()): + builds = found[name]['builds'] # type: ignore[assignment] + here = any((envs[index]['python_version'], envs[index]['sys_platform']) == host + for index in builds) + hint = ('install .[{}] first'.format(','.join(EXTRAS)) if here else + 'its marker leaves it out here, so pip did not install it; run this on one ' + 'of those builds, or install it here by hand') + reasons.append(f'{name} is needed on {describe(builds, envs)} ({"; ".join(sorted(via))}), ' + f'and Python {host[0]} on {host[1]}, which runs this, does not have it: {hint}') + return NotInstalled('\n'.join(reasons)) + + +def derive(requirements: 'list[Requirement]', envs: 'list[dict[str, str]]', + requires: 'Callable[[str], Optional[list[str]]]') -> 'dict[str, dict[str, object]]': + """Each package to vendor, by canonical name: its class and the builds needing it. + + ``requires`` returns a distribution's ``Requires-Dist`` lines, as + :func:`importlib.metadata.requires` does, and raises + :class:`importlib.metadata.PackageNotFoundError` for one not installed. + A requirement live in no environment is recorded as ``dead`` and not walked. + One that is live but not installed here raises :class:`NotInstalled` once the + walk is done, naming every such package, what required it and where. + """ + # pylint: disable-next=import-outside-toplevel + from packaging.requirements import Requirement + from packaging.utils import canonicalize_name # pylint: disable=import-outside-toplevel + found = {} # type: dict[str, dict[str, object]] + missing = {} # type: dict[str, set[str]] + queue = [(req, frozenset(range(len(envs))), 'needed', f'pyproject.toml asks for {req}') + for req in requirements] + seen = set() # type: set[tuple[str, frozenset[str], int]] + while queue: + req, parents, kind, via = queue.pop(0) + name = canonicalize_name(req.name) + where = frozenset(index for index in parents if _live(req.marker, envs[index], ())) + if not where: + if name not in found and kind == 'needed': + found[name] = {'class': 'dead', 'builds': frozenset(), 'marker': str(req.marker)} + continue + entry = found.get(name) + if entry is None or entry['class'] == 'dead': + entry = found[name] = {'class': kind, 'builds': frozenset()} + elif kind == 'needed': + entry['class'] = 'needed' + entry['builds'] = entry['builds'] | where # type: ignore[operator] + extras = frozenset(canonicalize_name(extra) for extra in req.extras) + if name in missing: + missing[name].add(via) + continue + for index in where: + key = (name, extras, index) + if key in seen: + continue + seen.add(key) + try: + declared = requires(name) or () + except importlib.metadata.PackageNotFoundError: + missing[name] = {via} + break + for line in declared: + child = Requirement(line) + if _live(child.marker, envs[index], extras): + queue.append((Requirement(str(child).split(';')[0]), + frozenset((index,)), 'transitive', f'{name} requires {child}')) + if missing: + raise _not_installed(missing, found, envs) + return found + + +def render_marker(builds: 'frozenset[int]', envs: 'list[dict[str, str]]') -> 'Optional[str]': + """The marker selecting exactly ``builds``, or :data:`None` when that is all of them.""" + if len(builds) == len(envs): + return None + pythons = sorted({envs[index]['python_version'] for index in builds}, + key=lambda version: tuple(map(int, version.split('.')))) + whole = {index for index, env in enumerate(envs) if env['python_version'] in pythons} + if whole != set(builds): + raise ValueError('needed on some platforms of one Python and not the others; ' + 'conda/requirements.txt cannot say so with a python_version marker') + return ' or '.join(f'python_version == "{python}"' for python in pythons) + + +def lines(found: 'dict[str, dict[str, object]]', envs: 'list[dict[str, str]]', + version: 'Callable[[str], str]') -> 'list[str]': + """:file:`conda/requirements.txt`'s lines, sorted, one pin per vendored package.""" + result = [] + for name, entry in sorted(found.items()): + if entry['class'] == 'dead': + continue + marker = render_marker(entry['builds'], envs) # type: ignore[arg-type] + pin = f'{name}=={version(name)}' + result.append(pin if marker is None else f'{pin}; {marker}') + return result + + +def parse(text: 'str') -> 'dict[str, Optional[str]]': + """A requirements file's packages, by canonical name, each with its marker.""" + from packaging.requirements import Requirement # pylint: disable=import-outside-toplevel + from packaging.utils import canonicalize_name # pylint: disable=import-outside-toplevel + result = {} # type: dict[str, Optional[str]] + for line in text.splitlines(): + line = line.split('#', 1)[0].strip() + if not line: + continue + req = Requirement(line) + result[canonicalize_name(req.name)] = None if req.marker is None else str(req.marker) + return result + + +def compare(text: 'str', found: 'dict[str, dict[str, object]]', + envs: 'list[dict[str, str]]') -> 'list[str]': + """Why ``text`` is stale against ``found``, one line per package; empty when current.""" + from packaging.markers import Marker # pylint: disable=import-outside-toplevel + have = parse(text) + want = {name: render_marker(entry['builds'], envs) # type: ignore[arg-type] + for name, entry in found.items() if entry['class'] != 'dead'} + want = {name: None if marker is None else str(Marker(marker)) for name, marker in want.items()} + problems = [] + for name in sorted(set(have) | set(want)): + if name not in have: + problems.append(f'missing: {name} ({found[name]["class"]})') + elif name not in want: + if name in found: + problems.append(f'dead: {name} -- its marker {found[name]["marker"]} is false ' + 'on every conda build') + else: + problems.append(f'stale: {name} -- neither the dependencies nor the ' + f'{", ".join(EXTRAS)} extra need it, directly or not') + elif have[name] != want[name]: + problems.append(f'marker: {name} has {have[name]!r}, derived {want[name]!r}') + return problems + + +def _requires(name: 'str') -> 'Optional[list[str]]': + return importlib.metadata.requires(name) + + +def _version(name: 'str') -> 'str': + return importlib.metadata.version(name) + + +#: Run with ``-I -S`` and the unpacked ``site-packages`` as its argument, so that +#: nothing outside the package can satisfy an import. +PROBE = textwrap.dedent('''\ + import importlib, os, sys + sys.path.insert(0, sys.argv[1]) + import pcapkit + extern = os.path.realpath(pcapkit._extern) + assert extern in map(os.path.realpath, sys.path), 'pcapkit/_extern is not on sys.path' + failures = 0 + infos = sorted(name for name in os.listdir(extern) if name.endswith('.dist-info')) + tops = {} + for info in infos: + with open(os.path.join(extern, info, 'RECORD'), encoding='utf-8') as file: + paths = [row.rsplit(',', 2)[0] for row in file.read().splitlines() if row] + paths = [path for path in paths if not path.startswith('..') + and '__pycache__' not in path.split('/')] + missing = [path for path in paths if not os.path.exists(os.path.join(extern, path))] + names = set() + for path in paths: + head = path.split('/', 1)[0] + if head.endswith('.dist-info') or head.endswith('.data') or head in ('bin', 'share'): + continue + if '/' in path or head.endswith('.py') or '.cpython-' in head or head.endswith(('.so', '.pyd')): + names.add(head.split('.', 1)[0]) + tops[info] = sorted(names) + status = 'ok' if not missing else 'MISSING ' + ', '.join(missing[:5]) + print(f'{info}: {len(paths) - len(missing)}/{len(paths)} files, {status}') + failures += bool(missing) + provided = {name for names in tops.values() for name in names} + for info, names in tops.items(): + for name in names: + try: + module = importlib.import_module(name) + except ModuleNotFoundError as error: + if (error.name or '').split('.')[0] in provided: + print(f' {name}: FAILED, {error!r}') + failures += 1 + else: + print(f' {name}: skipped, needs {error.name}, which is not vendored') + continue + except Exception as error: # pylint: disable=broad-except + print(f' {name}: FAILED, {error!r}') + failures += 1 + continue + where = os.path.realpath(getattr(module, '__file__', None) or list(module.__path__)[0]) + if not where.startswith(extern + os.sep): + print(f' {name}: FAILED, imported from {where}') + failures += 1 + else: + print(f' {name}: imported from pcapkit/_extern') + if not infos: + print('pcapkit/_extern holds no dist-info at all') + failures += 1 + sys.exit(1 if failures else 0) +''') + + +def smoke(package: 'pathlib.Path') -> 'int': + """Unpack a built ``.tar.bz2`` conda package and run :data:`PROBE` against it.""" + with tempfile.TemporaryDirectory() as temp: + with tarfile.open(package, 'r:bz2') as archive: + archive.extractall(temp, filter='data') + inits = sorted(pathlib.Path(temp).glob('**/site-packages/pcapkit/__init__.py')) + if len(inits) != 1: + print(f'{package}: expected one site-packages/pcapkit, found {len(inits)}', file=sys.stderr) + return 1 + site = inits[0].parents[1] + built = site.parent.name + if built != f'python{sys.version_info.major}.{sys.version_info.minor}': + print(f'{package} is built for {built}, and this is {sys.version}', file=sys.stderr) + return 1 + return subprocess.run([sys.executable, '-I', '-S', '-c', PROBE, str(site)], # nosec B603 + check=False).returncode + + +def main(argv: 'Optional[list[str]]' = None) -> 'int': + parser = argparse.ArgumentParser(description=__doc__.split('\n', 1)[0]) + group = parser.add_mutually_exclusive_group() + group.add_argument('--check', action='store_true', + help='write nothing; exit 1 when conda/requirements.txt is stale') + group.add_argument('--smoke', metavar='PACKAGE', type=pathlib.Path, + help='check the vendored packages of a built .tar.bz2 conda package') + args = parser.parse_args(argv) + + if args.smoke is not None: + return smoke(args.smoke) + + envs = environments(conda_pythons()) + try: + found = derive(direct(), envs, _requires) + except NotInstalled as error: + print(error, file=sys.stderr) + return 2 + + if args.check: + problems = compare(FILE.read_text(encoding='utf-8'), found, envs) + for problem in problems: + print(f'{FILE.relative_to(ROOT)}: {problem}') + return 1 if problems else 0 + + FILE.write_text(''.join(f'{line}\n' for line in lines(found, envs, _version)), encoding='utf-8') + return 0 + + +if __name__ == '__main__': + sys.exit(main())