fix(conda): vendor only the needed dependencies and ship their data files (#1814)

.github/workflows/create-release.yml

@@ -1221,10 +1221,10 @@ jobs:
           - ubuntu-latest
           # - windows-latest  # skip for now
         # #1633: the supported tier only, from .github/python-versions.toml
-        # through ``version_check`` (#1660). There is no KTLO 3.10 leg because it
-        # can only fail: conda/requirements.txt pins soupsieve==3.0.3
-        # (Requires-Python >=3.11.5), and util/conda-dist.py pins the newest
-        # release of every dependency, so that floor only rises.
+        # through ``version_check`` (#1660). There is no KTLO 3.10 leg: the list
+        # util/conda_vendor.py derives is for the supported tier, so it leaves
+        # out typing-extensions, which pcapkit/utilities/compat.py imports below
+        # 3.11, and pins the newest release of each dependency, a rising floor.
         python-version: ${{ fromJSON(needs.version_check.outputs.PYTHON_CONDA) }}
     steps:
       # #1634: this run's commit, not ``ref: conda-<version>+0``, then the check

.github/workflows/cron-conda.yml

@@ -156,28 +156,25 @@ jobs:
         with:
           python-version: ${{ steps.python-versions.outputs.latest }}
 
-      # `.[all,dev]`, not `.[all]` alone: `util/conda-dist.py` below only
-      # *refreshes* the version pin of every package already named in
-      # `conda/requirements.txt` -- via `importlib.metadata.version(name)`,
-      # which raises unless that distribution is installed here -- and that
-      # file names `dpkt`, `scapy`, `pyshark`, `requests` and `beautifulsoup4`
-      # with no marker. #910 narrowed `all` to core addons only, so those five
-      # come from `dev` instead; without it this step would install
-      # cleanly and the next one would crash on the first missing name.
+      # `.[all]`, which is what `util/conda_vendor.py` below vendors (#1814):
+      # it derives `conda/requirements.txt` from `dependencies` and the `all`
+      # extra, walks what they require through the installed metadata, and
+      # pins each package at the version installed here. So this install is
+      # the step that resolves the newest releases, and the script itself
+      # reads nothing from the network.
       - name: Install and Setup
         run: |
           set -x
 
           python -m pip install -U pip setuptools wheel packaging
-          python -m pip install -e .[all,dev]
-          #python -m pip install pathlib2 typing_extensions
+          python -m pip install -e .[all]
 
       - name: Setup Conda Distribution
         run: |
           export PCAPKIT_DEVMODE=1
           export PCAPKIT_VERBOSE=1
 
-          python util/conda-dist.py
+          python util/conda_vendor.py
 
       - name: Verify Changed files
         uses: tj-actions/verify-changed-files@v17
@@ -452,14 +449,12 @@ jobs:
 
           python -m pip install -U pip setuptools wheel
           python -m pip install -e .[all]
-          python -m pip install pathlib2 typing_extensions
 
       - name: Install and Setup (Windows)
         if: ${{ matrix.os == 'windows-latest' }}
         run: |
           python -m pip install -U pip setuptools wheel
           python -m pip install -e .[all]
-          python -m pip install pathlib2 typing_extensions
 
       - name: Setup Conda Distribution
         if: ${{ matrix.os != 'windows-latest' }}
@@ -530,6 +525,8 @@ jobs:
               ;;
           esac
 
+          echo "host_package=$host_package" >> "$GITHUB_OUTPUT"
+
           host_platform="$(basename "$(dirname "$host_package")")"
           if [ -n "$target_platform" ] && [ "$target_platform" != "$host_platform" ]; then
             conda convert "$host_package" -o "$out_dir" -p "$target_platform"
@@ -550,6 +547,20 @@ jobs:
             echo EOF
           } >> "$GITHUB_OUTPUT"
 
+      # #1814: the package declares no run requirements, so the packages it
+      # vendors in `pcapkit/_extern` are all it has, and they once shipped as
+      # modules alone. This unpacks the package built for this runner and
+      # checks that every vendored distribution has every file its RECORD lists
+      # and imports from there, with no site-packages on the path, before
+      # anything is uploaded. setup-python's interpreter, not conda's: it is
+      # the matrix version the package was built for, which conda's base is
+      # not on 3.15.
+      - name: Smoke-test the vendored packages
+        env:
+          PCAPKIT_CONDA_PACKAGE: ${{ steps.conda_build.outputs.host_package }}
+        run: |
+          "${pythonLocation}/bin/python" util/conda_vendor.py --smoke "$PCAPKIT_CONDA_PACKAGE"
+
       - name: Upload conda packages to Anaconda
         uses: anaconda/actions/upload-package@v0.3.1
         with:

MANIFEST.in

@@ -87,6 +87,7 @@ recursive-include docs/source/changelog *.rst
 # Nothing here names ``pcapkit/_extern``. ``conda/pypcapkit/build.sh`` and
 # ``bld.bat`` create it and then run ``pip install .``, which builds the wheel
 # straight from the tree, so the conda recipe builds no sdist; its modules
-# reach that wheel through package discovery. A line naming it would warn in
-# every other build, and ``include`` of a directory matched nothing even in the
-# conda one (#1794).
+# reach that wheel through package discovery, and every other file under it
+# through the ``package-data`` entry in ``pyproject.toml`` (#1814). A line naming
+# it would warn in every other build, and ``include`` of a directory matched
+# nothing even in the conda one (#1794).

conda/pypcapkit/meta.yaml

@@ -24,7 +24,7 @@ requirements:
     - git
     - python
     - pip
-    - setuptools>=61.0.0
+    - setuptools>=62.3.0
 
     # # version compatibility
     # - bpc-f2format
@@ -49,7 +49,7 @@ requirements:
   host:
     - python
     - pip
-    - setuptools>=61.0.0
+    - setuptools>=62.3.0
 
     # # version compatibility
     # - bpc-f2format

conda/requirements.txt

@@ -1,25 +1,9 @@
 aenum==3.1.17
-appdirs==1.4.4
-beautifulsoup4==4.15.0
-certifi==2026.7.22
+cffi==2.1.1
 chardet==7.6.0
-charset-normalizer==3.5.2
+cryptography==50.0.2
 dictdumper==0.8.4.post6
-dpkt==1.9.8
 emoji==2.16.0
-html5lib==1.1
-idna==3.20
-lxml==6.1.3
-packaging==26.3
-pathlib2; python_version < "3.5"
-pyshark==0.6
-pysocks==1.7.1
-requests==2.34.2
-scapy==2.8.0
-six==1.17.0
-soupsieve==3.0.3
+pycparser==3.11
+pycrate==0.8.1
 tbtrim==0.3.1
-termcolor==3.3.0
-typing-extensions==4.16.0; python_version < "3.11"
-urllib3==2.8.0
-webencodings==0.6.1

pyproject.toml

@@ -1,6 +1,10 @@
 [build-system]
 requires = [
-    "setuptools>=61.0.0",
+    # 62.3.0 is the first release whose ``package-data`` globs read ``**`` as any
+    # depth, which the ``pcapkit._extern`` entry below needs (#1814). Measured:
+    # 61.0.0 and 62.2.0 read it as ``*``, and ship only the files one directory
+    # down from it.
+    "setuptools>=62.3.0",
 
     # version compatibility
     "bpc-f2format; python_version < '3.6'",
@@ -50,7 +54,7 @@ license = { text="BSD 3-Clause License" }
 # blocking, and a failure there is a warning on a green leg (#1672). Each 3.7 leg
 # fails a step by design: the import fails as above, and the ``test`` extra's
 # ``pytest>=8`` needs 3.8. 3.6 is not run at all: setup-python
-# builds it only for retired Ubuntu images, and the build needs ``setuptools>=61``
+# builds it only for retired Ubuntu images, and the build needs ``setuptools>=62.3``
 # besides. The ``PCAP_CT`` extra needs 3.10 regardless.
 requires-python = ">=3.6, <4"
 description = "PyPCAPKit: comprehensive network packet analysis library"
@@ -368,6 +372,21 @@ exclude = [
     "sample*",
 ]
 
+# Only the conda recipe creates ``pcapkit/_extern``: ``conda/pypcapkit/build.sh``
+# and ``bld.bat`` install the packages ``conda/requirements.txt`` names there,
+# then run ``pip install .``, which builds the wheel from the tree. Package
+# discovery reaches their modules alone, so every other file -- ``emoji``'s JSON,
+# ``chardet``'s models, compiled extensions, each ``dist-info`` -- stays behind
+# without this entry (#1814). Elsewhere ``pcapkit._extern`` is no package at all,
+# and setuptools ignores these entries without a warning; ``MANIFEST.in`` could not
+# do this, since a line naming a missing directory warns in every such build. The
+# bytecode ``pip install --target`` compiles stays out, as from any other wheel.
+[tool.setuptools.package-data]
+"pcapkit._extern" = [ "**/*" ]
+
+[tool.setuptools.exclude-package-data]
+"pcapkit._extern" = [ "*.pyc" ]
+
 [tool.setuptools.dynamic]
 version = { attr="pcapkit.__version__" }