diff --git a/docs/source/pcapkit/vendor/index.rst b/docs/source/pcapkit/vendor/index.rst index f860acc7f..01f678391 100644 --- a/docs/source/pcapkit/vendor/index.rst +++ b/docs/source/pcapkit/vendor/index.rst @@ -8,6 +8,19 @@ This module contains all web crawlers of :mod:`pcapkit`, which are automatically generating the :mod:`pcapkit.const` module's constant enumerations. +The crawlers need the ``vendor`` extra, ``pip install "pypcapkit[vendor]"``. + +.. versionchanged:: 1.5.0 + + Without ``requests`` or ``beautifulsoup4``, which the crawlers import at + module scope, importing this package succeeds and imports no crawler. + Reading any name in its ``__all__``, or + :class:`~pcapkit.vendor.default.Vendor`, then raises + :exc:`ModuleNotFoundError` naming the missing packages and the extra to + install (:issue:`1850`). Importing a crawler module directly, such as + :mod:`pcapkit.vendor.reg`, still raises the missing package's own + :exc:`ModuleNotFoundError`. Before, importing this package raised it. + Crawler Implementations ======================= @@ -95,3 +108,11 @@ constant enumerations. options: -h, --help show this help message and exit -V, --version show program's version number and exit + +.. versionchanged:: 1.5.0 + + Without ``requests`` or ``beautifulsoup4`` from the ``vendor`` extra, + :program:`pcapkit-vendor` writes one line to standard error naming the + missing packages and the extra to install, and exits with status ``1`` + before any target runs (:issue:`1850`). It used to fail with a traceback. + ``--help`` and ``--version`` work without them. diff --git a/pcapkit/vendor/__init__.py b/pcapkit/vendor/__init__.py index 46d27a5f0..dcd058938 100644 --- a/pcapkit/vendor/__init__.py +++ b/pcapkit/vendor/__init__.py @@ -9,51 +9,117 @@ automatically generating the :mod:`pcapkit.const` module's constant enumerations. +The crawlers need the ``vendor`` extra, ``pip install "pypcapkit[vendor]"``. + +.. versionchanged:: 1.5.0 + + Without ``requests`` or ``beautifulsoup4``, which the crawlers import at + module scope, importing this package succeeds and imports no crawler. + Reading any name in its ``__all__``, or + :class:`~pcapkit.vendor.default.Vendor`, then raises + :exc:`ModuleNotFoundError` naming the missing packages and the extra to + install, and :program:`pcapkit-vendor` exits with that message on one line + (:issue:`1850`). Importing a crawler module directly, such as + :mod:`pcapkit.vendor.reg`, still raises the missing package's own + :exc:`ModuleNotFoundError`. Before, importing this package raised it. + """ +import typing from pcapkit.utilities.compat import ModuleNotFoundError # pylint: disable=redefined-builtin from pcapkit.utilities.exceptions import stacklevel from pcapkit.utilities.warnings import VendorWarning, warn +#: list[str]: Distributions of the ``vendor`` extra whose modules the crawlers +#: import at module scope, and which cannot be imported here. While it is not +#: empty no crawler is imported: :func:`__getattr__` raises for their names, +#: and :func:`pcapkit.vendor.__main__.main` exits (:issue:`1850`). :mod:`pcapkit` +#: has already put the conda package's :file:`pcapkit/_extern` on +#: :data:`sys.path` by then, so a package vendored there counts as installed. +_MISSING = [] # type: list[str] + try: import requests +except ModuleNotFoundError: + _MISSING.append('requests') + +try: + import bs4 +except ModuleNotFoundError: + _MISSING.append('beautifulsoup4') + +# The ``[socks]`` and ``[html5lib]`` extras of those packages are needed only at +# run time, by a crawler that goes through a SOCKS proxy or parses HTML, so +# their absence stays a warning. +try: import socks except ModuleNotFoundError: warn("dependency package 'requests[socks]' not found", VendorWarning, stacklevel=stacklevel()) try: - import bs4 import html5lib except ModuleNotFoundError: warn("dependency package 'beautifulsoup4[html5lib]' not found", VendorWarning, stacklevel=stacklevel()) -# base crawler -from pcapkit.vendor.default import Vendor - -# IANA registration -from pcapkit.vendor.reg import * - -# Miscellanous -from pcapkit.vendor.pcapng import * - -# per protocol -from pcapkit.vendor.arp import * -from pcapkit.vendor.esp import * -from pcapkit.vendor.ftp import * -from pcapkit.vendor.hip import * -from pcapkit.vendor.http import * -from pcapkit.vendor.ipv4 import * -from pcapkit.vendor.ipv6 import * -from pcapkit.vendor.ipx import * -from pcapkit.vendor.l2tp import * -from pcapkit.vendor.mh import * -from pcapkit.vendor.ngap import * -from pcapkit.vendor.ospf import * -from pcapkit.vendor.sctp import * -from pcapkit.vendor.tcp import * -from pcapkit.vendor.vlan import * + +def _missing_message() -> 'str': + """Why no crawler was imported, naming the extra to install. + + Returns: + The packages in :data:`_MISSING` and the extra that installs them, on + one line, or an empty string if :data:`_MISSING` is empty. + + """ + if not _MISSING: + return '' + return (f"missing dependencies: {', '.join(_MISSING)}; " + 'install the vendor extra: pip install "pypcapkit[vendor]"') + + +if _MISSING: + def __getattr__(name: 'str') -> 'typing.NoReturn': + """Raise for a crawler, or for its base class, which were not imported. + + :pep:`562` calls this only for a name missing from this module's + globals, so it is defined only while :data:`_MISSING` is not empty. + + Raises: + ModuleNotFoundError: If ``name`` is in ``__all__``, or is + ``Vendor``, naming the missing packages and the extra. + AttributeError: For any other name, as without this hook. + + """ + if name == 'Vendor' or name in __all__: + raise ModuleNotFoundError(f'cannot import {name!r} from {__name__!r}: {_missing_message()}') + raise AttributeError(f'module {__name__!r} has no attribute {name!r}') +else: + # base crawler + from pcapkit.vendor.default import Vendor + + # IANA registration + from pcapkit.vendor.reg import * + + # Miscellanous + from pcapkit.vendor.pcapng import * + + # per protocol + from pcapkit.vendor.arp import * + from pcapkit.vendor.esp import * + from pcapkit.vendor.ftp import * + from pcapkit.vendor.hip import * + from pcapkit.vendor.http import * + from pcapkit.vendor.ipv4 import * + from pcapkit.vendor.ipv6 import * + from pcapkit.vendor.ipx import * + from pcapkit.vendor.l2tp import * + from pcapkit.vendor.mh import * + from pcapkit.vendor.ngap import * + from pcapkit.vendor.ospf import * + from pcapkit.vendor.sctp import * + from pcapkit.vendor.tcp import * + from pcapkit.vendor.vlan import * # Crawlers only. This list is also the default target list of # :mod:`pcapkit.vendor.__main__`, which runs every name in it, so the base diff --git a/pcapkit/vendor/__main__.py b/pcapkit/vendor/__main__.py index 6e9e64bad..4e607dd1a 100644 --- a/pcapkit/vendor/__main__.py +++ b/pcapkit/vendor/__main__.py @@ -24,6 +24,7 @@ from pcapkit import vendor as vendor_module from pcapkit.utilities.logging import VERBOSE, get_logger from pcapkit.utilities.warnings import InvalidVendorWarning, VendorRuntimeWarning, warn +from pcapkit.vendor import _missing_message if TYPE_CHECKING: from argparse import ArgumentParser @@ -212,15 +213,30 @@ def main() -> 'int': from a real one at the process boundary, which is what CI actually checks. Returns: - ``0`` if every target ran to completion, ``1`` if any target raised. + ``0`` if every target ran to completion, ``1`` if any target raised, or + if a package the crawlers import at module scope is not installed. Warns: InvalidVendorWarning: If vendor target not found in :mod:`pcapkit.vendor` module. + .. versionchanged:: 1.5.0 + + Without ``requests`` or ``beautifulsoup4`` from the ``vendor`` extra, + writes one line to :data:`sys.stderr` naming the missing packages and + the extra to install, and returns ``1`` before any target runs + (:issue:`1850`). Importing this module used to raise + :exc:`ModuleNotFoundError` with a traceback. ``--help`` and + ``--version`` work without them. + """ parser = get_parser() args = parser.parse_args() + missing = _missing_message() + if missing: + print(f'{parser.prog}: error: {missing}', file=sys.stderr) + return 1 + target_list = [] # type: list[Type[Vendor]] for target in args.target: try: diff --git a/tests/vendor/test_crawler_reachability_unit.py b/tests/vendor/test_crawler_reachability_unit.py index 400a37030..52f55c955 100644 --- a/tests/vendor/test_crawler_reachability_unit.py +++ b/tests/vendor/test_crawler_reachability_unit.py @@ -79,12 +79,12 @@ ROOT = pathlib.Path(__file__).resolve().parents[2] #: Whether the crawler machinery is importable at all. Importing -#: :mod:`pcapkit.vendor` pulls in every subpackage unconditionally: -#: :mod:`pcapkit.vendor.default` imports :mod:`requests` at module scope and -#: several of the crawlers import :mod:`bs4` at module scope, so both are needed -#: for this file rather than optional to it. (:mod:`pcapkit.vendor`'s own -#: ``try: import bs4`` downgrades the *warning*, not the -#: :exc:`ModuleNotFoundError` raised by the crawler imports below it.) +#: :mod:`pcapkit.vendor` pulls in every subpackage only when :mod:`requests` and +#: :mod:`bs4` are installed: :mod:`pcapkit.vendor.default` imports +#: :mod:`requests` at module scope and several of the crawlers import :mod:`bs4` +#: at module scope, so both are needed for this file rather than optional to it. +#: (Without either, :mod:`pcapkit.vendor` imports no crawler and raises +#: :exc:`ModuleNotFoundError` for a crawler's name, #1850.) #: #: Both ship in the ``test`` extra as well as ``vendor`` #: (:file:`pyproject.toml`), so this guard should not fire in CI; it is kept as diff --git a/tests/vendor/test_ipx_packet_unit.py b/tests/vendor/test_ipx_packet_unit.py index 32e74eac4..e5699543c 100644 --- a/tests/vendor/test_ipx_packet_unit.py +++ b/tests/vendor/test_ipx_packet_unit.py @@ -43,13 +43,12 @@ #: Repository root, i.e. the grandparent of the directory holding this file. ROOT = pathlib.Path(__file__).resolve().parents[2] -#: Every distribution importing :mod:`pcapkit.vendor` needs. ``requests`` is the -#: obvious one -- ``pcapkit.vendor.default`` imports it at module scope -- but it -#: is not sufficient: importing *any* crawler imports the ``pcapkit.vendor`` -#: package, whose :file:`__init__.py` pulls in all seventeen subpackages, seven of -#: which ``import bs4`` at module scope. So a guard on ``requests`` alone lets the -#: suite error instead of skipping on a machine that happens to have ``requests`` -#: and not ``beautifulsoup4``. +#: Every distribution importing :mod:`pcapkit.vendor` with its crawlers needs. +#: ``pcapkit.vendor.default`` imports ``requests`` at module scope, and several +#: crawlers ``import bs4`` at module scope. Without either, the ``pcapkit.vendor`` +#: package imports none of its crawlers (#1850), so this file's crawler, which +#: imports no ``bs4``, would import with ``requests`` alone; the gate names the +#: whole set so the package imports here as it does with the ``vendor`` extra. VENDOR_DEPS = ('requests', 'bs4', 'html5lib') #: Whether the crawlers are importable at all. ``html5lib`` ships in the diff --git a/tests/vendor/test_ipx_socket_unit.py b/tests/vendor/test_ipx_socket_unit.py index be52b6677..47fbc5f26 100644 --- a/tests/vendor/test_ipx_socket_unit.py +++ b/tests/vendor/test_ipx_socket_unit.py @@ -37,13 +37,12 @@ #: Repository root, i.e. the grandparent of the directory holding this file. ROOT = pathlib.Path(__file__).resolve().parents[2] -#: Whether the crawler machinery is importable at all. Two packages are needed -#: and neither is optional to this file: ``pcapkit.vendor.default`` imports -#: :mod:`requests` at module scope, and :mod:`pcapkit.vendor` then imports every -#: crawler unconditionally, several of which import :mod:`bs4` at module scope -- -#: so importing *one* crawler needs both. (:mod:`pcapkit.vendor`'s own -#: ``try: import bs4`` suppresses the warning, not the ``ModuleNotFoundError`` -#: raised by the crawler imports below it.) +#: Whether the crawler machinery is importable at all. ``pcapkit.vendor.default`` +#: imports :mod:`requests` at module scope, and :mod:`pcapkit.vendor` imports its +#: crawlers, several of which import :mod:`bs4` at module scope, only when +#: :mod:`requests` and :mod:`bs4` are installed (#1850). This file's crawler +#: imports no :mod:`bs4`, so :mod:`bs4` is in the gate for the package to import +#: as it does in CI, not because the import would fail without it. #: #: Both ship in the ``test`` extra as well as ``vendor`` #: (:file:`pyproject.toml`), because CI installs ``.[test]`` and a skipped pin diff --git a/tests/vendor/test_request_prompt_unit.py b/tests/vendor/test_request_prompt_unit.py index d02a2e67b..1539e6bec 100644 --- a/tests/vendor/test_request_prompt_unit.py +++ b/tests/vendor/test_request_prompt_unit.py @@ -91,11 +91,13 @@ #: Repository root, i.e. the grandparent of the directory holding this file. ROOT = pathlib.Path(__file__).resolve().parents[2] -#: Every distribution importing :mod:`pcapkit.vendor` needs. Copied from -#: :file:`tests/vendor/test_user_agent_unit.py`, and for its reason: ``requests`` -#: alone is not enough, because importing :mod:`pcapkit.vendor.default` imports the -#: :mod:`pcapkit.vendor` package first and its :file:`__init__.py` pulls in -#: subpackages that ``import bs4`` at module scope. +#: Every distribution importing :mod:`pcapkit.vendor` with its crawlers needs. +#: Copied from :file:`tests/vendor/test_user_agent_unit.py`, and for its reason: +#: importing :mod:`pcapkit.vendor.default` imports the :mod:`pcapkit.vendor` +#: package first, whose :file:`__init__.py` pulls in subpackages that +#: ``import bs4`` at module scope only when ``requests`` and ``bs4`` are +#: installed (#1850), and the gate keeps the package as the ``vendor`` extra +#: imports it. VENDOR_DEPS = ('requests', 'bs4', 'html5lib') #: Whether the crawlers are importable at all. ``html5lib`` ships in the diff --git a/tests/vendor/test_user_agent_unit.py b/tests/vendor/test_user_agent_unit.py index bbf9a9d81..16beba92b 100644 --- a/tests/vendor/test_user_agent_unit.py +++ b/tests/vendor/test_user_agent_unit.py @@ -53,13 +53,14 @@ #: Repository root, i.e. the grandparent of the directory holding this file. ROOT = pathlib.Path(__file__).resolve().parents[2] -#: Every distribution importing :mod:`pcapkit.vendor` needs. ``requests`` is the -#: obvious one -- ``pcapkit.vendor.default`` imports it at module scope -- but it -#: is not sufficient: importing ``pcapkit.vendor.default`` imports the -#: ``pcapkit.vendor`` package first, and its :file:`__init__.py` pulls in every -#: subpackage, several of which ``import bs4`` at module scope. So a -#: guard on ``requests`` alone lets the suite error instead of skipping on a -#: machine that happens to have ``requests`` and not ``beautifulsoup4``. +#: Every distribution importing :mod:`pcapkit.vendor` with its crawlers needs. +#: ``pcapkit.vendor.default`` imports ``requests`` at module scope, and +#: importing it imports the ``pcapkit.vendor`` package first, whose +#: :file:`__init__.py` pulls in every subpackage, several of which +#: ``import bs4`` at module scope, only when ``requests`` and ``bs4`` are +#: installed (#1850). So ``requests`` alone would import +#: ``pcapkit.vendor.default``; the gate names the whole set so the package +#: imports here as it does with the ``vendor`` extra. VENDOR_DEPS = ('requests', 'bs4', 'html5lib') #: Whether the crawlers are importable at all. ``html5lib`` ships in the diff --git a/tests/vendor/test_vendor_dest_path_unit.py b/tests/vendor/test_vendor_dest_path_unit.py index 4093cb0d8..19f7891e5 100644 --- a/tests/vendor/test_vendor_dest_path_unit.py +++ b/tests/vendor/test_vendor_dest_path_unit.py @@ -83,12 +83,13 @@ :file:`tests/vendor/test_user_agent_unit.py` already use for the same reason. The suite is unit-tier (see :mod:`tests._tiers`): it reads no capture and makes -no network call. Both classes are gated on :data:`HAS_VENDOR_DEPS`, the same -two-dependency gate :file:`tests/vendor/test_crawler_reachability_unit.py` -uses, because importing :mod:`pcapkit.vendor.default` first imports the -:mod:`pcapkit.vendor` package, whose :file:`__init__.py` pulls in every -crawler subpackage unconditionally -- several of which ``import bs4`` at module -scope. ``html5lib`` is deliberately *not* in the gate: it appears nowhere at +no network call. Its classes are gated on :data:`HAS_VENDOR_DEPS`, the same +gate :file:`tests/vendor/test_crawler_reachability_unit.py` uses, because they +read the crawlers off the :mod:`pcapkit.vendor` package, whose +:file:`__init__.py` imports them only when ``requests`` and ``bs4`` are +installed (#1850) -- :mod:`pcapkit.vendor.default` imports ``requests`` at +module scope, and several crawlers ``import bs4`` at module scope. +``html5lib`` is deliberately *not* in the gate: it appears nowhere at module scope, only as the string parser name in the crawlers' ``bs4.BeautifulSoup(text, 'html5lib')`` calls at actual crawl time, which nothing here reaches -- no test in this file calls :meth:`Vendor.__init__` or diff --git a/tests/vendor/test_vendor_exit_code_unit.py b/tests/vendor/test_vendor_exit_code_unit.py index 2173ac2f2..db4f828f9 100644 --- a/tests/vendor/test_vendor_exit_code_unit.py +++ b/tests/vendor/test_vendor_exit_code_unit.py @@ -63,13 +63,14 @@ import warnings from unittest import mock -#: Same two-dependency gate :file:`tests/vendor/test_crawler_reachability_unit.py` -#: and :file:`tests/vendor/test_vendor_dest_path_unit.py` use. Importing +#: Same gate :file:`tests/vendor/test_crawler_reachability_unit.py` and +#: :file:`tests/vendor/test_vendor_dest_path_unit.py` use. Importing #: :mod:`pcapkit.vendor.__main__` imports :mod:`pcapkit.vendor` (for the #: ``vendor_module`` default-target-list fallback this suite patches), which -#: pulls in every crawler subpackage unconditionally -- -#: :mod:`pcapkit.vendor.default` imports :mod:`requests` at module scope and -#: several crawlers import :mod:`bs4` at module scope. +#: imports every crawler subpackage when :mod:`requests` and :mod:`bs4` are +#: installed and none otherwise (#1850). :mod:`pcapkit.vendor.default`, whose +#: :class:`~pcapkit.vendor.default.Vendor` this suite subclasses, imports +#: :mod:`requests` at module scope. VENDOR_DEPS = ('requests', 'bs4') #: Whether the crawler machinery is importable at all. Both ship in the diff --git a/tests/vendor/test_vendor_missing_extra_1850_unit.py b/tests/vendor/test_vendor_missing_extra_1850_unit.py new file mode 100644 index 000000000..e44f0dff5 --- /dev/null +++ b/tests/vendor/test_vendor_missing_extra_1850_unit.py @@ -0,0 +1,208 @@ +# -*- coding: utf-8 -*- +"""``pcapkit-vendor`` without the ``vendor`` extra exits on one line, not with a traceback. + +GitHub issue #1850. :mod:`pcapkit.vendor` imported every crawler unconditionally, +and :mod:`pcapkit.vendor.default` imports ``requests`` at module scope, so without +the extra importing the package raised :exc:`ModuleNotFoundError`. The console +script and ``python -m pcapkit.vendor`` import that package before any line of +:mod:`pcapkit.vendor.__main__` runs, so the user got a traceback; the +:class:`~pcapkit.utilities.warnings.VendorWarning` emitted first is a +:exc:`ResourceWarning`, which Python hides by default. The owner ruled that the +command should instead exit with one line naming the missing packages and the +extra to install, for a pip install without ``[vendor]`` as much as for the conda +package, which does not vendor the crawlers' dependencies (#1814). + +The package now records what is missing in ``_MISSING`` and imports no crawler +while it is not empty. Its :pep:`562` ``__getattr__`` raises a +:exc:`ModuleNotFoundError` carrying the message for a crawler's name, and +:func:`~pcapkit.vendor.__main__.main` prints the message and returns ``1``. + +The missing packages are simulated in a fresh interpreter, by a +:file:`sitecustomize.py` that binds them to :data:`None` in :data:`sys.modules` +before anything else is imported, so importing one raises +:exc:`ModuleNotFoundError` just as a missing package does. This interpreter keeps +them, so nothing here disturbs the rest of the suite. ``PYTHONPATH`` puts this +checkout ahead of any installed copy, and +:meth:`MissingExtraTests.test_import_succeeds_and_a_crawler_name_raises` checks +that the child imported this checkout. + +Against the unfixed tree every test of :class:`MissingExtraTests` fails on the +traceback of that import, and :class:`MainTests` fails with :exc:`AttributeError`, +as there is no ``_MISSING`` to patch. :class:`PresentExtraTests` passes before and +after the fix, as it should: it pins that nothing changes with the extra installed. + +""" +from __future__ import annotations + +import contextlib +import importlib.util +import io +import os +import pathlib +import subprocess # nosec: B404 +import sys +import tempfile +import textwrap +import unittest +from typing import TYPE_CHECKING +from unittest import mock + +from tests._support import scale_timeout + +if TYPE_CHECKING: + from typing import Sequence + +#: The package under test, found from this file rather than by importing it, so +#: that a child interpreter can be pointed at the same tree. +PACKAGE = pathlib.Path(__file__).resolve().parents[2] / 'pcapkit' + +#: Same gate as the rest of :mod:`tests.vendor`, for the tests that need the +#: crawlers' dependencies installed here: ``requests`` for +#: :mod:`pcapkit.vendor.default` and ``bs4`` for the crawlers that parse HTML. +VENDOR_DEPS = ('requests', 'bs4') + +#: Whether the crawler machinery is importable at all. The ``test`` extra ships +#: these (:file:`pyproject.toml`), so this should not skip in CI. +HAS_VENDOR_DEPS = all(importlib.util.find_spec(name) is not None for name in VENDOR_DEPS) + +#: What a pip install without ``[vendor]`` lacks: the import names of the extra's +#: ``requests[socks]`` and ``beautifulsoup4[html5lib]``. ``pycrate`` stays +#: importable, being a core addon of the ``all`` extra as well, and the crawlers +#: that need it import it only when they run. +EXTRA_MODULES = ('requests', 'socks', 'bs4', 'html5lib') + +#: Why no crawler can be imported when none of :data:`EXTRA_MODULES` is installed. +REASON = ('missing dependencies: requests, beautifulsoup4; ' + 'install the vendor extra: pip install "pypcapkit[vendor]"') + +#: The line :func:`~pcapkit.vendor.__main__.main` writes to stderr for :data:`REASON`. +MESSAGE = f'pcapkit-vendor: error: {REASON}' + +#: What the ``pcapkit-vendor`` console script runs: the wrapper pip writes for a +#: ``[project.scripts]`` entry imports the entry point at module scope, so an +#: exception raised by that import escapes it. +CONSOLE_SCRIPT = ('-c', 'import sys; from pcapkit.vendor.__main__ import main; sys.exit(main())') + +#: ``python -m pcapkit.vendor``. +RUN_MODULE = ('-m', 'pcapkit.vendor') + + +def child_env() -> 'dict[str, str]': + """The environment of a child interpreter, with nothing that adds to its stderr. + + ``PCAPKIT_DEVMODE`` writes a warning through :func:`warnings.showwarning`, + which no filter stops, and ``PYTHONWARNINGS`` or ``PYTHONDEVMODE`` would show + the :class:`~pcapkit.utilities.warnings.VendorWarning` that Python hides by + default. ``COLUMNS`` and ``PYTHON_COLORS`` fix how :mod:`argparse` lays out + and colours its help, so it can be compared with this interpreter's. + + """ + env = {key: value for key, value in os.environ.items() + if not key.startswith('PCAPKIT_') + and key not in ('PYTHONSAFEPATH', 'PYTHONWARNINGS', 'PYTHONDEVMODE')} + env.update(COLUMNS='80', PYTHON_COLORS='0') + return env + + +def run(command: 'Sequence[str]', *args: str, + blocked: 'Sequence[str]' = ()) -> 'subprocess.CompletedProcess[str]': + """Run ``command`` with ``args`` in a fresh interpreter, with ``blocked`` not importable.""" + env = child_env() + with tempfile.TemporaryDirectory() as tempdir: + with open(os.path.join(tempdir, 'sitecustomize.py'), 'w', encoding='utf-8') as file: + file.write(f'import sys\nfor name in {tuple(blocked)!r}:\n sys.modules[name] = None\n') + env['PYTHONPATH'] = os.pathsep.join(filter(None, [tempdir, str(PACKAGE.parent), + os.environ.get('PYTHONPATH')])) + return subprocess.run([sys.executable, *command, *args], cwd=PACKAGE.parent, # nosec: B603 + env=env, capture_output=True, text=True, + timeout=scale_timeout(120), check=False) + + +class MissingExtraTests(unittest.TestCase): + """Without the ``vendor`` extra, in a child interpreter.""" + + def assert_one_line_exit(self, result: 'subprocess.CompletedProcess[str]', message: 'str') -> None: + """``result`` exited ``1`` with ``message`` as its only output.""" + self.assertNotIn('Traceback', result.stderr) + self.assertEqual((result.returncode, result.stdout, result.stderr.splitlines()), + (1, '', [message]), result.stderr) + + def test_console_script_exits_on_one_line(self) -> None: + self.assert_one_line_exit(run(CONSOLE_SCRIPT, blocked=EXTRA_MODULES), MESSAGE) + + def test_python_m_exits_on_one_line(self) -> None: + self.assert_one_line_exit(run(RUN_MODULE, blocked=EXTRA_MODULES), MESSAGE) + + def test_a_named_target_gets_the_same_line(self) -> None: + # main() imports a named target itself and reports an ImportError as an + # invalid target, which would hide the missing package behind a usage error. + self.assert_one_line_exit(run(RUN_MODULE, 'reg', blocked=EXTRA_MODULES), MESSAGE) + + @unittest.skipUnless(HAS_VENDOR_DEPS, f'vendor extra not installed ({", ".join(VENDOR_DEPS)})') + def test_the_line_names_only_what_is_missing(self) -> None: + self.assert_one_line_exit(run(RUN_MODULE, blocked=('bs4',)), + MESSAGE.replace('requests, ', '')) + + def test_help_needs_no_extra(self) -> None: + result = run(RUN_MODULE, '--help', blocked=EXTRA_MODULES) + self.assertEqual((result.returncode, result.stderr), (0, ''), result.stderr) + self.assertTrue(result.stdout.startswith('usage: pcapkit-vendor '), result.stdout) + + def test_import_succeeds_and_a_crawler_name_raises(self) -> None: + code = textwrap.dedent('''\ + import pcapkit.vendor + print(pcapkit.vendor.__file__) + for statement in ('from pcapkit.vendor import EtherType', + 'pcapkit.vendor.Vendor', + 'from pcapkit.vendor import *', + 'pcapkit.vendor.no_such_name'): + try: + exec(statement, {'pcapkit': pcapkit}) + except (ImportError, AttributeError) as error: + print(type(error).__name__, error, sep=': ') + ''') + result = run(('-c', code), blocked=EXTRA_MODULES) + self.assertEqual((result.returncode, result.stderr), (0, ''), result.stderr) + self.assertEqual(result.stdout.splitlines(), [ + str(PACKAGE / 'vendor' / '__init__.py'), + f"ModuleNotFoundError: cannot import 'EtherType' from 'pcapkit.vendor': {REASON}", + f"ModuleNotFoundError: cannot import 'Vendor' from 'pcapkit.vendor': {REASON}", + # import * reads __all__ in order, so the first crawler it names raises. + f"ModuleNotFoundError: cannot import 'EtherType' from 'pcapkit.vendor': {REASON}", + "AttributeError: module 'pcapkit.vendor' has no attribute 'no_such_name'", + ]) + + +@unittest.skipUnless(HAS_VENDOR_DEPS, f'vendor extra not installed ({", ".join(VENDOR_DEPS)})') +class PresentExtraTests(unittest.TestCase): + """With the crawlers' dependencies installed, nothing changes.""" + + def test_help_prints_the_parser_help(self) -> None: + from pcapkit.vendor.__main__ import get_parser + + result = run(RUN_MODULE, '--help') + self.assertEqual((result.returncode, result.stderr), (0, ''), result.stderr) + with mock.patch.dict(os.environ, child_env(), clear=True): + expected = get_parser().format_help() + self.assertEqual(result.stdout, expected) + + +class MainTests(unittest.TestCase): + """:func:`~pcapkit.vendor.__main__.main` in this interpreter, with ``_MISSING`` patched.""" + + def test_main_reports_on_one_line_and_runs_no_target(self) -> None: + import pcapkit.vendor + import pcapkit.vendor.__main__ as vendor_main + + stderr = io.StringIO() + with mock.patch.object(pcapkit.vendor, '_MISSING', ['requests', 'beautifulsoup4']), \ + mock.patch.object(vendor_main, 'run', side_effect=AssertionError('a target ran')) as runner, \ + mock.patch.object(sys, 'argv', ['pcapkit-vendor']), \ + contextlib.redirect_stderr(stderr): + self.assertEqual(vendor_main.main(), 1) + runner.assert_not_called() + self.assertEqual(stderr.getvalue().splitlines(), [MESSAGE]) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/vendor/test_vendor_snapshot_restore_unit.py b/tests/vendor/test_vendor_snapshot_restore_unit.py index 0b30143d8..211f606a0 100644 --- a/tests/vendor/test_vendor_snapshot_restore_unit.py +++ b/tests/vendor/test_vendor_snapshot_restore_unit.py @@ -174,12 +174,13 @@ import unittest from unittest import mock -#: Same two-dependency gate the rest of :file:`tests/vendor/` uses. Importing +#: Same gate the rest of :file:`tests/vendor/` uses. Importing #: :mod:`pcapkit.vendor.__main__` imports :mod:`pcapkit.vendor` (for its -#: ``vendor_module`` default-target-list fallback), which pulls in every -#: crawler subpackage unconditionally -- :mod:`pcapkit.vendor.default` -#: itself imports :mod:`requests` at module scope and several crawlers import -#: :mod:`bs4` at module scope. +#: ``vendor_module`` default-target-list fallback), which imports every +#: crawler subpackage when :mod:`requests` and :mod:`bs4` are installed and +#: none otherwise (#1850) -- :mod:`pcapkit.vendor.default` itself imports +#: :mod:`requests` at module scope and several crawlers import :mod:`bs4` at +#: module scope. VENDOR_DEPS = ('requests', 'bs4') #: Whether the crawler machinery is importable at all. Both ship in the