fix(vendor): exit on one line without the vendor extra (#1850)

docs/source/pcapkit/vendor/index.rst

@@ -8,6 +8,19 @@ This module contains all web crawlers of :mod:`pcapkit`, which are
 automatically generating the :mod:`pcapkit.const` module's constant
 enumerations.
 
+The crawlers need the ``vendor`` extra, ``pip install "pypcapkit[vendor]"``.
+
+.. versionchanged:: 1.5.0
+
+   Without ``requests`` or ``beautifulsoup4``, which the crawlers import at
+   module scope, importing this package succeeds and imports no crawler.
+   Reading any name in its ``__all__``, or
+   :class:`~pcapkit.vendor.default.Vendor`, then raises
+   :exc:`ModuleNotFoundError` naming the missing packages and the extra to
+   install (:issue:`1850`). Importing a crawler module directly, such as
+   :mod:`pcapkit.vendor.reg`, still raises the missing package's own
+   :exc:`ModuleNotFoundError`. Before, importing this package raised it.
+
 Crawler Implementations
 =======================
 
@@ -95,3 +108,11 @@ constant enumerations.
    options:
      -h, --help     show this help message and exit
      -V, --version  show program's version number and exit
+
+.. versionchanged:: 1.5.0
+
+   Without ``requests`` or ``beautifulsoup4`` from the ``vendor`` extra,
+   :program:`pcapkit-vendor` writes one line to standard error naming the
+   missing packages and the extra to install, and exits with status ``1``
+   before any target runs (:issue:`1850`). It used to fail with a traceback.
+   ``--help`` and ``--version`` work without them.

pcapkit/vendor/__init__.py

@@ -9,51 +9,117 @@
 automatically generating the :mod:`pcapkit.const` module's constant
 enumerations.
 
+The crawlers need the ``vendor`` extra, ``pip install "pypcapkit[vendor]"``.
+
+.. versionchanged:: 1.5.0
+
+   Without ``requests`` or ``beautifulsoup4``, which the crawlers import at
+   module scope, importing this package succeeds and imports no crawler.
+   Reading any name in its ``__all__``, or
+   :class:`~pcapkit.vendor.default.Vendor`, then raises
+   :exc:`ModuleNotFoundError` naming the missing packages and the extra to
+   install, and :program:`pcapkit-vendor` exits with that message on one line
+   (:issue:`1850`). Importing a crawler module directly, such as
+   :mod:`pcapkit.vendor.reg`, still raises the missing package's own
+   :exc:`ModuleNotFoundError`. Before, importing this package raised it.
+
 """
+import typing
 
 from pcapkit.utilities.compat import ModuleNotFoundError  # pylint: disable=redefined-builtin
 from pcapkit.utilities.exceptions import stacklevel
 from pcapkit.utilities.warnings import VendorWarning, warn
 
+#: list[str]: Distributions of the ``vendor`` extra whose modules the crawlers
+#: import at module scope, and which cannot be imported here. While it is not
+#: empty no crawler is imported: :func:`__getattr__` raises for their names,
+#: and :func:`pcapkit.vendor.__main__.main` exits (:issue:`1850`). :mod:`pcapkit`
+#: has already put the conda package's :file:`pcapkit/_extern` on
+#: :data:`sys.path` by then, so a package vendored there counts as installed.
+_MISSING = []  # type: list[str]
+
 try:
     import requests
+except ModuleNotFoundError:
+    _MISSING.append('requests')
+
+try:
+    import bs4
+except ModuleNotFoundError:
+    _MISSING.append('beautifulsoup4')
+
+# The ``[socks]`` and ``[html5lib]`` extras of those packages are needed only at
+# run time, by a crawler that goes through a SOCKS proxy or parses HTML, so
+# their absence stays a warning.
+try:
     import socks
 except ModuleNotFoundError:
     warn("dependency package 'requests[socks]' not found",
          VendorWarning, stacklevel=stacklevel())
 
 try:
-    import bs4
     import html5lib
 except ModuleNotFoundError:
     warn("dependency package 'beautifulsoup4[html5lib]' not found",
          VendorWarning, stacklevel=stacklevel())
 
-# base crawler
-from pcapkit.vendor.default import Vendor
-
-# IANA registration
-from pcapkit.vendor.reg import *
-
-# Miscellanous
-from pcapkit.vendor.pcapng import *
-
-# per protocol
-from pcapkit.vendor.arp import *
-from pcapkit.vendor.esp import *
-from pcapkit.vendor.ftp import *
-from pcapkit.vendor.hip import *
-from pcapkit.vendor.http import *
-from pcapkit.vendor.ipv4 import *
-from pcapkit.vendor.ipv6 import *
-from pcapkit.vendor.ipx import *
-from pcapkit.vendor.l2tp import *
-from pcapkit.vendor.mh import *
-from pcapkit.vendor.ngap import *
-from pcapkit.vendor.ospf import *
-from pcapkit.vendor.sctp import *
-from pcapkit.vendor.tcp import *
-from pcapkit.vendor.vlan import *
+
+def _missing_message() -> 'str':
+    """Why no crawler was imported, naming the extra to install.
+
+    Returns:
+        The packages in :data:`_MISSING` and the extra that installs them, on
+        one line, or an empty string if :data:`_MISSING` is empty.
+
+    """
+    if not _MISSING:
+        return ''
+    return (f"missing dependencies: {', '.join(_MISSING)}; "
+            'install the vendor extra: pip install "pypcapkit[vendor]"')
+
+
+if _MISSING:
+    def __getattr__(name: 'str') -> 'typing.NoReturn':
+        """Raise for a crawler, or for its base class, which were not imported.
+
+        :pep:`562` calls this only for a name missing from this module's
+        globals, so it is defined only while :data:`_MISSING` is not empty.
+
+        Raises:
+            ModuleNotFoundError: If ``name`` is in ``__all__``, or is
+                ``Vendor``, naming the missing packages and the extra.
+            AttributeError: For any other name, as without this hook.
+
+        """
+        if name == 'Vendor' or name in __all__:
+            raise ModuleNotFoundError(f'cannot import {name!r} from {__name__!r}: {_missing_message()}')
+        raise AttributeError(f'module {__name__!r} has no attribute {name!r}')
+else:
+    # base crawler
+    from pcapkit.vendor.default import Vendor
+
+    # IANA registration
+    from pcapkit.vendor.reg import *
+
+    # Miscellanous
+    from pcapkit.vendor.pcapng import *
+
+    # per protocol
+    from pcapkit.vendor.arp import *
+    from pcapkit.vendor.esp import *
+    from pcapkit.vendor.ftp import *
+    from pcapkit.vendor.hip import *
+    from pcapkit.vendor.http import *
+    from pcapkit.vendor.ipv4 import *
+    from pcapkit.vendor.ipv6 import *
+    from pcapkit.vendor.ipx import *
+    from pcapkit.vendor.l2tp import *
+    from pcapkit.vendor.mh import *
+    from pcapkit.vendor.ngap import *
+    from pcapkit.vendor.ospf import *
+    from pcapkit.vendor.sctp import *
+    from pcapkit.vendor.tcp import *
+    from pcapkit.vendor.vlan import *
 
 # Crawlers only. This list is also the default target list of
 # :mod:`pcapkit.vendor.__main__`, which runs every name in it, so the base

pcapkit/vendor/__main__.py

@@ -24,6 +24,7 @@
 from pcapkit import vendor as vendor_module
 from pcapkit.utilities.logging import VERBOSE, get_logger
 from pcapkit.utilities.warnings import InvalidVendorWarning, VendorRuntimeWarning, warn
+from pcapkit.vendor import _missing_message
 
 if TYPE_CHECKING:
     from argparse import ArgumentParser
@@ -212,15 +213,30 @@ def main() -> 'int':
     from a real one at the process boundary, which is what CI actually checks.
 
     Returns:
-        ``0`` if every target ran to completion, ``1`` if any target raised.
+        ``0`` if every target ran to completion, ``1`` if any target raised, or
+        if a package the crawlers import at module scope is not installed.
 
     Warns:
         InvalidVendorWarning: If vendor target not found in :mod:`pcapkit.vendor` module.
 
+    .. versionchanged:: 1.5.0
+
+       Without ``requests`` or ``beautifulsoup4`` from the ``vendor`` extra,
+       writes one line to :data:`sys.stderr` naming the missing packages and
+       the extra to install, and returns ``1`` before any target runs
+       (:issue:`1850`). Importing this module used to raise
+       :exc:`ModuleNotFoundError` with a traceback. ``--help`` and
+       ``--version`` work without them.
+
     """
     parser = get_parser()
     args = parser.parse_args()
 
+    missing = _missing_message()
+    if missing:
+        print(f'{parser.prog}: error: {missing}', file=sys.stderr)
+        return 1
+
     target_list = []  # type: list[Type[Vendor]]
     for target in args.target:
         try:

tests/vendor/test_crawler_reachability_unit.py

@@ -79,12 +79,12 @@
 ROOT = pathlib.Path(__file__).resolve().parents[2]
 
 #: Whether the crawler machinery is importable at all. Importing
-#: :mod:`pcapkit.vendor` pulls in every subpackage unconditionally:
-#: :mod:`pcapkit.vendor.default` imports :mod:`requests` at module scope and
-#: several of the crawlers import :mod:`bs4` at module scope, so both are needed
-#: for this file rather than optional to it. (:mod:`pcapkit.vendor`'s own
-#: ``try: import bs4`` downgrades the *warning*, not the
-#: :exc:`ModuleNotFoundError` raised by the crawler imports below it.)
+#: :mod:`pcapkit.vendor` pulls in every subpackage only when :mod:`requests` and
+#: :mod:`bs4` are installed: :mod:`pcapkit.vendor.default` imports
+#: :mod:`requests` at module scope and several of the crawlers import :mod:`bs4`
+#: at module scope, so both are needed for this file rather than optional to it.
+#: (Without either, :mod:`pcapkit.vendor` imports no crawler and raises
+#: :exc:`ModuleNotFoundError` for a crawler's name, #1850.)
 #:
 #: Both ship in the ``test`` extra as well as ``vendor``
 #: (:file:`pyproject.toml`), so this guard should not fire in CI; it is kept as

tests/vendor/test_ipx_packet_unit.py

@@ -43,13 +43,12 @@
 #: Repository root, i.e. the grandparent of the directory holding this file.
 ROOT = pathlib.Path(__file__).resolve().parents[2]
 
-#: Every distribution importing :mod:`pcapkit.vendor` needs. ``requests`` is the
-#: obvious one -- ``pcapkit.vendor.default`` imports it at module scope -- but it
-#: is not sufficient: importing *any* crawler imports the ``pcapkit.vendor``
-#: package, whose :file:`__init__.py` pulls in all seventeen subpackages, seven of
-#: which ``import bs4`` at module scope. So a guard on ``requests`` alone lets the
-#: suite error instead of skipping on a machine that happens to have ``requests``
-#: and not ``beautifulsoup4``.
+#: Every distribution importing :mod:`pcapkit.vendor` with its crawlers needs.
+#: ``pcapkit.vendor.default`` imports ``requests`` at module scope, and several
+#: crawlers ``import bs4`` at module scope. Without either, the ``pcapkit.vendor``
+#: package imports none of its crawlers (#1850), so this file's crawler, which
+#: imports no ``bs4``, would import with ``requests`` alone; the gate names the
+#: whole set so the package imports here as it does with the ``vendor`` extra.
 VENDOR_DEPS = ('requests', 'bs4', 'html5lib')
 
 #: Whether the crawlers are importable at all. ``html5lib`` ships in the

tests/vendor/test_ipx_socket_unit.py

@@ -37,13 +37,12 @@
 #: Repository root, i.e. the grandparent of the directory holding this file.
 ROOT = pathlib.Path(__file__).resolve().parents[2]
 
-#: Whether the crawler machinery is importable at all. Two packages are needed
-#: and neither is optional to this file: ``pcapkit.vendor.default`` imports
-#: :mod:`requests` at module scope, and :mod:`pcapkit.vendor` then imports every
-#: crawler unconditionally, several of which import :mod:`bs4` at module scope --
-#: so importing *one* crawler needs both. (:mod:`pcapkit.vendor`'s own
-#: ``try: import bs4`` suppresses the warning, not the ``ModuleNotFoundError``
-#: raised by the crawler imports below it.)
+#: Whether the crawler machinery is importable at all. ``pcapkit.vendor.default``
+#: imports :mod:`requests` at module scope, and :mod:`pcapkit.vendor` imports its
+#: crawlers, several of which import :mod:`bs4` at module scope, only when
+#: :mod:`requests` and :mod:`bs4` are installed (#1850). This file's crawler
+#: imports no :mod:`bs4`, so :mod:`bs4` is in the gate for the package to import
+#: as it does in CI, not because the import would fail without it.
 #:
 #: Both ship in the ``test`` extra as well as ``vendor``
 #: (:file:`pyproject.toml`), because CI installs ``.[test]`` and a skipped pin

tests/vendor/test_request_prompt_unit.py

@@ -91,11 +91,13 @@
 #: Repository root, i.e. the grandparent of the directory holding this file.
 ROOT = pathlib.Path(__file__).resolve().parents[2]
 
-#: Every distribution importing :mod:`pcapkit.vendor` needs. Copied from
-#: :file:`tests/vendor/test_user_agent_unit.py`, and for its reason: ``requests``
-#: alone is not enough, because importing :mod:`pcapkit.vendor.default` imports the
-#: :mod:`pcapkit.vendor` package first and its :file:`__init__.py` pulls in
-#: subpackages that ``import bs4`` at module scope.
+#: Every distribution importing :mod:`pcapkit.vendor` with its crawlers needs.
+#: Copied from :file:`tests/vendor/test_user_agent_unit.py`, and for its reason:
+#: importing :mod:`pcapkit.vendor.default` imports the :mod:`pcapkit.vendor`
+#: package first, whose :file:`__init__.py` pulls in subpackages that
+#: ``import bs4`` at module scope only when ``requests`` and ``bs4`` are
+#: installed (#1850), and the gate keeps the package as the ``vendor`` extra
+#: imports it.
 VENDOR_DEPS = ('requests', 'bs4', 'html5lib')
 
 #: Whether the crawlers are importable at all. ``html5lib`` ships in the

tests/vendor/test_user_agent_unit.py

@@ -53,13 +53,14 @@
 #: Repository root, i.e. the grandparent of the directory holding this file.
 ROOT = pathlib.Path(__file__).resolve().parents[2]
 
-#: Every distribution importing :mod:`pcapkit.vendor` needs. ``requests`` is the
-#: obvious one -- ``pcapkit.vendor.default`` imports it at module scope -- but it
-#: is not sufficient: importing ``pcapkit.vendor.default`` imports the
-#: ``pcapkit.vendor`` package first, and its :file:`__init__.py` pulls in every
-#: subpackage, several of which ``import bs4`` at module scope. So a
-#: guard on ``requests`` alone lets the suite error instead of skipping on a
-#: machine that happens to have ``requests`` and not ``beautifulsoup4``.
+#: Every distribution importing :mod:`pcapkit.vendor` with its crawlers needs.
+#: ``pcapkit.vendor.default`` imports ``requests`` at module scope, and
+#: importing it imports the ``pcapkit.vendor`` package first, whose
+#: :file:`__init__.py` pulls in every subpackage, several of which
+#: ``import bs4`` at module scope, only when ``requests`` and ``bs4`` are
+#: installed (#1850). So ``requests`` alone would import
+#: ``pcapkit.vendor.default``; the gate names the whole set so the package
+#: imports here as it does with the ``vendor`` extra.
 VENDOR_DEPS = ('requests', 'bs4', 'html5lib')
 
 #: Whether the crawlers are importable at all. ``html5lib`` ships in the

tests/vendor/test_vendor_dest_path_unit.py

@@ -83,12 +83,13 @@
 :file:`tests/vendor/test_user_agent_unit.py` already use for the same reason.
 
 The suite is unit-tier (see :mod:`tests._tiers`): it reads no capture and makes
-no network call. Both classes are gated on :data:`HAS_VENDOR_DEPS`, the same
-two-dependency gate :file:`tests/vendor/test_crawler_reachability_unit.py`
-uses, because importing :mod:`pcapkit.vendor.default` first imports the
-:mod:`pcapkit.vendor` package, whose :file:`__init__.py` pulls in every
-crawler subpackage unconditionally -- several of which ``import bs4`` at module
-scope. ``html5lib`` is deliberately *not* in the gate: it appears nowhere at
+no network call. Its classes are gated on :data:`HAS_VENDOR_DEPS`, the same
+gate :file:`tests/vendor/test_crawler_reachability_unit.py` uses, because they
+read the crawlers off the :mod:`pcapkit.vendor` package, whose
+:file:`__init__.py` imports them only when ``requests`` and ``bs4`` are
+installed (#1850) -- :mod:`pcapkit.vendor.default` imports ``requests`` at
+module scope, and several crawlers ``import bs4`` at module scope.
+``html5lib`` is deliberately *not* in the gate: it appears nowhere at
 module scope, only as the string parser name in the crawlers'
 ``bs4.BeautifulSoup(text, 'html5lib')`` calls at actual crawl time, which
 nothing here reaches -- no test in this file calls :meth:`Vendor.__init__` or

tests/vendor/test_vendor_exit_code_unit.py

@@ -63,13 +63,14 @@
 import warnings
 from unittest import mock
 
-#: Same two-dependency gate :file:`tests/vendor/test_crawler_reachability_unit.py`
-#: and :file:`tests/vendor/test_vendor_dest_path_unit.py` use. Importing
+#: Same gate :file:`tests/vendor/test_crawler_reachability_unit.py` and
+#: :file:`tests/vendor/test_vendor_dest_path_unit.py` use. Importing
 #: :mod:`pcapkit.vendor.__main__` imports :mod:`pcapkit.vendor` (for the
 #: ``vendor_module`` default-target-list fallback this suite patches), which
-#: pulls in every crawler subpackage unconditionally --
-#: :mod:`pcapkit.vendor.default` imports :mod:`requests` at module scope and
-#: several crawlers import :mod:`bs4` at module scope.
+#: imports every crawler subpackage when :mod:`requests` and :mod:`bs4` are
+#: installed and none otherwise (#1850). :mod:`pcapkit.vendor.default`, whose
+#: :class:`~pcapkit.vendor.default.Vendor` this suite subclasses, imports
+#: :mod:`requests` at module scope.
 VENDOR_DEPS = ('requests', 'bs4')
 
 #: Whether the crawler machinery is importable at all. Both ship in the