feat(engines): type the third-party engine packages with local stubs (#1878)

MANIFEST.in

@@ -10,6 +10,11 @@ prune .venv
 prune docs
 prune examples/captures
 prune temp
+# The engine stubs mypy reads (#1878) check this tree and are not for shipping.
+# Not load-bearing today: ``global-include *.py`` does not match a ``.pyi``,
+# measured by building an sdist without this line. It keeps them out should a
+# pattern above come to match one.
+prune typings
 
 # Belt-and-braces, both of them: setuptools ships these two whether or not this
 # file mentions them, so neither line is load-bearing. An earlier version of this

docs/source/contributing/conventions/process.rst

@@ -57,12 +57,25 @@ back to the default engine rather than raising when a requested engine is absent
 
    The ``dev`` extra exists **because** of this narrowing, and is not a second
    catch-all. pylint, mypy and autodoc all resolve imports against what is installed,
-   so the engines leaving ``all`` would have made them newly unresolvable to the
+   mypy for every package without a stub under :file:`typings/` (below), so the
+   engines leaving ``all`` would have made them newly unresolvable to the
    toolchain. ``dev`` is defined by what the toolchain must be able to *see*, and the
    workflows that need full resolution install ``.[all,dev]``. Do not add
    ``pypcap``/``pcap-ct`` to it to silence a lint finding; :file:`.github/workflows/lint.yml`
    carries a tracked ``import-error`` count that is deliberate rather than accidental.
 
+.. note::
+
+   mypy types the engine packages that ship no types -- ``dpkt``, ``pyshark``,
+   ``pcap`` and ``pcapfile`` -- from local stubs under :file:`typings/`, which
+   :file:`mypy.ini` registers through ``mypy_path`` and which ship in neither the
+   wheel nor the sdist (:issue:`1878`). ``scapy`` ships its own. A stub declares only
+   what :mod:`pcapkit` uses and cites the version it was written against, so a name
+   :mod:`pcapkit` starts to use goes into the stub first. Each engine's ``_expkg`` is
+   a :class:`~typing.Protocol` of what the engine reads through it, defined in its
+   module's ``if TYPE_CHECKING:`` block; :file:`tests/project/test_engine_stubs_1878_unit.py`
+   checks both against the code.
+
 Changelog Entry Granularity
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

mypy.ini

@@ -1,4 +1,7 @@
 [mypy]
+# Local stubs for the untyped engine packages pcapkit uses (#1878). Relative to
+# this file rather than to the working directory, which a bare path would be.
+mypy_path = $MYPY_CONFIG_FILE_DIR/typings
 ignore_missing_imports = True
 disallow_untyped_defs = True
 disallow_incomplete_defs = True

pcapkit/foundation/engines/dpkt.py

@@ -29,14 +29,18 @@
 __all__ = ['DPKT', 'PCAPNGReader']
 
 if TYPE_CHECKING:
-    from types import ModuleType
-    from typing import IO, Iterator, Literal, Optional, Type, Union
+    from typing import IO, Iterator, Literal, Optional, Protocol, Type, Union
 
     from dpkt.dpkt import Packet as DPKTPacket
+    from dpkt.ethernet import Ethernet as DPKTEthernet
+    from dpkt.ip import IP as DPKTIP
+    from dpkt.ip6 import IP6 as DPKTIP6
+    from dpkt.loopback import Loopback as DPKTLoopback
     from dpkt.pcap import Reader as PCAPReader
 
     from pcapkit.foundation.engines.pcapng import Context
     from pcapkit.foundation.extraction import Extractor
+    from pcapkit.protocols.data.misc.pcapng import IF_TSOffsetOption as Data_IF_TSOffsetOption
     from pcapkit.protocols.data.misc.pcapng import IF_TSResolOption as Data_IF_TSResolOption
     from pcapkit.protocols.data.misc.pcapng import \
         InterfaceDescriptionBlock as Data_InterfaceDescriptionBlock
@@ -45,6 +49,57 @@
 
     Reader = Union[PCAPReader, 'PCAPNGReader']
 
+    class _DPKTCoreModule(Protocol):
+        """:mod:`dpkt.dpkt`, as :class:`DPKT` reads it."""
+
+        @property
+        def Packet(self) -> 'Type[DPKTPacket]': ...
+
+    class _DPKTEthernetModule(Protocol):
+        """:mod:`dpkt.ethernet`, as :class:`DPKT` reads it."""
+
+        @property
+        def Ethernet(self) -> 'Type[DPKTEthernet]': ...
+
+    class _DPKTIPModule(Protocol):
+        """:mod:`dpkt.ip`, as :class:`DPKT` reads it."""
+
+        @property
+        def IP(self) -> 'Type[DPKTIP]': ...
+
+    class _DPKTIP6Module(Protocol):
+        """:mod:`dpkt.ip6`, as :class:`DPKT` reads it."""
+
+        @property
+        def IP6(self) -> 'Type[DPKTIP6]': ...
+
+    class _DPKTLoopbackModule(Protocol):
+        """:mod:`dpkt.loopback`, as :class:`DPKT` reads it."""
+
+        @property
+        def Loopback(self) -> 'Type[DPKTLoopback]': ...
+
+    class _DPKTPackage(Protocol):
+        """:mod:`dpkt`, as :class:`DPKT` reads it through :attr:`DPKT._expkg` (#1878).
+
+        Each attribute is a submodule, described by what the engine reads of it.
+        mypy matches a module with a protocol at the top level only, so the
+        submodules are checked through the stub types they name, under
+        :file:`typings/dpkt`.
+
+        """
+
+        @property
+        def dpkt(self) -> '_DPKTCoreModule': ...
+        @property
+        def ethernet(self) -> '_DPKTEthernetModule': ...
+        @property
+        def ip(self) -> '_DPKTIPModule': ...
+        @property
+        def ip6(self) -> '_DPKTIP6Module': ...
+        @property
+        def loopback(self) -> '_DPKTLoopbackModule': ...
+
 #: logging.Logger: Module-level logger, a child of the package-wide
 #: :data:`pcapkit.utilities.logging.logger`.
 logger = get_logger(__name__)
@@ -182,8 +237,10 @@ def _read_packet(self, buf: 'bytes', block_type: 'int',
             raise FormatError(f'PCAP-NG: [{tag}] invalid interface ID: {iface_id}')
         interface = self._ctx.interfaces[iface_id]
 
-        tsresol = interface.options.get(Enum_OptionType.if_tsresol)
-        tsoffset = interface.options.get(Enum_OptionType.if_tsoffset)
+        tsresol = cast('Optional[Data_IF_TSResolOption]',
+                       interface.options.get(Enum_OptionType.if_tsresol))
+        tsoffset = cast('Optional[Data_IF_TSOffsetOption]',
+                        interface.options.get(Enum_OptionType.if_tsoffset))
         resolution = DEFAULT_RESOLUTION if tsresol is None else tsresol.resolution
         offset = 0 if tsoffset is None else tsoffset.offset
 
@@ -268,7 +325,7 @@ def _pcap_reader() -> 'Type[PCAPReader]':
     #: Attributes of :class:`dpkt.pcap.Reader` the loop below reads, as of dpkt 1.9.8.
     private = ('_Reader__f', '_Reader__ph', '_divisor')
 
-    class _Reader(dpkt.pcap.Reader):  # type: ignore[name-defined,misc]
+    class _Reader(dpkt.pcap.Reader):
         def __init__(self, fileobj: 'IO[bytes]') -> 'None':
             super().__init__(fileobj)
             missing = [name for name in private if not hasattr(self, name)]
@@ -284,8 +341,9 @@ def __init__(self, fileobj: 'IO[bytes]') -> 'None':
         # NOTE: the loop is :meth:`dpkt.pcap.Reader.__iter__`'s, over the stream
         # and the record header type that method reads them through. A generator,
         # so ``dpkt.pcap.Reader.__init__`` calling it runs none of it before
-        # :attr:`_upstream` is set.
-        def __iter__(self) -> 'Iterator[tuple[float, bytes]]':
+        # :attr:`_upstream` is set. The timestamp is a ``Decimal`` for a
+        # nanosecond file, from upstream's loop and from this one alike.
+        def __iter__(self) -> 'Iterator[tuple[float | decimal.Decimal, bytes]]':
             if self._upstream:
                 yield from super().__iter__()
                 return
@@ -347,7 +405,10 @@ def _loopback_protocol(base: 'Type[DPKTPacket]', ip: 'Type[DPKTPacket]', ip6: 'T
 
     layers = {2: ip, 24: ip6, 28: ip6, 30: ip6}
 
-    class Loopback(base):
+    # ``base`` is an argument so that the class derives from the given module's
+    # own ``Loopback``; mypy takes a base class only from a name it can resolve
+    # statically
+    class Loopback(base):  # type: ignore[valid-type,misc]
         """BSD loopback encapsulation."""
 
         def unpack(ext, buf: 'bytes') -> 'None':
@@ -382,7 +443,7 @@ class DPKT(EngineBase['DPKTPacket']):
     """
     if TYPE_CHECKING:
         #: Engine extraction package, i.e. :mod:`dpkt`.
-        _expkg: 'ModuleType'
+        _expkg: '_DPKTPackage'
         #: Engine extraction temporary storage.
         _extmp: 'Reader'
 
@@ -464,7 +525,7 @@ def run(self) -> 'None':
 
         if ext.magic_number in PCAP.MAGIC_NUMBER:
             logger.debug('dpkt: reading %s as PCAP', ext._ifnm)
-            reader = _pcap_reader()(ext._ifile)
+            reader = _pcap_reader()(ext._ifile)  # type: Reader
         elif ext.magic_number in PCAPNG.MAGIC_NUMBER:
             logger.debug('dpkt: reading %s as PCAP-NG', ext._ifnm)
             reader = PCAPNGReader(ext._ifile)
@@ -594,6 +655,7 @@ def _get_protocol(self, linktype: 'Optional[Enum_LinkType]' = None,
             linktype = Enum_LinkType.get(reader.datalink())
         version = data[0] >> 4 if data else None
 
+        pkg: 'Type[DPKTPacket]'
         if linktype == Enum_LinkType.ETHERNET:
             pkg = dpkt.ethernet.Ethernet
         elif linktype.value == Enum_LinkType.IPV4 or (linktype.value == Enum_LinkType.RAW and version == 4):

pcapkit/foundation/engines/pcap_ct.py

@@ -24,13 +24,18 @@
 __all__ = ['PCAP_CT']
 
 if TYPE_CHECKING:
-    from types import ModuleType
-    from typing import Iterator, Optional
+    from typing import Iterator, Optional, Protocol, Type
 
     from pcap import pcap as Handle
 
     from pcapkit.foundation.extraction import Extractor
 
+    class _PCAPCTPackage(Protocol):
+        """:mod:`pcap`, as :class:`PCAP_CT` reads it through :attr:`PCAP_CT._expkg` (#1878)."""
+
+        @property
+        def pcap(self) -> 'Type[Handle]': ...
+
     #: A `pcap-ct`_ "frame": the ``(timestamp, bytes)`` pair that
     #: :class:`pcap.pcap` iteration yields. Deliberately *not* named ``Frame``,
     #: so that it is not mistaken for
@@ -113,7 +118,7 @@ class PCAP_CT(EngineBase['RawFrame']):
     """
     if TYPE_CHECKING:
         #: Engine extraction package, i.e. :mod:`pcap`.
-        _expkg: 'ModuleType'
+        _expkg: '_PCAPCTPackage'
         #: Capture handle, kept separately from the iterator it is read through
         #: so that :meth:`close` does not depend on ``iter(handle) is handle``.
         _handle: 'Handle'
@@ -395,7 +400,7 @@ def run(self) -> 'None':
         # want that attempt to request promiscuous mode. (That fallback is also
         # why a bad path surfaces as ``OSError: ... No such device exists``,
         # which is what the ``os.path.isfile`` check above pre-empts.)
-        self._handle = cast('Handle', self._expkg.pcap(name=ext._ifnm, promisc=False))
+        self._handle = self._expkg.pcap(name=ext._ifnm, promisc=False)
         self._dlink = Enum_LinkType.get(self._handle.datalink())
 
         # setup verbose handler

pcapkit/foundation/engines/pypcap.py

@@ -31,13 +31,18 @@
 __all__ = ['PyPCAP']
 
 if TYPE_CHECKING:
-    from types import ModuleType
-    from typing import Iterator, Optional
+    from typing import Iterator, Optional, Protocol, Type
 
     from pcap import pcap as Handle
 
     from pcapkit.foundation.extraction import Extractor
 
+    class _PyPCAPPackage(Protocol):
+        """:mod:`pcap`, as :class:`PyPCAP` reads it through :attr:`PyPCAP._expkg` (#1878)."""
+
+        @property
+        def pcap(self) -> 'Type[Handle]': ...
+
     #: A PyPCAP "frame": the ``(timestamp, bytes)`` pair that :class:`pcap.pcap`
     #: iteration yields. Deliberately *not* named ``Frame``, so that it is not
     #: mistaken for :class:`pcapkit.protocols.misc.pcap.frame.Frame`, which is
@@ -88,7 +93,7 @@ class PyPCAP(EngineBase['RawFrame']):
     """
     if TYPE_CHECKING:
         #: Engine extraction package, i.e. :mod:`pcap`.
-        _expkg: 'ModuleType'
+        _expkg: '_PyPCAPPackage'
         #: Engine extraction temporary storage.
         _extmp: 'Iterator[RawFrame]'
         #: Data link layer protocol, from the capture handle.
@@ -308,7 +313,7 @@ def run(self) -> 'None':
         # ``pcap.pcap`` ignores it, but should ``pcap_open_offline`` ever fail the
         # constructor falls through to opening the name as a live device, and we
         # do not want that attempt to request promiscuous mode.
-        handle = cast('Handle', self._expkg.pcap(name=ext._ifnm, promisc=False))
+        handle = self._expkg.pcap(name=ext._ifnm, promisc=False)
         self._dlink = Enum_LinkType.get(handle.datalink())
 
         # setup verbose handler

pcapkit/foundation/engines/pypcapfile.py

@@ -27,14 +27,49 @@
 __all__ = ['PyPCAPFile']
 
 if TYPE_CHECKING:
-    from types import ModuleType
-    from typing import Any, Callable, Iterator, Optional
+    from typing import Any, Callable, Iterator, Optional, Protocol, Type
 
+    from pcapfile.protocols.linklayer.ethernet import Ethernet as PCAPFileEthernet
+    from pcapfile.savefile import pcap_savefile as PCAPFileSavefile
     from pcapfile.structs import __pcap_header__ as PCAPFileHeader
     from pcapfile.structs import pcap_packet as PCAPFilePacket
 
     from pcapkit.foundation.extraction import Extractor
 
+    class _PCAPFileLinkLayerModule(Protocol):
+        """:mod:`pcapfile.linklayer`, as :class:`PyPCAPFile` reads it."""
+
+        def clookup(self, ll_type: 'int') -> 'Optional[Type[PCAPFileEthernet]]': ...
+
+    class _PCAPFileSavefileModule(Protocol):
+        """:mod:`pcapfile.savefile`, as :class:`PyPCAPFile` reads it."""
+
+        def load_savefile(self, input_file: 'NamedStream', layers: 'int' = ...,
+                          verbose: 'bool' = ..., lazy: 'bool' = ...) -> 'PCAPFileSavefile': ...
+
+    class _PCAPFileStructsModule(Protocol):
+        """:mod:`pcapfile.structs`, as :class:`PyPCAPFile` reads it."""
+
+        @property
+        def pcap_packet(self) -> 'Type[PCAPFilePacket]': ...
+
+    class _PCAPFilePackage(Protocol):
+        """:mod:`pcapfile`, as :class:`PyPCAPFile` reads it through :attr:`PyPCAPFile._expkg` (#1878).
+
+        Each attribute is a submodule, described by what the engine reads of it.
+        mypy matches a module with a protocol at the top level only, so the
+        submodules are checked through the stub types they name, under
+        :file:`typings/pcapfile`.
+
+        """
+
+        @property
+        def linklayer(self) -> '_PCAPFileLinkLayerModule': ...
+        @property
+        def savefile(self) -> '_PCAPFileSavefileModule': ...
+        @property
+        def structs(self) -> '_PCAPFileStructsModule': ...
+
 
 class PyPCAPFile(EngineBase['PCAPFilePacket']):
     """PyPCAPFile engine support.
@@ -70,7 +105,7 @@ class PyPCAPFile(EngineBase['PCAPFilePacket']):
     """
     if TYPE_CHECKING:
         #: Engine extraction package, i.e. :mod:`pcapfile`.
-        _expkg: 'ModuleType'
+        _expkg: '_PCAPFilePackage'
         #: Engine extraction temporary storage.
         _extmp: 'Iterator[PCAPFilePacket]'
         #: Data link layer protocol, from the savefile header.
@@ -372,9 +407,9 @@ def _read_records(self, file: 'NamedStream',
                 warn(f'PCAP: [Frame {number}] captured length {capture_len} runs past the data, '
                      f'which holds {len(data)} octet(s); kept as captured',
                      ProtocolWarning, stacklevel=stacklevel())
-            yield cast('PCAPFilePacket', self._expkg.structs.pcap_packet(
+            yield self._expkg.structs.pcap_packet(
                 hdrp, timestamp, timestamp_us, capture_len, packet_len, binascii.hexlify(data),
-            ))
+            )
 
     def _decode(self, packet: 'PCAPFilePacket', frnum: 'int') -> 'PCAPFilePacket':
         """Decode a raw savefile packet down to :attr:`LAYERS` depth.
@@ -417,7 +452,7 @@ def _decode(self, packet: 'PCAPFilePacket', frnum: 'int') -> 'PCAPFilePacket':
                  'frame left undecoded', AttributeWarning, stacklevel=stacklevel())
             return packet
 
-        return cast('PCAPFilePacket', self._expkg.structs.pcap_packet(
+        return self._expkg.structs.pcap_packet(
             packet.header, packet.timestamp, packet.timestamp_us,
             packet.capture_len, packet.packet_len, decoded,
-        ))
+        )