docs(foundation): second pass over engines, toolkit and reassembly (#1103)

docs/source/pcapkit/dumpkit/common.rst

@@ -7,12 +7,6 @@ Common Utilities
 :mod:`pcapkit.dumpkit`, chiefly customised hooks for :class:`dictdumper.Dumper`
 classes.
 
-.. versionremoved:: 1.5.0
-
-   ``pcapkit.dumpkit.common.FALLBACK_LIMIT``, now
-   :data:`pcapkit.utilities.config.FALLBACK_LIMIT`, set with
-   :envvar:`PCAPKIT_FALLBACK_LIMIT` (:issue:`1726`).
-
 .. autofunction:: pcapkit.dumpkit.common.make_dumper
 
 .. autoclass:: pcapkit.dumpkit.common.Dumper

docs/source/pcapkit/foundation/reassembly/tcp.rst

@@ -236,7 +236,7 @@ Terminology
            |     |--> 'payload' : (tuple) partially reassembled payload
            |     |                 |--> (bytes) payload fragment
            |     |                 |--> ...
-           |     |--> 'packet' : (None) not implemented
+           |     |--> 'packet' : (None) not parsed
            |     |--> 'conflict' : (tuple) sequence ranges on which two segments disagreed
            |     |                  |--> (tuple) (first, last), absolute and inclusive
            |     |                  |--> ...
@@ -342,8 +342,6 @@ Data Models
 .. module:: pcapkit.foundation.reassembly.data.tcp
 .. currentmodule:: pcapkit.foundation.reassembly.tcp
 
-.. autodata:: pcapkit.foundation.reassembly.data.tcp.BufferID
-
 .. autoclass:: pcapkit.foundation.reassembly.data.tcp.Packet
    :members:
    :show-inheritance:

docs/source/pcapkit/toolkit/3rdparty.rst

@@ -182,12 +182,6 @@ cannot be used for it.
 
 .. _PyPCAPFile: https://github.com/kisom/pypcapfile
 
-.. versionremoved:: 1.5.0
-
-   ``pcapkit.toolkit.pypcapfile.IPV6_TCP_WARNED_LIMIT``, now
-   :data:`pcapkit.utilities.config.IPV6_TCP_WARNED_LIMIT`, set with
-   :envvar:`PCAPKIT_IPV6_TCP_WARNED_LIMIT` (:issue:`1726`).
-
 .. note::
 
    `PyPCAPFile`_ has no IPv6 decoder, so :func:`~pcapkit.toolkit.pypcapfile.ipv6_reassembly`

docs/source/pcapkit/utilities/chardet.rst

@@ -15,9 +15,3 @@ which is why it lives here rather than beside either of them.
 .. autofunction:: pcapkit.utilities.chardet.detect
 
 The cache holds :data:`~pcapkit.utilities.config.DETECT_CACHE_SIZE` entries.
-
-.. versionremoved:: 1.5.0
-
-   ``pcapkit.utilities.chardet.DETECT_CACHE_SIZE``, now
-   :data:`pcapkit.utilities.config.DETECT_CACHE_SIZE`, set with
-   :envvar:`PCAPKIT_DETECT_CACHE_SIZE` (:issue:`1726`).

pcapkit/__init__.py

@@ -20,7 +20,7 @@
 .. _DPKT: https://dpkt.readthedocs.io
 .. _PyShark: https://kiminewt.github.io/pyshark
 
-In :mod:`pcapkit`, all files can be described as following eight
+In :mod:`pcapkit`, all files can be described as following nine
 different components.
 
 - Interface (:mod:`pcapkit.interface`)
@@ -62,6 +62,11 @@
   Constant enumerations used in :mod:`pcapkit` for protocol
   family extraction and representation.
 
+- Vendor Crawlers (:mod:`pcapkit.vendor`)
+
+  Web crawlers that generate the constant enumerations in
+  :mod:`pcapkit.const`.
+
 """
 ###############################################################################
 # conda ``_extern`` module support

pcapkit/dumpkit/common.py

@@ -9,12 +9,6 @@
 generally the customised hooks for :class:`dictdumper.Dumper`
 classes.
 
-.. versionremoved:: 1.5.0
-
-   ``pcapkit.dumpkit.common.FALLBACK_LIMIT``, now
-   :data:`pcapkit.utilities.config.FALLBACK_LIMIT`, set with
-   :envvar:`PCAPKIT_FALLBACK_LIMIT` (:issue:`1726`).
-
 """
 import collections
 import datetime
@@ -320,8 +314,9 @@ def make_dumper(output: 'Type[ABCDumper]') -> 'Type[ABCDumper]':
        as the name, ``Flags::0 [0]`` rather than ``Flags::None [0]``
        (:issue:`648`). An :class:`~pcapkit.corekit.multidict.OrderedMultiDict`
        dumps as a list of one-key mappings in wire order, not grouped by key
-       (:issue:`1263`). A timezone dumps as its offset in seconds, and
-       :data:`~pcapkit.corekit.sentinels.NULL` and
+       (:issue:`1263`). An :class:`~ipaddress.IPv4Network` or
+       :class:`~ipaddress.IPv6Network` dumps as its string, a timezone as its
+       offset in seconds, and :data:`~pcapkit.corekit.sentinels.NULL` and
        :data:`~pcapkit.corekit.sentinels.NO_VALUE` as ``'<NULL>'`` and
        ``'<NO_VALUE>'`` (:issue:`1258`). In ``plist`` output, :data:`None` is
        kept (:issue:`1257`), and a string carrying a character XML forbids
@@ -601,7 +596,7 @@ def _append_dict(self, value: 'dict[Any, Any]', file: 'TextIO') -> 'None':
                     Each :data:`None` value is first converted through
                     :meth:`~dictdumper.plist.PLIST._encode_value`, whose own
                     rendering of it is ``{'type': 'NoneType', 'value': 'None'}``,
-                    so that the writer no longer skips it.
+                    so that the writer does not skip it.
 
                 """
                 value = {key: self._encode_value(val) if val is None else val for key, val in value.items()}

pcapkit/foundation/engines/dpkt.py

@@ -529,7 +529,7 @@ def read_frame(self) -> 'DPKTPacket':
         ext._frnum += 1
         ext._vfunc(ext, packet)
 
-        # write plist
+        # write output
         frnum = f'Frame {ext._frnum}'
         if not ext._flag_q:
             info = packet2dict(packet, timestamp, data_link=linktype)

pcapkit/foundation/engines/engine.py

@@ -244,8 +244,8 @@ class MyMixin(Engine):  # not registered
 
        Registration is opt-in, under the ``engine`` class keyword, which
        replaces 1.4.x's ``name`` with no alias; 1.4.x registered every
-       subclass, under its class name when ``name`` was omitted
-       (:issue:`514`).
+       subclass, under its :attr:`~EngineBase.name` (its ``__engine_name__``,
+       else its class name) when ``name`` was omitted (:issue:`514`).
 
     """
 

pcapkit/foundation/engines/pcap.py

@@ -168,7 +168,7 @@ def read_frame(self) -> 'Frame':
         # verbose output
         ext._vfunc(ext, frame)
 
-        # write plist
+        # write output
         frnum = f'Frame {ext._frnum}'
         if not ext._flag_q:
             # NOTE: every writer takes the frame's info: a PCAP writer for its

pcapkit/foundation/engines/pcap_ct.py

@@ -185,14 +185,14 @@ def unsupported_reason(cls) -> 'Optional[str]':
         Consulted by :meth:`pcapkit.foundation.extraction.Extractor.run` *before*
         the import test, and it answers two questions the import test cannot.
 
-        **Which distribution owns :mod:`pcap`.** Upstream `PyPCAP`_ owns that name
+        **Which distribution owns** :mod:`pcap`. Upstream `PyPCAP`_ owns that name
         just as legitimately, and ``import pcap._pcap`` -- what
         :attr:`__engine_module__` names -- separates them only as long as upstream
         keeps shipping a single extension module. Asking
         :func:`~pcapkit.foundation.engines._pcap_backend.probe` states the check
         rather than inferring it from an import that happens to fail.
 
-        **Whether a system :manpage:`libpcap(3)` exists at all.** This is the
+        **Whether a system** :manpage:`libpcap(3)` **exists at all.** This is the
         important one, and it is measured rather than theoretical. `pcap-ct`_ needs
         no compiler because it is :mod:`ctypes`, but at *runtime* it imports the
         `libpcap`_ distribution, whose Linux loader calls
@@ -209,7 +209,7 @@ def unsupported_reason(cls) -> 'Optional[str]':
         1.11.0b29 installs and reads :file:`examples/captures/in.pcap` identically
         on both, so this engine covers the whole range the project supports. The
         floor in the ``PCAP_CT`` extra's markers exists only because
-        :file:`pyproject.toml` still advertises ``requires-python >= 3.6``, and on
+        :file:`pyproject.toml` advertises ``requires-python >= 3.6``, and on
         an interpreter below 3.10 the distributions simply will not be installed --
         which the import test then reports correctly on its own.
 
@@ -442,7 +442,7 @@ def read_frame(self) -> 'RawFrame':
         ext._frnum += 1
         ext._vfunc(ext, frame)
 
-        # write plist
+        # write output
         frnum = f'Frame {ext._frnum}'
         if not ext._flag_q:
             info = packet2dict(packet, timestamp, data_link=self._dlink)

pcapkit/foundation/engines/pcapng.py

@@ -299,7 +299,7 @@ def read_frame(self) -> 'P_PCAPNG':
         # verbose output
         ext._vfunc(ext, block)
 
-        # write plist
+        # write output
         self._write_file(block.info, name=f'Frame {ext._frnum}')
 
         # record fragments

pcapkit/foundation/engines/pypcap.py

@@ -344,7 +344,7 @@ def read_frame(self) -> 'RawFrame':
         ext._frnum += 1
         ext._vfunc(ext, frame)
 
-        # write plist
+        # write output
         frnum = f'Frame {ext._frnum}'
         if not ext._flag_q:
             info = packet2dict(packet, timestamp, data_link=self._dlink)

pcapkit/foundation/engines/pypcapfile.py

@@ -252,7 +252,7 @@ def read_frame(self) -> 'PCAPFilePacket':
         ext._frnum += 1
         ext._vfunc(ext, packet)
 
-        # write plist
+        # write output
         frnum = f'Frame {ext._frnum}'
         if not ext._flag_q:
             info = packet2dict(packet, data_link=self._dlink)

pcapkit/foundation/engines/pyshark.py

@@ -298,7 +298,7 @@ def read_frame(self) -> 'PySharkPacket':
         ext._frnum += 1
         ext._vfunc(ext, packet)
 
-        # write plist
+        # write output
         frnum = f'Frame {ext._frnum}'
         if not ext._flag_q:
             info = packet2dict(packet)

pcapkit/foundation/engines/scapy.py

@@ -430,7 +430,7 @@ def read_frame(self) -> 'ScapyPacket':
         ext._frnum += 1
         ext._vfunc(ext, packet)
 
-        # write plist
+        # write output
         frnum = f'Frame {ext._frnum}'
         if not ext._flag_q:
             info = packet2dict(packet)

pcapkit/foundation/reassembly/ip.py

@@ -59,10 +59,11 @@ class IP(ReassemblyBase[Packet[_AT], Datagram[_AT], BufferID, Buffer[_AT]], Gene
 
     .. versionchanged:: 1.5.0
 
-       An overlapping fragment that disagrees with octets already received is
-       recorded in :attr:`Datagram.conflict
-       <pcapkit.foundation.reassembly.data.ip.Datagram.conflict>` rather than
-       overwriting them unnoticed (:issue:`477`). An incomplete buffer expires
+       An overlapping fragment that disagrees with octets already received
+       overwrites them, as :rfc:`791` asks, and the disagreement is recorded in
+       :attr:`Datagram.conflict
+       <pcapkit.foundation.reassembly.data.ip.Datagram.conflict>`
+       (:issue:`477`). An incomplete buffer expires
        after ``timeout`` seconds of capture time, 60 by default for both IPv4
        and IPv6.
 
@@ -417,7 +418,7 @@ def submit(self, buf: 'Buffer[_AT]', *, bufid: 'tuple[_AT, _AT, int, TransType]'
             Completion.TIMEOUT if timeout else Completion.PARTIAL
         )
 
-        # if datagram is not implemented
+        # if datagram is incomplete
         if not flag and self._flag_s:
             data = []  # type: list[bytes]
             byte = bytearray()

pcapkit/foundation/reassembly/ipv6.py

@@ -135,10 +135,9 @@ def _rectify_header(self, header: 'bytes', proto: 'TransType') -> 'bytes':
         the reassembled packet, and that the Next Header field of the last header
         of the unfragmentable part comes from the Fragment header's. Left alone,
         the reassembled datagram would advertise a Fragment header on a datagram
-        that is by definition no longer a fragment. The toolkit adapters differ
-        over whether the Fragment header's *octets* belong to ``header``, but none
-        rewrites the field that points at it; the octets are already excluded, so
-        only that field is left to fix.
+        that is by definition no longer a fragment. Every toolkit adapter leaves
+        the Fragment header's *octets* out of ``header``, but none rewrites the
+        field that points at it, so only that field is left to fix.
 
         Args:
             header: Raw header octets of the fragment at fragment offset zero.

pcapkit/foundation/reassembly/tcp.py

@@ -702,7 +702,7 @@ def submit(self, buf: 'Buffer', *, bufid: 'BufferID',  # type: ignore[override]
                 Completion.TIMEOUT if timeout else Completion.PARTIAL
             )
 
-            # if this buffer is not implemented
+            # if this buffer is incomplete
             # go through every hole and extract received payload
             if completion is not Completion.COMPLETE and self._flag_s:
                 data = []  # type: list[bytes]
@@ -731,7 +731,7 @@ def submit(self, buf: 'Buffer', *, bufid: 'BufferID',  # type: ignore[override]
                     )
                     datagram.append(packet)
 
-            # if this buffer is implemented -- or if it is not, and ``strict``
+            # if this buffer is complete -- or if it is not, and ``strict``
             # asked for one contiguous payload rather than the received runs
             #
             # NOTE: ``strict=False`` deliberately keeps reporting the whole

pcapkit/foundation/registry/protocols.py

@@ -199,9 +199,9 @@ class rather than supplied by a caller, and this function is the funnel
     name as a plain string, and the ``protocol`` properties on
     :class:`~pcapkit.foundation.reassembly.reassembly.ReassemblyMeta` and
     :class:`~pcapkit.foundation.traceflow.traceflow.TraceFlowMeta` fall back to
-    :class:`~pcapkit.protocols.misc.raw.Raw`. Re-keying is therefore part of the
-    registry redesign in :issue:`514`, and reporting the collision here is the
-    step that redesign is sequenced behind.
+    :class:`~pcapkit.protocols.misc.raw.Raw`. The registry redesign of
+    :issue:`514` kept this key, so the collision is reported here rather than
+    resolved.
 
     Args:
         protocol: Protocol class.

pcapkit/foundation/traceflow/data/data.py

@@ -68,8 +68,8 @@ def __call__(self) -> 'tuple[TCP_Datagram, ...]':
 class FrameRecord(dict):
     """A frame as a third-party engine's flow tracing adapter reports it.
 
-    The mapping is the engine's own dissection of the frame, exactly as before, so
-    every trace format that takes a mapping writes it unchanged. Beside it, as
+    The mapping is the engine's own dissection of the frame, which every trace
+    format that takes a mapping writes unchanged. Beside it, as
     attributes the mapping does not show, it carries what the PCAP trace dumper
     reads of a frame -- :meth:`PCAPIO._append_value
     <pcapkit.dumpkit.pcap.PCAPIO._append_value>` takes ``packet``,

pcapkit/toolkit/dpkt.py

@@ -291,7 +291,7 @@ def packet2frame(packet: 'Packet', timestamp: 'float | Decimal', *,
 
     Returns:
         The :func:`packet2dict` mapping of the packet, which every trace format
-        but PCAP writes as it always has, carrying the captured octets, from
+        but PCAP writes unchanged, carrying the captured octets, from
         :func:`packet2bytes`, the PCAP record header the PCAP trace dumper
         writes, and the exact timestamp it reads the header's resolution from.
 
@@ -857,13 +857,13 @@ def tcp_traceflow(packet: 'Packet', timestamp: 'float', *,
         count: Packet index. If not provided, default to ``-1``.
 
     Returns:
-        Data for TCP reassembly.
+        Data for TCP flow tracing.
 
-        * If the ``packet`` can be used for TCP flow tracing. A packet can be reassembled
+        * If the ``packet`` can be used for TCP flow tracing. A packet can be traced
           if its network layer carries TCP layer (:class:`dpkt.tcp.TCP`), behind any IPv6
           extension headers :mod:`dpkt` decodes. TCP in an IP-in-IP tunnel, or behind an AH or
           IPv6 extension header carried in IPv4, is not read.
-        * If the ``packet`` can be reassembled, then the :obj:`dict` mapping of data for TCP
+        * If the ``packet`` can be traced, then the :obj:`dict` mapping of data for TCP
           flow tracing (:term:`trace.tcp.packet`) will be returned; otherwise, returns :data:`None`.
 
     Note:

pcapkit/toolkit/pcap.py

@@ -200,7 +200,7 @@ def ipv4_reassembly(frame: 'Frame') -> 'IP_Packet[IPv4Address] | None':
         frame: PCAP frame.
 
     Returns:
-       Data for IPv4 reassembly.
+        Data for IPv4 reassembly.
 
         * If the ``frame`` can be used for IPv4 reassembly. A frame can be reassembled
           if it contains IPv4 layer (:class:`~pcapkit.protocols.internet.ipv4.IPv4`) and
@@ -246,7 +246,7 @@ def ipv6_reassembly(frame: 'Frame') -> 'IP_Packet[IPv6Address] | None':
         frame: PCAP frame.
 
     Returns:
-        A tuple of data for IPv6 reassembly.
+        Data for IPv6 reassembly.
 
         * If the ``frame`` can be used for IPv6 reassembly. A frame can be reassembled
           if it contains IPv6 layer (:class:`~pcapkit.protocols.internet.ipv6.IPv6`) and
@@ -310,7 +310,7 @@ def tcp_reassembly(frame: 'Frame') -> 'TCP_Packet | None':
         frame: PCAP frame.
 
     Returns:
-        A tuple of data for TCP reassembly.
+        Data for TCP reassembly.
 
         * If the ``frame`` can be used for TCP reassembly. A frame can be reassembled
           if it contains TCP layer (:class:`~pcapkit.protocols.transport.tcp.TCP`),
@@ -357,13 +357,13 @@ def tcp_traceflow(frame: 'Frame', *, data_link: 'LinkType') -> 'TF_TCP_Packet |
         data_link: Data link layer protocol (from global header).
 
     Returns:
-        Data for TCP reassembly.
+        Data for TCP flow tracing.
 
-        * If the ``packet`` can be used for TCP flow tracing. A frame can be reassembled
+        * If the ``frame`` can be used for TCP flow tracing. A frame can be traced
           if it contains TCP layer (:class:`~pcapkit.protocols.transport.tcp.TCP`),
           or a TCP segment whose header the TCP parser rejected, as it does one the
           snapshot length cut short (:issue:`1518`).
-        * If the ``frame`` can be reassembled, then the :obj:`dict` mapping of data for TCP
+        * If the ``frame`` can be traced, then the :obj:`dict` mapping of data for TCP
           flow tracing (:term:`trace.tcp.packet`) will be returned; otherwise, returns :data:`None`.
 
     See Also: