From 3f07b5ef3689c3c7e0966e6c6f4e9980d65a4eae Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Thu, 15 Oct 2026 13:02:07 -0400 Subject: [PATCH] chore: drop stale build leftovers and move the examples' READMEs to Markdown (#1773) - setup.py: drop the bdist_wheel override, which converted dist_dir/pcapkit after the wheel was written; build_py and install already convert the tree bdist_wheel archives. test_setup.py loses its now-dead bdist_wheel fake. - setup.py: the f2format and poseur failures name f-string and positional-only parameter backports, not assignment expressions. - Makefile: `profile` delegates to examples/legacy_smoke (moved in 78e28d811). - MANIFEST.in: drop `prune test` (no such directory); add `recursive-include examples *.md`. `global-include *.rst` now matches nothing shipped (sdist identical without it); left for a later pass. - examples/{benchmark,legacy_smoke}/README.rst -> README.md; .gitignore, benchmark.py, the tier test and the documentation conventions page follow. Sdist listings differ only by the renames and the new test; wheels match. test_build_leftovers_1773.py fails with each fix reverted; tests/project green. Closes #1773 --- .gitignore | 2 +- MANIFEST.in | 3 +- Makefile | 4 +- .../conventions/documentation.rst | 8 +- examples/benchmark/README.md | 470 +++++++++++++++ examples/benchmark/README.rst | 536 ------------------ examples/benchmark/benchmark.py | 2 +- examples/legacy_smoke/README.md | 126 ++++ examples/legacy_smoke/README.rst | 156 ----- setup.py | 25 +- tests/project/test_build_leftovers_1773.py | 264 +++++++++ .../project/test_python_matrix_tiers_1633.py | 2 +- tests/project/test_setup.py | 1 - 13 files changed, 877 insertions(+), 722 deletions(-) create mode 100644 examples/benchmark/README.md delete mode 100644 examples/benchmark/README.rst create mode 100644 examples/legacy_smoke/README.md delete mode 100644 examples/legacy_smoke/README.rst create mode 100644 tests/project/test_build_leftovers_1773.py diff --git a/.gitignore b/.gitignore index 4dc889a4a0..7ff2aba208 100644 --- a/.gitignore +++ b/.gitignore @@ -18,7 +18,7 @@ examples/captures/* # PCAP-NG packet blocks their captured octets back. Regenerating them on every such # change is work with no reader, and the tier guard in `tests/_tiers.py` asks git # which captures are tracked rather than hardcoding a list, so dropping them here -# is the whole change -- see `examples/legacy_smoke/README.rst` for how to rebuild +# is the whole change -- see `examples/legacy_smoke/README.md` for how to rebuild # them locally. !examples/captures/in.pcap !examples/captures/dhcp.pcapng diff --git a/MANIFEST.in b/MANIFEST.in index b1aff2a5ff..75f7cc0e79 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,6 +2,8 @@ global-exclude .DS_Store global-include *.rst global-include *.py +# Neither global-include reaches Markdown, which the examples' READMEs are (#719). +recursive-include examples *.md prune .eggs prune .github @@ -9,7 +11,6 @@ prune .venv prune docs prune examples/captures prune temp -prune test # Belt-and-braces, both of them: setuptools ships these two whether or not this # file mentions them, so neither line is load-bearing. An earlier version of this diff --git a/Makefile b/Makefile index 1ed79d3bc0..dd328d61c0 100644 --- a/Makefile +++ b/Makefile @@ -193,8 +193,10 @@ mypy: bandit: $(RUN) bandit $(BANDIT_FLAGS) pcapkit +# Profile one extraction and render its call graph; the tools the recipe needs +# are listed in examples/legacy_smoke/README.md. profile: - $(MAKE) -C test profile + $(MAKE) -C examples/legacy_smoke profile mypy-types: pipenv run mypy --install-types diff --git a/docs/source/contributing/conventions/documentation.rst b/docs/source/contributing/conventions/documentation.rst index 36165d8755..04315fd9cb 100644 --- a/docs/source/contributing/conventions/documentation.rst +++ b/docs/source/contributing/conventions/documentation.rst @@ -289,10 +289,10 @@ Format and Mechanics ruled this on :issue:`719`, correcting a blanket *always* ``.rst`` that had been in circulation: the Sphinx documentation is reST, and the other documents -- the READMEs included -- are Markdown where that applies. ``CONTRIBUTING.md``'s own - *Documentation* section records the same split. One trap arrived with the ruling: - :file:`MANIFEST.in` reaches the two READMEs under :file:`examples/` through - ``global-include *.rst`` and has no ``*.md`` equivalent, so renaming one without - adding coverage drops it from every source distribution. + *Documentation* section records the same split. Neither ``global-include`` in + :file:`MANIFEST.in` reaches Markdown, so the two READMEs under :file:`examples/` + ship through ``recursive-include examples *.md``, and any other Markdown file + ships only if setuptools or an ``include`` line adds it. * **Prose names the real defining module, not a re-export.** One strand of the consistency sweep on :doc:`the roadmap `. The sentinels are the worked example: all four are defined in :mod:`pcapkit.corekit.sentinels`, and diff --git a/examples/benchmark/README.md b/examples/benchmark/README.md new file mode 100644 index 0000000000..267d2d214b --- /dev/null +++ b/examples/benchmark/README.md @@ -0,0 +1,470 @@ +# Engine Benchmark Suite + +A containerised benchmark that produces the engine speed table in the project docs, +`docs/source/index.rst` -- the whole table, every supported Python version, in one +run. One command, on any machine with Docker, and the output is two paste-ready +reStructuredText snippets: + +```shell +examples/benchmark/run.sh +``` + +It exists because that table could not previously be completed on **any** single +machine, and because the numbers in it were not reproducible. Both problems are the +same problem -- the environment was never pinned and could not hold every engine at +once -- so the fix is to define the environment precisely and ship it. Once the +environment is a container, the number of them is a loop, which is what makes a +complete table a single command rather than five hand-run measurements pasted +together. + +## The matrix + +**One image per Python version**, five of them by default, each pinned by digest in +[python-images.txt](python-images.txt) -- which is the matrix, and is a table +rather than logic so that a reader can check it. Inside each image, **one or two +virtualenvs**, so an "environment" here is a *(Python version, pcap provider)* +pair: + +| Python | environments | why | +| --- | --- | --- | +| 3.10, 3.11 | `-pypcap` and `-pcap_ct` | both `pcap` providers install, and they are mutually exclusive | +| 3.12, 3.13, 3.14 | `-pcap_ct` only | `pypcap` cannot be installed at all, so a second virtualenv would compile for minutes to produce nothing | +| 3.15 | opt-in, `-pcap_ct` only | no released image; see below | + +Seven environments for a default run, and a version named with `--pythons` that is +not in the file is a typo rather than a column. + +## Why this cannot be one virtualenv, or one Python + +Two hard constraints shape everything here. Neither is a `pcapkit` defect and +neither can be worked around from inside `pcapkit`: + +1. **pypcap and pcap-ct are mutually exclusive.** Both distributions install + a top-level module named `pcap`. With both present, `import pcap` resolves to + `pcap-ct`'s package directory and upstream's extension module is shadowed and + unreachable, so `engine='pypcap'` cannot run at all. No install order changes + that. **So no single environment can hold all seven engines**, and an image that + can hold `pypcap` carries two virtualenvs -- `/opt/venv-pypcap` and + `/opt/venv-pcap_ct`, identical but for which `pcap` provider they have. +2. **3.11 is the** *last* **interpreter on which every engine runs**, and 3.10 is the + other one. `pypcap` 1.3.0 ships a `pcap.c` pre-generated by Cython 0.29.32 + that will not compile against the 3.12+ C API; `pypcapfile` 0.12.0 imports + `imp`, removed in 3.12; `pyshark` 0.6 calls + `asyncio.get_event_loop_policy().get_event_loop()`, which raises from 3.14. + **That is a fact the table reports rather than a constraint on the run**: each + engine's ceiling shows up as `--` cells with the reason, in the columns past it. + +The `default` engine -- `pcapkit`'s own parser, also spelled `pcapkit` -- is +present in every virtualenv of every image, and is what joins all of them into one +table. See "How the runs are joined" below. + +## Two tables, because there are two questions + +The run emits both, and neither replaces the other: + +| File | Question it answers | Arithmetic | +| --- | --- | --- | +| `table-versions.rst` | how does each engine do on each interpreter? | **absolute** milliseconds per packet, one column per Python version | +| `table.rst` | which engine is faster, on any machine? | **ratios** against `default`, pooled across the whole matrix | + +**The per-version table is absolute, and must be.** Every column of it came out of +one run on one host, differing in the interpreter and nothing else, which is +precisely the condition under which absolute times are comparable -- with each other. +A ratio there would be worse than redundant: ratios are normalised *within* one +environment, so dividing across two interpreters yields a number that is neither +engine's speed nor the interpreter's. The snippet says so in its own prose, because +the person reading the table in the docs later is not the person who read this +file. + +**The ratio table is normalised to the default engine measured in the same +virtualenv on the same pass.** An absolute timing describes the machine it was taken +on at least as much as it describes the engine, so a table of absolute numbers cannot +be reproduced across machines -- which is exactly the state the old table was in. A +ratio divides the machine out, and pooling it across every interpreter is what makes +it an answer about engines rather than about one interpreter. The observed range then +includes any disagreement *between* interpreters, which is real uncertainty about the +engine and is not something to average away; the per-version table is where it gets +broken out. + +One absolute figure survives inside the ratio snippet, as a footnote: the `default` +engine's own milliseconds per packet. That is what lets a reader convert the ratios +back into times on the machine the run happened on. + +Both snippets carry their own `Test Results` heading -- the per-version one takes +the name, since it is what the docs' **Test Results** section is, and the ratio +one is `Test Results (Relative)` so that pasting both does not produce two sections +with one name. The `Test Environment` block, which serves both, is in +`table.rst`. + +**The run-to-run spread is reported too.** A single pass cannot tell a real difference +between two engines from the machine having been briefly busy, so the whole engine set +is measured several times (`--repeats`, default 3) and every row carries the range +actually observed. Rows whose ranges overlap are marked `‡`: this run does not +establish which of them is faster, and printing their medians as though it did would +be inventing precision. The footnote states the median and the widest spread seen, so +a reader can judge any gap in the table for themselves. + +## The property that matters most + +**An engine that cannot run does not raise.** `Extractor.run` emits an +`EngineWarning` and falls back to `pcapkit`'s own parser, so an extraction that +appears to succeed can be timing something entirely different from what was asked +for. A benchmark that does not check this will happily report the default engine's +speed three times over under three different names, and nothing in the output would +look wrong. + +So the harness checks `type(extractor.engine).__engine_name__` on **every single +extraction**, not just the first, against a list of expected drivers derived from +`Extractor.__engine__` rather than written out by hand. A run whose driver ever +disagrees raises and is discarded; no number is reported for it. This is asserted, not +assumed, and it is the single most important correctness property of the suite. + +An engine that genuinely cannot run here is recorded as `*not measured*` with the +reason the engine itself gives through `unsupported_reason()` -- never omitted, and +never as a zero. + +## One flaky engine must not cost the others their rows + +This is not hypothetical; it is what the first full run of this harness did. +`pyshark` spawns a `tshark` process per extraction, so a default run spawns **six +thousand** of them, and one of them died -- `TSharkCrashException ... retcode: 255`, +on the third pass. The exception propagated, `benchmark.py` exited, no JSON was +written, and the run produced **no table at all**. Six working engines went unreported +because the seventh hiccupped once. + +So failure is contained at two levels, and counted at both: + +- **A failed extraction** is discarded and the pass carries on, up to + `TOLERATED_FAILURES` (5) per engine per pass. A crashed extraction is a + *non-measurement*, not a slow measurement, so discarding its sample is correct -- + what would not be honest is discarding it silently, so the count and the first + reason are reported. +- **A failed pass** costs that engine that pass, not the run. The row is built from + the passes that did work, its `Passes` count shows fewer, and it is marked `¶` + with what was lost. If *every* pass failed, the engine becomes `*not measured*` + with the exception as its reason. + +Two things are never tolerated, because tolerating them would mean reporting a number +that is not true: the escalated `EngineWarning` that means the engine fell back to +`pcapkit`'s parser, and a capture that yielded no packets. Both end the run. + +The same principle covers the build. `pypcap` is the only engine in the set that +compiles, and therefore the only one whose *install* can fail for reasons these pins +do not control. A fatal failure there would mean no table at all, so the Dockerfile +records the pip error to `/opt/install-failures/pypcap.txt` and carries on; +`benchmark.py` reads it and reports the compiler error as the reason `pypcap` was +not measured. Everything else in the image is a wheel on every architecture *and every +interpreter in the matrix* -- checked on 3.10, 3.12, 3.13 and 3.14 as well as 3.11 -- +so a failure installing it is a real error and stays fatal. + +## A missing column is held to the same standard as a missing row + +With five images there are five chances for something outside these pins to go wrong: +a base image withdrawn, a wheel that stops publishing for one interpreter, a daemon +that runs out of disk halfway through. **None of those ends the run**, and none of +them silently narrows the table: + +- `run.sh` catches the failure per version, writes the reason to + `out/missing/.txt`, and carries on to the next one. That is the same + mechanism `/opt/install-failures/` uses for `pypcap`, one level up. +- the reporting pass turns each of those files into a column of `--` **with the + reason attached** -- in the per-version table's notes, and in the `Test + Environment` block as a `Python 3.15 (not measured)` row, so neither snippet + reads as a complete matrix when it is not. +- only a run in which *every* version failed exits without a table, since then there + is genuinely nothing to report. + +**The exit status says which happened**, because a scheduled invocation reads nothing +else: `0` when every requested version was measured, `3` **when the tables were +written but a version was lost**, `1` when no table was produced at all, and `2` +for bad arguments. A `make bench` that measured one column out of five is not a +failure -- the tables are there -- but it is not success either, and it should not be +indistinguishable from it. + +**pypcap on 3.12+ is deliberately not reported this way.** It did not fail; it +cannot be installed, which is a fact about the interpreter rather than about this +build. The image records that in `/opt/not-attempted/pypcap.txt`, and that note +*replaces* the engine's own `unsupported_reason()` -- which on those images says the +installed `pcap` module is `pcap-ct`, true but a description of the consequence. +The reader is owed the cause. + +## How the runs are joined + +Every virtualenv measures the **whole** engine list, not just the engine unique to +it. That is deliberate: `pcap_ct` in the `pypcap` environment comes back +unmeasured with the mutual exclusion stated by the engine itself, which is better +documentation of the constraint than anything this README could assert. An engine +measured in *any* environment gets a row; one that could not be measured anywhere gets +the reason, attributed per environment when they disagree. + +The two tables join the environments differently, and the difference is the point: + +- **the per-version table pools within one interpreter and never across two.** An + engine measured in both virtualenvs of 3.11 contributes both readings to the 3.11 + cell, since they differ only in which `pcap` provider was installed alongside. A + cell is `--` only when *no* environment on that interpreter measured it. +- **the ratio table pools everything**, because each ratio was normalised in its own + environment before pooling and is therefore already machine-independent. + +A pass whose `default` reading is missing or zero is dropped from the ratio table -- +there is nothing to divide by -- but kept in the per-version table, where the +millisecond figure is complete on its own. Losing a real measurement to a rule that +does not apply to it would be the wrong kind of caution. + +The text report additionally prints each shared engine's median *per environment*: if +two environments disagree about `dpkt`, the join on `default` is not doing its +job, and that is the line that shows it. With seven environments that line is also the +quickest way to see whether an engine behaves differently on one interpreter. + +## Departures from the legacy methodology + +The shape comes from `examples/legacy_smoke/test_time.py` so the figures stay +comparable in kind with what the project has already published: 1,000 timed +extractions of `examples/captures/in.pcap` per engine, timed with +`time.perf_counter_ns`, first sample discarded as a warm-up, reported as +milliseconds per packet. What changed, and why: + +| Change | Why | +| --- | --- | +| The whole set is measured `--repeats` times, not once | one pass gives no way to tell a real gap from noise, which is most of what was wrong with the old table | +| Every Python version is measured in one run, not one per sitting | the legacy script measures whichever interpreter invokes it, so a complete table was five separate sittings hand-assembled; a column measured last week is not comparable in absolute terms with one measured today, which is the property the per-version table depends on | +| A second table reports ratios, with one absolute footnote | absolute times are not comparable across machines, and "which engine is faster" is a question about engines rather than about this host | +| The driver is asserted on every extraction | the legacy script preflights once; an engine that stops being itself mid-run would go unnoticed | +| Progress is printed to stderr every 100 rounds, not to stdout every round | stdout carries the JSON document, and a flushed write per round is jitter bought for nothing | +| Warnings other than `EngineWarning` are silenced during timing | `ExtractionWarning: EOF reached` fires on every extraction of this capture; `EngineWarning` is escalated to an exception so a mid-run fallback cannot be missed | +| Engines run inside the repeat loop, not outside | a machine that slows down partway then affects every engine in a pass about equally, which is what lets the within-pass ratio cancel the drift | + +Not changed: the capture, the round count, the clock, the warm-up discard, and the +`store=False, nofile=True` extraction arguments. + +## Apple Silicon, arm64 and emulation + +`run.sh` builds for **this machine's architecture by default**, which on Apple +Silicon means native `linux/arm64` images. That is the setting that produces usable +numbers, and it should not need overriding: + +- Every base image digest in `python-images.txt` is a multi-arch OCI *index* that + includes `linux/arm64/v8`, so the pins resolve natively rather than forcing + emulation. +- `pcap-ct` and `libpcap` publish `py3-none-any` wheels -- arch-independent, + nothing to compile. (`libpcap` does vendor an `aarch64` `libpcap.so` under + `libpcap/_platform/`, but its published config sets `LIBPCAP = None` and falls + through to `find_library("pcap")`, so the image deliberately uses Debian's libpcap + instead, which is closer to what a user gets.) +- `dpkt`, `scapy`, `pyshark` and `pypcapfile` are pure Python. `lxml`, + `pyshark`'s only compiled dependency, publishes manylinux wheels for aarch64 on + every interpreter in the matrix (cp310 through cp314), so no column needs a compiler + for it. +- `pypcap` is the only thing that compiles, and nothing in it is + architecture-specific: its pre-generated `pcap.c` contains no arch guards or + inline assembly, and its `setup.py` branches on `sys.maxsize > 2**32` rather + than on the machine. Its library-search list does omit `aarch64-linux-gnu` -- it + names only the x86 multiarch paths -- but the list ends in an empty prefix that makes + it walk `/usr` recursively, which finds + `/usr/lib/aarch64-linux-gnu/libpcap.so`. The consequence on arm64 is a slower + build, not a failed one. + +**If you do pass** `--platform linux/amd64` **on an Apple Silicon machine, the run +happens under QEMU emulation.** Numbers taken that way are not comparable with native +ones, and the ratios are only as trustworthy as the emulator is uniform across the +very different work each engine does -- which for a C extension against a `ctypes` +binding against a subprocess spawn is not an assumption worth making. `run.sh` +detects the mismatch, warns on stderr, **and threads the warning into the emitted +table**, because the person reading the table later is not the person who saw the +stderr. + +And if `pypcap` does fail to build on arm64 despite the above, the run still +produces a table: the row says `*not measured*` and carries the compiler error, per +the section above. That is the honest outcome, and it is better than the two +alternatives -- no table, or an `amd64` run under emulation whose numbers cannot be +compared with anything. + +## Reproducibility: what is pinned and what is not + +Pinned exactly: + +- every base image, **by digest**, in `python-images.txt` rather than by tag: + `3.11-slim-bookworm` and its four siblings are mutable and rebuilt whenever Debian + or CPython ships a patch, so a tag pin is a pin in name only. They are all + **slim-bookworm**, which is a measurement decision rather than a packaging one -- + the per-version table's columns are compared with each other, so everything but the + interpreter is held constant, down to the same `tshark` and the same libpcap; +- every engine package, in `requirements-common.txt`, `requirements-pypcap.txt` + and `requirements-pcap_ct.txt`. Note that `pcap-ct` and `libpcap` are + pre-release-only projects, so `pip install pcap-ct` with no version resolves to + nothing at all -- an exact pre-release pin is honoured without `--pre`, which is + why the versions are spelled out; +- `pcapkit`'s own runtime dependencies, because they are on the baseline's hot path + and the baseline is what every ratio divides by. + +Not pinned, deliberately: + +- **Debian package versions** (`tshark`, `libpcap-dev`). The archive drops old + versions, so a pinned `apt-get` line breaks rather than reproduces. The resolved + `tshark` and libpcap versions are *recorded in the report* instead, which is the + honest form of reproducibility here. The libpcap version is read through `ctypes` + from the library `find_library("pcap")` actually resolves -- neither `pypcap` + nor `pcap-ct` exposes a `lib_version` attribute, so reading one off the module + silently yields nothing. +- **The transitive closure** beyond the direct pins. It is frozen at build time inside + each image and copied out beside the results, one file per environment, so + `pip freeze` for 3.11's `pypcap` virtualenv does not overwrite 3.12's. +- **pcapkit itself**, which is installed from the working tree rather than from + PyPI. Its git revision is recorded instead, with a `-dirty` suffix when the tree + has uncommitted changes under `pcapkit/` -- a revision that does not say so is a + claim about code that was not measured. + +## Output + +Everything lands in `--out` (default `examples/benchmark/out/`, which is +untracked): + +| File | What it is | +| --- | --- | +| `table-versions.rst` | absolute ms/packet per engine and Python version -- the docs' **Test Results** section, verbatim | +| `table.rst` | `Test Environment` plus the machine-independent ratio table | +| `-.json` | the raw per-environment measurements, every pass, e.g. `3.11-pypcap.json` | +| `-.txt` | `pip freeze` of that virtualenv as built | +| `missing/.txt` | why a version produced no measurements, when one did not | + +`run.sh` clears the previous run's `*.json`, `*.rst`, `*.txt` and +`missing/` before it starts, **and that is not tidiness.** The reporting pass reports +on every document it is handed, so a document left over from a run on another +`pcapkit` revision would fill a column of this run's table and pool into this run's +ratios, while the provenance block -- which reports one revision -- would not show the +disagreement. A stale table would likewise make "no table was produced" +unfalsifiable. + +`out/` is already covered by the repository's `.gitignore`, so nothing here needs +a new ignore rule. + +The report prints an image reference **per Python version**, the `pcapkit` +revision, every interpreter measured, the architecture, the capture and its digest, +the iteration count, and every engine package's resolved version per environment. It +prints **nothing that identifies the host** -- no hostname, no kernel, no CPU model. +Containerising the benchmark is what makes the host irrelevant; printing host details +would undo that, and would leak machine identity into a public README. + +## How long it takes + +**A full default matrix is on the order of two hours**, and it splits roughly in two: + +- **build: 15-25 minutes.** Five images, each installing the same pinned set, plus a + `pypcap` C extension compiled twice (3.10 and 3.11). Docker's layer cache does not + help across versions -- a different base image is a different layer chain -- so this + cost is per column. Measured on this suite: a 3.12 image, which compiles nothing, + takes about a minute and a half; a 3.11 image, which does, takes about three. +- **measure: 1.5-2 hours.** Seven environments x 1,000 rounds x 3 passes, and **almost + all of it is** `pyshark`, which spawns a `tshark` process per extraction and is + two to three orders of magnitude slower than anything else in the table. It runs in + six of the seven environments (everything but 3.14, where it cannot run at all). + +Three ways to cut it, in descending order of how much they save: + +```shell +# drop pyshark and the run is minutes rather than hours +examples/benchmark/run.sh --engines default,dpkt,scapy,pypcap,pcap_ct,pypcapfile + +# fewer columns: one version is two environments at most +examples/benchmark/run.sh --pythons 3.11 + +# a smoke check that the harness works, not a measurement +examples/benchmark/run.sh --quick --pythons 3.11,3.12 --engines default,dpkt +``` + +`--quick` is 50 rounds over 2 passes. It is enough to confirm every engine runs and +both tables render; it is not enough to publish, and the spread it reports will say +so. + +## Options + +```text +--rounds N timed extractions per engine per pass (default 1000) +--repeats N passes over the whole engine set (default 3) +--engines LIST comma-separated; must include 'default', which every ratio uses +--pythons LIST comma-separated Python versions (default: every 'default'-tier + row of python-images.txt, i.e. 3.10 through 3.14) +--quick --rounds 50 --repeats 2 +--platform P docker platform; defaults to this machine's architecture +--out DIR where to write results (default examples/benchmark/out) +--no-build reuse the existing images instead of rebuilding +``` + +From the repository root: `make bench` runs `run.sh` with no arguments, +`make bench-quick` runs a reduced smoke matrix, and `make bench-test` runs the +self-tests below. + +## Python 3.15 + +Opt-in, and allowed to fail -- the benchmark's own choice, not CI's: the repository's CI +has run 3.15 as a blocking, supported leg since #1633. There is no released +`python:3.15-slim-bookworm`; the +pins file names a `3.15-rc` digest, which freezes one particular release candidate +and will name an older one as time passes. Measure it with: + +```shell +examples/benchmark/run.sh --pythons 3.10,3.11,3.12,3.13,3.14,3.15 +``` + +If that image cannot be built or the run fails in it, the result is a `3.15` column +of `--` with the reason attached, not a missing column and not an aborted run. + +As it happens the pinned rc **does** build and measure: every pinned requirement +installs on it, and `default`, `dpkt`, `scapy` and `pcap_ct` all produced +figures. `pypcap`, `pypcapfile` and `pyshark` come back unmeasured with the same +reasons they give on 3.14. That is a fact about one release candidate on one day +rather than a promise about 3.15, which is why the row stays opt-in. + +## Self-tests + +```shell +python -m pytest examples/benchmark/test_harness.py +``` + +No docker needed. These cover the parts of a benchmark that *can* be checked without +a stopwatch, which is more than it sounds: that a ratio is taken against the right +baseline, that machine drift cancels, that repeats are paired one to one rather than +averaged first, that the environments stitch together correctly, that an unmeasured +engine keeps its reason, that the overlap marking neither over- nor under-claims, and +that the emitted reStructuredText parses under **plain docutils** with no warnings and +uses no Sphinx-only roles. That last one is a conservative choice rather than a +hard requirement: the snippets land in `docs/source/index.rst`, which Sphinx +renders, so a `:mod:` role would survive there. Keeping them docutils-clean is what +lets a table be pasted into any reStructuredText a plain docutils reader will see -- +where a `:mod:` role comes out as a visible error block while looking perfectly fine +in the project's own Sphinx docs. + +The matrix adds its own statements to check, since most of the ways a version column +can be wrong are silent: that a document lands in the column its *interpreter* names +rather than the one its label claims; that `3.9` sorts before `3.10`; that a cell +pools the two virtualenvs of one interpreter and never divides across two of them; +that a cell measured in one virtualenv is not reported as a gap; that a version which +produced nothing is a marked column of `--` with a reason rather than an absent +column, and does not mark the engines for it; and that the absolute table contains no +ratio anywhere, `pcapkit` included. + +They also cover the driver assertion, including that a measurement whose engine fell +back to `pcapkit`'s parser refuses to report a number. + +## Troubleshooting + +- `docker is installed but not responding` -- start Docker Desktop and wait for it + to report "running". +- The `pypcap` layer takes minutes on arm64 -- expected, and explained above: + its `setup.py` walks `/usr` recursively to find a library its search list does + not name. It is built for 3.10 and 3.11 only, so this cost is paid twice, not five + times. +- `pyshark` reports not measured, "the tshark binary ... is not installed" -- + the image installs `tshark`, so this means the apt install was skipped or failed; + rebuild without `--no-build`. +- A row you expected is `*not measured*` -- read the reason. It comes from the + engine's own `unsupported_reason()`, which names the actual cause rather than + guessing at one -- except for `pypcap` on 3.12+, where the interpreter ceiling is + reported instead. +- A whole column is `--` -- read the note under the table, and + `out/missing/.txt`. The version was attempted; the build or the run + failed, and the reason is recorded rather than the column dropped. +- `'3.1' is not in python-images.txt` -- `--pythons` takes versions the pins + file knows. An unknown one is fatal rather than reported as a missing column, since + a column saying "3.1 could not be measured" would be an honest-looking answer to a + question nobody asked. diff --git a/examples/benchmark/README.rst b/examples/benchmark/README.rst deleted file mode 100644 index 7899f3b0c3..0000000000 --- a/examples/benchmark/README.rst +++ /dev/null @@ -1,536 +0,0 @@ -====================== -Engine Benchmark Suite -====================== - -A containerised benchmark that produces the engine speed table in the project docs, -``docs/source/index.rst`` -- the whole table, every supported Python version, in one -run. One command, on any machine with Docker, and the output is two paste-ready -reStructuredText snippets: - -.. code-block:: shell - - examples/benchmark/run.sh - -It exists because that table could not previously be completed on **any** single -machine, and because the numbers in it were not reproducible. Both problems are the -same problem -- the environment was never pinned and could not hold every engine at -once -- so the fix is to define the environment precisely and ship it. Once the -environment is a container, the number of them is a loop, which is what makes a -complete table a single command rather than five hand-run measurements pasted -together. - -The matrix ----------- - -**One image per Python version**, five of them by default, each pinned by digest in -`python-images.txt `_ -- which is the matrix, and is a table -rather than logic so that a reader can check it. Inside each image, **one or two -virtualenvs**, so an "environment" here is a *(Python version, pcap provider)* -pair: - -.. list-table:: - :header-rows: 1 - - * - Python - - environments - - why - * - 3.10, 3.11 - - ``-pypcap`` and ``-pcap_ct`` - - both ``pcap`` providers install, and they are mutually exclusive - * - 3.12, 3.13, 3.14 - - ``-pcap_ct`` only - - ``pypcap`` cannot be installed at all, so a second virtualenv would compile - for minutes to produce nothing - * - 3.15 - - opt-in, ``-pcap_ct`` only - - no released image; see below - -Seven environments for a default run, and a version named with ``--pythons`` that is -not in the file is a typo rather than a column. - -Why this cannot be one virtualenv, or one Python ------------------------------------------------- - -Two hard constraints shape everything here. Neither is a ``pcapkit`` defect and -neither can be worked around from inside ``pcapkit``: - -1. **pypcap and pcap-ct are mutually exclusive.** Both distributions install - a top-level module named ``pcap``. With both present, ``import pcap`` resolves to - ``pcap-ct``'s package directory and upstream's extension module is shadowed and - unreachable, so ``engine='pypcap'`` cannot run at all. No install order changes - that. **So no single environment can hold all seven engines**, and an image that - can hold ``pypcap`` carries two virtualenvs -- ``/opt/venv-pypcap`` and - ``/opt/venv-pcap_ct``, identical but for which ``pcap`` provider they have. -2. **3.11 is the** *last* **interpreter on which every engine runs**, and 3.10 is the - other one. ``pypcap`` 1.3.0 ships a ``pcap.c`` pre-generated by Cython 0.29.32 - that will not compile against the 3.12+ C API; ``pypcapfile`` 0.12.0 imports - ``imp``, removed in 3.12; ``pyshark`` 0.6 calls - ``asyncio.get_event_loop_policy().get_event_loop()``, which raises from 3.14. - **That is a fact the table reports rather than a constraint on the run**: each - engine's ceiling shows up as ``--`` cells with the reason, in the columns past it. - -The ``default`` engine -- ``pcapkit``'s own parser, also spelled ``pcapkit`` -- is -present in every virtualenv of every image, and is what joins all of them into one -table. See "How the runs are joined" below. - -Two tables, because there are two questions -------------------------------------------- - -The run emits both, and neither replaces the other: - -.. list-table:: - :header-rows: 1 - - * - File - - Question it answers - - Arithmetic - * - ``table-versions.rst`` - - how does each engine do on each interpreter? - - **absolute** milliseconds per packet, one column per Python version - * - ``table.rst`` - - which engine is faster, on any machine? - - **ratios** against ``default``, pooled across the whole matrix - -**The per-version table is absolute, and must be.** Every column of it came out of -one run on one host, differing in the interpreter and nothing else, which is -precisely the condition under which absolute times are comparable -- with each other. -A ratio there would be worse than redundant: ratios are normalised *within* one -environment, so dividing across two interpreters yields a number that is neither -engine's speed nor the interpreter's. The snippet says so in its own prose, because -the person reading the table in the docs later is not the person who read this -file. - -**The ratio table is normalised to the default engine measured in the same -virtualenv on the same pass.** An absolute timing describes the machine it was taken -on at least as much as it describes the engine, so a table of absolute numbers cannot -be reproduced across machines -- which is exactly the state the old table was in. A -ratio divides the machine out, and pooling it across every interpreter is what makes -it an answer about engines rather than about one interpreter. The observed range then -includes any disagreement *between* interpreters, which is real uncertainty about the -engine and is not something to average away; the per-version table is where it gets -broken out. - -One absolute figure survives inside the ratio snippet, as a footnote: the ``default`` -engine's own milliseconds per packet. That is what lets a reader convert the ratios -back into times on the machine the run happened on. - -Both snippets carry their own ``Test Results`` heading -- the per-version one takes -the name, since it is what the docs' **Test Results** section is, and the ratio -one is ``Test Results (Relative)`` so that pasting both does not produce two sections -with one name. The ``Test Environment`` block, which serves both, is in -``table.rst``. - -**The run-to-run spread is reported too.** A single pass cannot tell a real difference -between two engines from the machine having been briefly busy, so the whole engine set -is measured several times (``--repeats``, default 3) and every row carries the range -actually observed. Rows whose ranges overlap are marked ``‡``: this run does not -establish which of them is faster, and printing their medians as though it did would -be inventing precision. The footnote states the median and the widest spread seen, so -a reader can judge any gap in the table for themselves. - -The property that matters most ------------------------------- - -**An engine that cannot run does not raise.** ``Extractor.run`` emits an -``EngineWarning`` and falls back to ``pcapkit``'s own parser, so an extraction that -appears to succeed can be timing something entirely different from what was asked -for. A benchmark that does not check this will happily report the default engine's -speed three times over under three different names, and nothing in the output would -look wrong. - -So the harness checks ``type(extractor.engine).__engine_name__`` on **every single -extraction**, not just the first, against a list of expected drivers derived from -``Extractor.__engine__`` rather than written out by hand. A run whose driver ever -disagrees raises and is discarded; no number is reported for it. This is asserted, not -assumed, and it is the single most important correctness property of the suite. - -An engine that genuinely cannot run here is recorded as ``*not measured*`` with the -reason the engine itself gives through ``unsupported_reason()`` -- never omitted, and -never as a zero. - -One flaky engine must not cost the others their rows ----------------------------------------------------- - -This is not hypothetical; it is what the first full run of this harness did. -``pyshark`` spawns a ``tshark`` process per extraction, so a default run spawns **six -thousand** of them, and one of them died -- ``TSharkCrashException ... retcode: 255``, -on the third pass. The exception propagated, ``benchmark.py`` exited, no JSON was -written, and the run produced **no table at all**. Six working engines went unreported -because the seventh hiccupped once. - -So failure is contained at two levels, and counted at both: - -- **A failed extraction** is discarded and the pass carries on, up to - ``TOLERATED_FAILURES`` (5) per engine per pass. A crashed extraction is a - *non-measurement*, not a slow measurement, so discarding its sample is correct -- - what would not be honest is discarding it silently, so the count and the first - reason are reported. -- **A failed pass** costs that engine that pass, not the run. The row is built from - the passes that did work, its ``Passes`` count shows fewer, and it is marked ``¶`` - with what was lost. If *every* pass failed, the engine becomes ``*not measured*`` - with the exception as its reason. - -Two things are never tolerated, because tolerating them would mean reporting a number -that is not true: the escalated ``EngineWarning`` that means the engine fell back to -``pcapkit``'s parser, and a capture that yielded no packets. Both end the run. - -The same principle covers the build. ``pypcap`` is the only engine in the set that -compiles, and therefore the only one whose *install* can fail for reasons these pins -do not control. A fatal failure there would mean no table at all, so the Dockerfile -records the pip error to ``/opt/install-failures/pypcap.txt`` and carries on; -``benchmark.py`` reads it and reports the compiler error as the reason ``pypcap`` was -not measured. Everything else in the image is a wheel on every architecture *and every -interpreter in the matrix* -- checked on 3.10, 3.12, 3.13 and 3.14 as well as 3.11 -- -so a failure installing it is a real error and stays fatal. - -A missing column is held to the same standard as a missing row --------------------------------------------------------------- - -With five images there are five chances for something outside these pins to go wrong: -a base image withdrawn, a wheel that stops publishing for one interpreter, a daemon -that runs out of disk halfway through. **None of those ends the run**, and none of -them silently narrows the table: - -- ``run.sh`` catches the failure per version, writes the reason to - ``out/missing/.txt``, and carries on to the next one. That is the same - mechanism ``/opt/install-failures/`` uses for ``pypcap``, one level up. -- the reporting pass turns each of those files into a column of ``--`` **with the - reason attached** -- in the per-version table's notes, and in the ``Test - Environment`` block as a ``Python 3.15 (not measured)`` row, so neither snippet - reads as a complete matrix when it is not. -- only a run in which *every* version failed exits without a table, since then there - is genuinely nothing to report. - -**The exit status says which happened**, because a scheduled invocation reads nothing -else: ``0`` when every requested version was measured, ``3`` **when the tables were -written but a version was lost**, ``1`` when no table was produced at all, and ``2`` -for bad arguments. A ``make bench`` that measured one column out of five is not a -failure -- the tables are there -- but it is not success either, and it should not be -indistinguishable from it. - -**pypcap on 3.12+ is deliberately not reported this way.** It did not fail; it -cannot be installed, which is a fact about the interpreter rather than about this -build. The image records that in ``/opt/not-attempted/pypcap.txt``, and that note -*replaces* the engine's own ``unsupported_reason()`` -- which on those images says the -installed ``pcap`` module is ``pcap-ct``, true but a description of the consequence. -The reader is owed the cause. - -How the runs are joined ------------------------ - -Every virtualenv measures the **whole** engine list, not just the engine unique to -it. That is deliberate: ``pcap_ct`` in the ``pypcap`` environment comes back -unmeasured with the mutual exclusion stated by the engine itself, which is better -documentation of the constraint than anything this README could assert. An engine -measured in *any* environment gets a row; one that could not be measured anywhere gets -the reason, attributed per environment when they disagree. - -The two tables join the environments differently, and the difference is the point: - -- **the per-version table pools within one interpreter and never across two.** An - engine measured in both virtualenvs of 3.11 contributes both readings to the 3.11 - cell, since they differ only in which ``pcap`` provider was installed alongside. A - cell is ``--`` only when *no* environment on that interpreter measured it. -- **the ratio table pools everything**, because each ratio was normalised in its own - environment before pooling and is therefore already machine-independent. - -A pass whose ``default`` reading is missing or zero is dropped from the ratio table -- -there is nothing to divide by -- but kept in the per-version table, where the -millisecond figure is complete on its own. Losing a real measurement to a rule that -does not apply to it would be the wrong kind of caution. - -The text report additionally prints each shared engine's median *per environment*: if -two environments disagree about ``dpkt``, the join on ``default`` is not doing its -job, and that is the line that shows it. With seven environments that line is also the -quickest way to see whether an engine behaves differently on one interpreter. - -Departures from the legacy methodology --------------------------------------- - -The shape comes from ``examples/legacy_smoke/test_time.py`` so the figures stay -comparable in kind with what the project has already published: 1,000 timed -extractions of ``examples/captures/in.pcap`` per engine, timed with -``time.perf_counter_ns``, first sample discarded as a warm-up, reported as -milliseconds per packet. What changed, and why: - -.. list-table:: - :header-rows: 1 - - * - Change - - Why - * - The whole set is measured ``--repeats`` times, not once - - one pass gives no way to tell a real gap from noise, which is most of what was - wrong with the old table - * - Every Python version is measured in one run, not one per sitting - - the legacy script measures whichever interpreter invokes it, so a complete - table was five separate sittings hand-assembled; a column measured last week - is not comparable in absolute terms with one measured today, which is the - property the per-version table depends on - * - A second table reports ratios, with one absolute footnote - - absolute times are not comparable across machines, and "which engine is - faster" is a question about engines rather than about this host - * - The driver is asserted on every extraction - - the legacy script preflights once; an engine that stops being itself mid-run - would go unnoticed - * - Progress is printed to stderr every 100 rounds, not to stdout every round - - stdout carries the JSON document, and a flushed write per round is jitter - bought for nothing - * - Warnings other than ``EngineWarning`` are silenced during timing - - ``ExtractionWarning: EOF reached`` fires on every extraction of this capture; - ``EngineWarning`` is escalated to an exception so a mid-run fallback cannot be - missed - * - Engines run inside the repeat loop, not outside - - a machine that slows down partway then affects every engine in a pass about - equally, which is what lets the within-pass ratio cancel the drift - -Not changed: the capture, the round count, the clock, the warm-up discard, and the -``store=False, nofile=True`` extraction arguments. - -Apple Silicon, arm64 and emulation ----------------------------------- - -``run.sh`` builds for **this machine's architecture by default**, which on Apple -Silicon means native ``linux/arm64`` images. That is the setting that produces usable -numbers, and it should not need overriding: - -- Every base image digest in ``python-images.txt`` is a multi-arch OCI *index* that - includes ``linux/arm64/v8``, so the pins resolve natively rather than forcing - emulation. -- ``pcap-ct`` and ``libpcap`` publish ``py3-none-any`` wheels -- arch-independent, - nothing to compile. (``libpcap`` does vendor an ``aarch64`` ``libpcap.so`` under - ``libpcap/_platform/``, but its published config sets ``LIBPCAP = None`` and falls - through to ``find_library("pcap")``, so the image deliberately uses Debian's libpcap - instead, which is closer to what a user gets.) -- ``dpkt``, ``scapy``, ``pyshark`` and ``pypcapfile`` are pure Python. ``lxml``, - ``pyshark``'s only compiled dependency, publishes manylinux wheels for aarch64 on - every interpreter in the matrix (cp310 through cp314), so no column needs a compiler - for it. -- ``pypcap`` is the only thing that compiles, and nothing in it is - architecture-specific: its pre-generated ``pcap.c`` contains no arch guards or - inline assembly, and its ``setup.py`` branches on ``sys.maxsize > 2**32`` rather - than on the machine. Its library-search list does omit ``aarch64-linux-gnu`` -- it - names only the x86 multiarch paths -- but the list ends in an empty prefix that makes - it walk ``/usr`` recursively, which finds - ``/usr/lib/aarch64-linux-gnu/libpcap.so``. The consequence on arm64 is a slower - build, not a failed one. - -**If you do pass** ``--platform linux/amd64`` **on an Apple Silicon machine, the run -happens under QEMU emulation.** Numbers taken that way are not comparable with native -ones, and the ratios are only as trustworthy as the emulator is uniform across the -very different work each engine does -- which for a C extension against a ``ctypes`` -binding against a subprocess spawn is not an assumption worth making. ``run.sh`` -detects the mismatch, warns on stderr, **and threads the warning into the emitted -table**, because the person reading the table later is not the person who saw the -stderr. - -And if ``pypcap`` does fail to build on arm64 despite the above, the run still -produces a table: the row says ``*not measured*`` and carries the compiler error, per -the section above. That is the honest outcome, and it is better than the two -alternatives -- no table, or an ``amd64`` run under emulation whose numbers cannot be -compared with anything. - -Reproducibility: what is pinned and what is not ------------------------------------------------ - -Pinned exactly: - -- every base image, **by digest**, in ``python-images.txt`` rather than by tag: - ``3.11-slim-bookworm`` and its four siblings are mutable and rebuilt whenever Debian - or CPython ships a patch, so a tag pin is a pin in name only. They are all - **slim-bookworm**, which is a measurement decision rather than a packaging one -- - the per-version table's columns are compared with each other, so everything but the - interpreter is held constant, down to the same ``tshark`` and the same libpcap; -- every engine package, in ``requirements-common.txt``, ``requirements-pypcap.txt`` - and ``requirements-pcap_ct.txt``. Note that ``pcap-ct`` and ``libpcap`` are - pre-release-only projects, so ``pip install pcap-ct`` with no version resolves to - nothing at all -- an exact pre-release pin is honoured without ``--pre``, which is - why the versions are spelled out; -- ``pcapkit``'s own runtime dependencies, because they are on the baseline's hot path - and the baseline is what every ratio divides by. - -Not pinned, deliberately: - -- **Debian package versions** (``tshark``, ``libpcap-dev``). The archive drops old - versions, so a pinned ``apt-get`` line breaks rather than reproduces. The resolved - ``tshark`` and libpcap versions are *recorded in the report* instead, which is the - honest form of reproducibility here. The libpcap version is read through ``ctypes`` - from the library ``find_library("pcap")`` actually resolves -- neither ``pypcap`` - nor ``pcap-ct`` exposes a ``lib_version`` attribute, so reading one off the module - silently yields nothing. -- **The transitive closure** beyond the direct pins. It is frozen at build time inside - each image and copied out beside the results, one file per environment, so - ``pip freeze`` for 3.11's ``pypcap`` virtualenv does not overwrite 3.12's. -- **pcapkit itself**, which is installed from the working tree rather than from - PyPI. Its git revision is recorded instead, with a ``-dirty`` suffix when the tree - has uncommitted changes under ``pcapkit/`` -- a revision that does not say so is a - claim about code that was not measured. - -Output ------- - -Everything lands in ``--out`` (default ``examples/benchmark/out/``, which is -untracked): - -.. list-table:: - :header-rows: 1 - - * - File - - What it is - * - ``table-versions.rst`` - - absolute ms/packet per engine and Python version -- the docs' **Test - Results** section, verbatim - * - ``table.rst`` - - ``Test Environment`` plus the machine-independent ratio table - * - ``-.json`` - - the raw per-environment measurements, every pass, e.g. ``3.11-pypcap.json`` - * - ``-.txt`` - - ``pip freeze`` of that virtualenv as built - * - ``missing/.txt`` - - why a version produced no measurements, when one did not - -``run.sh`` clears the previous run's ``*.json``, ``*.rst``, ``*.txt`` and -``missing/`` before it starts, **and that is not tidiness.** The reporting pass reports -on every document it is handed, so a document left over from a run on another -``pcapkit`` revision would fill a column of this run's table and pool into this run's -ratios, while the provenance block -- which reports one revision -- would not show the -disagreement. A stale table would likewise make "no table was produced" -unfalsifiable. - -``out/`` is already covered by the repository's ``.gitignore``, so nothing here needs -a new ignore rule. - -The report prints an image reference **per Python version**, the ``pcapkit`` -revision, every interpreter measured, the architecture, the capture and its digest, -the iteration count, and every engine package's resolved version per environment. It -prints **nothing that identifies the host** -- no hostname, no kernel, no CPU model. -Containerising the benchmark is what makes the host irrelevant; printing host details -would undo that, and would leak machine identity into a public README. - -How long it takes ------------------ - -**A full default matrix is on the order of two hours**, and it splits roughly in two: - -- **build: 15-25 minutes.** Five images, each installing the same pinned set, plus a - ``pypcap`` C extension compiled twice (3.10 and 3.11). Docker's layer cache does not - help across versions -- a different base image is a different layer chain -- so this - cost is per column. Measured on this suite: a 3.12 image, which compiles nothing, - takes about a minute and a half; a 3.11 image, which does, takes about three. -- **measure: 1.5-2 hours.** Seven environments x 1,000 rounds x 3 passes, and **almost - all of it is** ``pyshark``, which spawns a ``tshark`` process per extraction and is - two to three orders of magnitude slower than anything else in the table. It runs in - six of the seven environments (everything but 3.14, where it cannot run at all). - -Three ways to cut it, in descending order of how much they save: - -.. code-block:: shell - - # drop pyshark and the run is minutes rather than hours - examples/benchmark/run.sh --engines default,dpkt,scapy,pypcap,pcap_ct,pypcapfile - - # fewer columns: one version is two environments at most - examples/benchmark/run.sh --pythons 3.11 - - # a smoke check that the harness works, not a measurement - examples/benchmark/run.sh --quick --pythons 3.11,3.12 --engines default,dpkt - -``--quick`` is 50 rounds over 2 passes. It is enough to confirm every engine runs and -both tables render; it is not enough to publish, and the spread it reports will say -so. - -Options -------- - -.. code-block:: text - - --rounds N timed extractions per engine per pass (default 1000) - --repeats N passes over the whole engine set (default 3) - --engines LIST comma-separated; must include 'default', which every ratio uses - --pythons LIST comma-separated Python versions (default: every 'default'-tier - row of python-images.txt, i.e. 3.10 through 3.14) - --quick --rounds 50 --repeats 2 - --platform P docker platform; defaults to this machine's architecture - --out DIR where to write results (default examples/benchmark/out) - --no-build reuse the existing images instead of rebuilding - -From the repository root: ``make bench`` runs ``run.sh`` with no arguments, -``make bench-quick`` runs a reduced smoke matrix, and ``make bench-test`` runs the -self-tests below. - -Python 3.15 ------------ - -Opt-in, and allowed to fail -- the benchmark's own choice, not CI's: the repository's CI -has run 3.15 as a blocking, supported leg since #1633. There is no released -``python:3.15-slim-bookworm``; the -pins file names a ``3.15-rc`` digest, which freezes one particular release candidate -and will name an older one as time passes. Measure it with: - -.. code-block:: shell - - examples/benchmark/run.sh --pythons 3.10,3.11,3.12,3.13,3.14,3.15 - -If that image cannot be built or the run fails in it, the result is a ``3.15`` column -of ``--`` with the reason attached, not a missing column and not an aborted run. - -As it happens the pinned rc **does** build and measure: every pinned requirement -installs on it, and ``default``, ``dpkt``, ``scapy`` and ``pcap_ct`` all produced -figures. ``pypcap``, ``pypcapfile`` and ``pyshark`` come back unmeasured with the same -reasons they give on 3.14. That is a fact about one release candidate on one day -rather than a promise about 3.15, which is why the row stays opt-in. - -Self-tests ----------- - -.. code-block:: shell - - python -m pytest examples/benchmark/test_harness.py - -No docker needed. These cover the parts of a benchmark that *can* be checked without -a stopwatch, which is more than it sounds: that a ratio is taken against the right -baseline, that machine drift cancels, that repeats are paired one to one rather than -averaged first, that the environments stitch together correctly, that an unmeasured -engine keeps its reason, that the overlap marking neither over- nor under-claims, and -that the emitted reStructuredText parses under **plain docutils** with no warnings and -uses no Sphinx-only roles. That last one is a conservative choice rather than a -hard requirement: the snippets land in ``docs/source/index.rst``, which Sphinx -renders, so a ``:mod:`` role would survive there. Keeping them docutils-clean is what -lets a table be pasted into any reStructuredText a plain docutils reader will see -- -where a ``:mod:`` role comes out as a visible error block while looking perfectly fine -in the project's own Sphinx docs. - -The matrix adds its own statements to check, since most of the ways a version column -can be wrong are silent: that a document lands in the column its *interpreter* names -rather than the one its label claims; that ``3.9`` sorts before ``3.10``; that a cell -pools the two virtualenvs of one interpreter and never divides across two of them; -that a cell measured in one virtualenv is not reported as a gap; that a version which -produced nothing is a marked column of ``--`` with a reason rather than an absent -column, and does not mark the engines for it; and that the absolute table contains no -ratio anywhere, ``pcapkit`` included. - -They also cover the driver assertion, including that a measurement whose engine fell -back to ``pcapkit``'s parser refuses to report a number. - -Troubleshooting ---------------- - -- ``docker is installed but not responding`` -- start Docker Desktop and wait for it - to report "running". -- The ``pypcap`` layer takes minutes on arm64 -- expected, and explained above: - its ``setup.py`` walks ``/usr`` recursively to find a library its search list does - not name. It is built for 3.10 and 3.11 only, so this cost is paid twice, not five - times. -- ``pyshark`` reports not measured, "the tshark binary ... is not installed" -- - the image installs ``tshark``, so this means the apt install was skipped or failed; - rebuild without ``--no-build``. -- A row you expected is ``*not measured*`` -- read the reason. It comes from the - engine's own ``unsupported_reason()``, which names the actual cause rather than - guessing at one -- except for ``pypcap`` on 3.12+, where the interpreter ceiling is - reported instead. -- A whole column is ``--`` -- read the note under the table, and - ``out/missing/.txt``. The version was attempted; the build or the run - failed, and the reason is recorded rather than the column dropped. -- ``'3.1' is not in python-images.txt`` -- ``--pythons`` takes versions the pins - file knows. An unknown one is fatal rather than reported as a missing column, since - a column saying "3.1 could not be measured" would be an honest-looking answer to a - question nobody asked. diff --git a/examples/benchmark/benchmark.py b/examples/benchmark/benchmark.py index c04b6e59c0..ffb6f0d87c 100644 --- a/examples/benchmark/benchmark.py +++ b/examples/benchmark/benchmark.py @@ -22,7 +22,7 @@ :data:`ROUNDS` timed extractions of the same capture per engine, timed with :func:`time.perf_counter_ns`, first sample discarded as a warm-up, reported as milliseconds per packet. The deliberate departures are listed in -:file:`README.rst` under "Departures from the legacy methodology". +:file:`README.md` under "Departures from the legacy methodology". Two properties matter more than the numbers themselves. diff --git a/examples/legacy_smoke/README.md b/examples/legacy_smoke/README.md new file mode 100644 index 0000000000..8b15157093 --- /dev/null +++ b/examples/legacy_smoke/README.md @@ -0,0 +1,126 @@ +# Demonstration Scripts + +A tour of `pcapkit` by example: each script here does one thing and prints or dumps +the result, so you can get a thorough view of the library either on how to use it or +on what it can do. They are demos rather than tests -- `pytest` does not collect +this directory (`testpaths = ["tests"]`) and nothing in CI runs them, so they are +also where API drift goes unnoticed the longest. Run them after changing anything +public. + +## Running them + +Every script reads its capture with a path relative to its own directory, so **run +them from inside** `examples/legacy_smoke/`, not from the repository root: + +```shell +cd examples/legacy_smoke +python test_basic.py +``` + +The captures live in `../captures/`. Only `in.pcap` and `dhcp.pcapng` are +committed; the rest are generated, so build them once before running anything else: + +```shell +python ../generators/make_samples.py # or: make samples, from the repository root +``` + +## The scripts + +| Script | What it demonstrates | +| --- | --- | +| [test_basic](test_basic.py) | the smallest useful call -- extract `in.pcap` to a tree-view text file | +| [test_extractor](test_extractor.py) | `pcapkit.extract` dumping the same capture as plist, JSON and tree | +| [test_file](test_file.py) | reading from an already-open binary file object rather than a path | +| [test_api](test_api.py) | most of the keyword surface at once: per-frame files, reassembly and flow tracing | +| [test_tcp](test_tcp.py) | extraction of TCP packets | +| [test_ipv6](test_ipv6.py) | extraction of IPv6 packets, one output file per frame | +| [test_pcapng](test_pcapng.py) | extraction of a PCAP-NG capture, which uses a different engine | +| [test_http](test_http.py) | iterating frames manually and testing `pcapkit.HTTP in frame` | +| [test_reassembly](test_reassembly.py) | TCP payload reassembly, writing the reassembled datagrams out | +| [test_ip_reasm](test_ip_reasm.py) | IPv4 fragment reassembly | +| [test_ipv6_reasm](test_ipv6_reasm.py) | IPv6 fragment reassembly | +| [test_analyse](test_analyse.py) | analysis of the application layer *after* reassembly, printing the recovered HTTP messages | +| [test_trace](test_trace.py) | tracing TCP flows and dumping the flow index | +| [test_engine](test_engine.py) | the same capture through each extraction engine, side by side | +| [test_time](test_time.py) | timing each engine, reported as milliseconds per packet | +| [test_profile](test_profile.py) | cProfile stats for one extraction; `make profile` renders the call graph | +| [test_perf](test_perf.py) | the same comparison under the `pyperf` benchmark harness | +| [test_stream](test_stream.py) | extracting a live capture streamed from `tcpdump` on stdin | +| [test_stream_askpass](test_stream_askpass.py) | the same, with `sudo -A` for a non-interactive password | + +`_engine_support.py` is not a demo -- it is the shared helper the engine demos use +to work out whether an engine can run on this host. + +## What needs more than pcapkit + +Most of these run with nothing but `pcapkit`. These do not: + +- **test_engine, test_time, test_perf** exercise the `dpkt`, `scapy` and + `pyshark` engines. Those are optional packages, and `pyshark` additionally + drives Wireshark's `tshark` binary. An engine that is missing is reported as + skipped, with the reason, and the rest still run -- see `_engine_support.py`. Two + things worth knowing: + + - `pyshark` 0.6 does not work on Python 3.14 at all. It asks + `asyncio.get_event_loop_policy().get_event_loop()` for a loop on a thread that + has none, which 3.14 no longer creates implicitly, so every extraction through it + raises `RuntimeError: There is no current event loop in thread 'MainThread'` + before `tshark` is even reached. That is a pyshark bug, not a pcapkit one, and + there is nothing to configure around it. + - When an engine's *package* is missing, `pcapkit.extract` does not fail. It warns + and falls back to pcapkit's own parser, so the extraction succeeds under the wrong + engine. The demos name the driver that actually ran, so a fallback is visible + rather than being reported as a healthy result. + +- **test_perf** needs the `pyperf` harness, which `pcapkit` does not depend on: + `python -m pip install pyperf`. Without it the script says so and stops. + `test_time.py` does the same comparison with no extra dependency. +- **test_profile**'s `make profile` target needs `gprof2dot`, `graphviz` and + `snakeviz` to render the call graph. The script itself only needs `pcapkit`. +- **test_stream, test_stream_askpass** run `sudo tcpdump` against a live interface, + so they need root and cannot run unattended. `INTERFACE` at the top of each is + `en0` (macOS); on Linux it is usually `eth0` or `wlan0`. They buffer the live + capture into a temporary file -- never into `../captures/`, which holds generated + captures the test suite pins. + +## Regenerating the reference reports + +Four files in `../captures/` are reports rather than inputs -- `out.json`, +`out.plist`, `out.txt` and `pcapng.txt` -- and they are produced by two of the +scripts here. **Git does not track them**, so a fresh clone does not have them. Build +them the way you would any other generated capture, and again after a change to how +`pcapkit` renders one: + +```shell +cd examples/legacy_smoke && make fixtures +``` + +That is the whole recipe. It runs, equivalently: + +```shell +TZ=UTC python test_extractor.py # ../captures/out.json, out.plist, out.txt <- in.pcap +TZ=UTC python test_pcapng.py # ../captures/pcapng.txt <- dhcp.pcapng +``` + +Both *inputs* are committed, so this works on a fresh clone with no `make samples` +first. Use `make fixtures PYTHON=../../.venv/bin/python` to pick a specific +interpreter. + +These four were tracked once, and #685 is why they are not any more. Nothing reads a +report back -- no test asserts on one, and the docs only quote them as prose -- so a +parser change that alters how a frame renders leaves the committed copy quietly wrong, +with no failure anywhere to say so. `pcapng.txt` sat recording `packet -> NIL` for +four blocks that had started carrying their captured octets, and the staleness was +found by reading the file rather than by anything breaking. Generating them on demand +means the copy on your disk describes the tree you are standing in, which was the only +thing that made them worth looking at. + +`TZ=UTC` **is not decoration.** `pcapkit` renders frame timestamps in the host's +local zone, so an unpinned run rewrites every timestamp line in all four files with +wherever it happened to be run -- which is how these reports came to disagree with +each other while they were still tracked, the PCAP three having been generated at +UTC-05:00 and `pcapng.txt` at UTC+08:00. Pinning UTC makes the output reproducible on +any host, so two runs can be diffed against each other. It also happens to be the only +zone in which PCAP-NG's `timestamp_epoch` is the true UNIX epoch: +`PCAPNG._read_timestamp` adds the zone's UTC offset to the value it returns, so a +report generated anywhere else bakes in that offset. diff --git a/examples/legacy_smoke/README.rst b/examples/legacy_smoke/README.rst deleted file mode 100644 index d7d769ec43..0000000000 --- a/examples/legacy_smoke/README.rst +++ /dev/null @@ -1,156 +0,0 @@ -===================== -Demonstration Scripts -===================== - -A tour of ``pcapkit`` by example: each script here does one thing and prints or dumps -the result, so you can get a thorough view of the library either on how to use it or -on what it can do. They are demos rather than tests -- ``pytest`` does not collect -this directory (``testpaths = ["tests"]``) and nothing in CI runs them, so they are -also where API drift goes unnoticed the longest. Run them after changing anything -public. - -Running them ------------- - -Every script reads its capture with a path relative to its own directory, so **run -them from inside** ``examples/legacy_smoke/``, not from the repository root: - -.. code-block:: shell - - cd examples/legacy_smoke - python test_basic.py - -The captures live in ``../captures/``. Only ``in.pcap`` and ``dhcp.pcapng`` are -committed; the rest are generated, so build them once before running anything else: - -.. code-block:: shell - - python ../generators/make_samples.py # or: make samples, from the repository root - -The scripts ------------ - -.. list-table:: - :header-rows: 1 - - * - Script - - What it demonstrates - * - `test_basic `_ - - the smallest useful call -- extract ``in.pcap`` to a tree-view text file - * - `test_extractor `_ - - ``pcapkit.extract`` dumping the same capture as plist, JSON and tree - * - `test_file `_ - - reading from an already-open binary file object rather than a path - * - `test_api `_ - - most of the keyword surface at once: per-frame files, reassembly and flow - tracing - * - `test_tcp `_ - - extraction of TCP packets - * - `test_ipv6 `_ - - extraction of IPv6 packets, one output file per frame - * - `test_pcapng `_ - - extraction of a PCAP-NG capture, which uses a different engine - * - `test_http `_ - - iterating frames manually and testing ``pcapkit.HTTP in frame`` - * - `test_reassembly `_ - - TCP payload reassembly, writing the reassembled datagrams out - * - `test_ip_reasm `_ - - IPv4 fragment reassembly - * - `test_ipv6_reasm `_ - - IPv6 fragment reassembly - * - `test_analyse `_ - - analysis of the application layer *after* reassembly, printing the recovered - HTTP messages - * - `test_trace `_ - - tracing TCP flows and dumping the flow index - * - `test_engine `_ - - the same capture through each extraction engine, side by side - * - `test_time `_ - - timing each engine, reported as milliseconds per packet - * - `test_profile `_ - - cProfile stats for one extraction; ``make profile`` renders the call graph - * - `test_perf `_ - - the same comparison under the ``pyperf`` benchmark harness - * - `test_stream `_ - - extracting a live capture streamed from ``tcpdump`` on stdin - * - `test_stream_askpass `_ - - the same, with ``sudo -A`` for a non-interactive password - -``_engine_support.py`` is not a demo -- it is the shared helper the engine demos use -to work out whether an engine can run on this host. - -What needs more than pcapkit ----------------------------- - -Most of these run with nothing but ``pcapkit``. These do not: - -- **test_engine, test_time, test_perf** exercise the ``dpkt``, ``scapy`` and - ``pyshark`` engines. Those are optional packages, and ``pyshark`` additionally - drives Wireshark's ``tshark`` binary. An engine that is missing is reported as - skipped, with the reason, and the rest still run -- see ``_engine_support.py``. Two - things worth knowing: - - - ``pyshark`` 0.6 does not work on Python 3.14 at all. It asks - ``asyncio.get_event_loop_policy().get_event_loop()`` for a loop on a thread that - has none, which 3.14 no longer creates implicitly, so every extraction through it - raises ``RuntimeError: There is no current event loop in thread 'MainThread'`` - before ``tshark`` is even reached. That is a pyshark bug, not a pcapkit one, and - there is nothing to configure around it. - - When an engine's *package* is missing, ``pcapkit.extract`` does not fail. It warns - and falls back to pcapkit's own parser, so the extraction succeeds under the wrong - engine. The demos name the driver that actually ran, so a fallback is visible - rather than being reported as a healthy result. - -- **test_perf** needs the ``pyperf`` harness, which ``pcapkit`` does not depend on: - ``python -m pip install pyperf``. Without it the script says so and stops. - ``test_time.py`` does the same comparison with no extra dependency. -- **test_profile**'s ``make profile`` target needs ``gprof2dot``, ``graphviz`` and - ``snakeviz`` to render the call graph. The script itself only needs ``pcapkit``. -- **test_stream, test_stream_askpass** run ``sudo tcpdump`` against a live interface, - so they need root and cannot run unattended. ``INTERFACE`` at the top of each is - ``en0`` (macOS); on Linux it is usually ``eth0`` or ``wlan0``. They buffer the live - capture into a temporary file -- never into ``../captures/``, which holds generated - captures the test suite pins. - -Regenerating the reference reports ----------------------------------- - -Four files in ``../captures/`` are reports rather than inputs -- ``out.json``, -``out.plist``, ``out.txt`` and ``pcapng.txt`` -- and they are produced by two of the -scripts here. **Git does not track them**, so a fresh clone does not have them. Build -them the way you would any other generated capture, and again after a change to how -``pcapkit`` renders one: - -.. code-block:: shell - - cd examples/legacy_smoke && make fixtures - -That is the whole recipe. It runs, equivalently: - -.. code-block:: shell - - TZ=UTC python test_extractor.py # ../captures/out.json, out.plist, out.txt <- in.pcap - TZ=UTC python test_pcapng.py # ../captures/pcapng.txt <- dhcp.pcapng - -Both *inputs* are committed, so this works on a fresh clone with no ``make samples`` -first. Use ``make fixtures PYTHON=../../.venv/bin/python`` to pick a specific -interpreter. - -These four were tracked once, and #685 is why they are not any more. Nothing reads a -report back -- no test asserts on one, and the docs only quote them as prose -- so a -parser change that alters how a frame renders leaves the committed copy quietly wrong, -with no failure anywhere to say so. ``pcapng.txt`` sat recording ``packet -> NIL`` for -four blocks that had started carrying their captured octets, and the staleness was -found by reading the file rather than by anything breaking. Generating them on demand -means the copy on your disk describes the tree you are standing in, which was the only -thing that made them worth looking at. - -``TZ=UTC`` **is not decoration.** ``pcapkit`` renders frame timestamps in the host's -local zone, so an unpinned run rewrites every timestamp line in all four files with -wherever it happened to be run -- which is how these reports came to disagree with -each other while they were still tracked, the PCAP three having been generated at -UTC-05:00 and ``pcapng.txt`` at UTC+08:00. Pinning UTC makes the output reproducible on -any host, so two runs can be diffed against each other. It also happens to be the only -zone in which PCAP-NG's ``timestamp_epoch`` is the true UNIX epoch: -``PCAPNG._read_timestamp`` adds the zone's UTC offset to the value it returns, so a -report generated anywhere else bakes in that offset. diff --git a/setup.py b/setup.py index 9b99e1a570..34f96857e5 100644 --- a/setup.py +++ b/setup.py @@ -47,7 +47,7 @@ def refactor(path: "str") -> "None": ) except subprocess.CalledProcessError as error: logger.error( - "Failed to perform assignment expression backport compiling. " + "Failed to perform f-string backport compiling. " "Please consider manually install `bpc-f2format` and try again." ) sys.exit(error.returncode) @@ -70,7 +70,7 @@ def refactor(path: "str") -> "None": ) except subprocess.CalledProcessError as error: logger.error( - "Failed to perform assignment expression backport compiling. " + "Failed to perform positional-only parameter backport compiling. " "Please consider manually install `bpc-poseur` and try again." ) sys.exit(error.returncode) @@ -156,24 +156,9 @@ def run(self) -> "None": "setuptools version is too old to support PyBPC conversion in install." ) -try: - from setuptools.command.bdist_wheel import bdist_wheel - - class pcapkit_bdist_wheel(bdist_wheel): - """Modified bdist_wheel to run PyBPC conversion.""" - - def run(self) -> "None": - super(pcapkit_bdist_wheel, self).run() - logger.info("running bdist_wheel") - - # PyBPC compatibility enforcement - refactor(os.path.join(self.dist_dir, "pcapkit")) # type: ignore[arg-type] - - cmdclass["bdist_wheel"] = pcapkit_bdist_wheel -except ImportError: - logger.warning( - "setuptools version is too old to support PyBPC conversion in bdist_wheel." - ) +# No bdist_wheel override: ``bdist_wheel`` runs ``build_py`` and ``install`` +# above, so the tree it archives is already converted before the wheel is +# written, and ``dist_dir`` holds only the finished wheels (#1773). setup( cmdclass=cmdclass, diff --git a/tests/project/test_build_leftovers_1773.py b/tests/project/test_build_leftovers_1773.py new file mode 100644 index 0000000000..425db7bc37 --- /dev/null +++ b/tests/project/test_build_leftovers_1773.py @@ -0,0 +1,264 @@ +# -*- coding: utf-8 -*- +"""Pins for the build and packaging leftovers fixed in #1773. + +* Every PyBPC conversion in :file:`setup.py` lands in a package tree before a + wheel exists, so none of them is a no-op on ``dist_dir``. Measured by building + a real wheel of a stub package with :file:`setup.py`'s own ``cmdclass``. +* Each backport's failure message names the feature that tool back-ports. +* ``make profile`` reaches a Makefile that exists, and every ``prune`` in + :file:`MANIFEST.in` names a directory the repository tracks or ignores. +* No reStructuredText is tracked outside ``docs/`` (the #719 ruling), the + examples' Markdown READMEs still ship in the sdist, and nothing points at a + README path that no longer exists. + +""" + +from __future__ import annotations + +import importlib.util +import json +import logging +import pathlib +import re +import shutil +import subprocess # nosec: B404 +import sys +import tempfile +import textwrap +import types +import unittest +from unittest import mock + +ROOT = pathlib.Path(__file__).resolve().parents[2] + + +def tracked() -> 'list[str]': + """Every path git tracks, or skip when this is not a checkout.""" + if shutil.which('git') is None or not (ROOT / '.git').exists(): + raise unittest.SkipTest('not a git checkout') + result = subprocess.run(['git', 'ls-files', '-z'], cwd=ROOT, # nosec + capture_output=True, check=True) + return [name for name in result.stdout.decode('utf-8').split('\0') if name] + + +def is_history(path: 'str') -> 'bool': + """The changelogs record old filenames on purpose.""" + return any(part.lower().startswith('changelog') for part in pathlib.PurePosixPath(path).parts) + + +#: Builds a wheel of a one-module stub package with setup.py's ``cmdclass``, +#: ``refactor`` replaced by a recorder, and writes what it saw as JSON. +WHEEL_DRIVER = textwrap.dedent(''' + import importlib.util, json, os, sys, zipfile + from unittest import mock + + import setuptools + from setuptools.dist import Distribution + + root, work, report = sys.argv[1:4] + captured = {} + with mock.patch.object(setuptools, 'setup', lambda **kwargs: captured.update(kwargs)): + spec = importlib.util.spec_from_file_location('project_setup', os.path.join(root, 'setup.py')) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + os.chdir(work) + os.makedirs('pcapkit') + with open(os.path.join('pcapkit', '__init__.py'), 'w') as file: + file.write('VALUE = 1\\n') + dist_dir = os.path.join(work, 'dist') + calls = [] + + def record(path): + init = os.path.join(path, '__init__.py') + wheels = os.path.isdir(dist_dir) and [n for n in os.listdir(dist_dir) if n.endswith('.whl')] + calls.append({'path': os.path.relpath(os.path.abspath(path), work), + 'package': os.path.isfile(init), 'after_wheel': bool(wheels)}) + if os.path.isfile(init): + with open(init, 'a') as file: + file.write('CONVERTED = True\\n') + + module.refactor = record + dist = Distribution({'name': 'probe', 'version': '0', 'packages': ['pcapkit'], + 'cmdclass': captured['cmdclass']}) + dist.script_name = 'setup.py' + dist.script_args = ['-q', 'bdist_wheel', '--dist-dir', dist_dir] + dist.parse_command_line() + dist.run_commands() + wheels = [n for n in os.listdir(dist_dir) if n.endswith('.whl')] + with zipfile.ZipFile(os.path.join(dist_dir, wheels[0])) as archive: + shipped = archive.read('pcapkit/__init__.py').decode() + with open(report, 'w') as file: + json.dump({'calls': calls, 'shipped': shipped}, file) +''') + + +class WheelConversionTests(unittest.TestCase): + """A wheel is archived from a converted tree, and nothing converts ``dist_dir``.""" + + @classmethod + def setUpClass(cls) -> None: + with tempfile.TemporaryDirectory() as work: + report = pathlib.Path(work) / 'report.json' + project = pathlib.Path(work) / 'project' + project.mkdir() + result = subprocess.run([sys.executable, '-c', WHEEL_DRIVER, # nosec + str(ROOT), str(project), str(report)], + capture_output=True, text=True, timeout=300) + if result.returncode != 0: + raise AssertionError(f'wheel build failed:\n{result.stdout}\n{result.stderr}') + cls.report = json.loads(report.read_text(encoding='utf-8')) + + def test_a_conversion_runs_during_the_wheel_build(self) -> None: + self.assertTrue(self.report['calls']) + + def test_every_conversion_targets_an_existing_package_tree(self) -> None: + for call in self.report['calls']: + with self.subTest(path=call['path']): + self.assertTrue(call['package'], f"refactor() pointed at {call['path']}, " + 'which holds no package') + + def test_no_conversion_runs_after_the_wheel_is_written(self) -> None: + for call in self.report['calls']: + with self.subTest(path=call['path']): + self.assertFalse(call['after_wheel'], f"refactor({call['path']}) ran after " + 'the wheel was written, so it cannot change it') + + def test_the_wheel_ships_the_converted_tree(self) -> None: + self.assertIn('CONVERTED = True', self.report['shipped']) + + +class BackportMessageTests(unittest.TestCase): + """Each backport's failure names the feature its tool back-ports.""" + + def load_setup(self) -> 'types.ModuleType': + fakes = {'setuptools': types.ModuleType('setuptools')} + fakes['setuptools'].setup = lambda **kwargs: None # type: ignore[attr-defined] + for name in ('sdist', 'build_py', 'develop', 'install'): + module = types.ModuleType(f'setuptools.command.{name}') + setattr(module, name, type(name, (), {})) + fakes[module.__name__] = module + logger = logging.getLogger('pcapkit.setup') + handlers = logger.handlers[:] + self.addCleanup(setattr, logger, 'handlers', handlers) + with mock.patch.dict(sys.modules, fakes): + spec = importlib.util.spec_from_file_location('project_setup_1773', ROOT / 'setup.py') + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + def failure_message(self, tool: 'str') -> 'str': + module = self.load_setup() + + def check_call(argv: 'list[str]') -> 'int': + if argv[2] == tool: + raise subprocess.CalledProcessError(2, argv) + return 0 + + with mock.patch.object(module, 'sys') as fake_sys, \ + mock.patch('subprocess.check_call', side_effect=check_call), \ + self.assertLogs('pcapkit.setup', 'ERROR') as logs: + fake_sys.executable = sys.executable + fake_sys.version_info = (3, 5) + fake_sys.exit.side_effect = SystemExit + with self.assertRaises(SystemExit): + module.refactor('pcapkit') + self.assertEqual(len(logs.output), 1) + return logs.output[0] + + def test_each_failure_names_its_own_feature(self) -> None: + for tool, feature in (('f2format', 'f-string'), + ('walrus', 'assignment expression'), + ('poseur', 'positional-only parameter')): + with self.subTest(tool=tool): + message = self.failure_message(tool) + self.assertIn(f'{feature} backport', message) + self.assertIn(f'bpc-{tool}', message) + + +class MakefileTests(unittest.TestCase): + """``make profile`` reaches a Makefile that exists.""" + + def test_every_sub_make_directory_exists(self) -> None: + text = (ROOT / 'Makefile').read_text(encoding='utf-8') + directories = re.findall(r'\$\(MAKE\) -C (\S+)', text) + self.assertIn('examples/legacy_smoke', directories) + for directory in directories: + with self.subTest(directory=directory): + self.assertTrue((ROOT / directory / 'Makefile').is_file()) + + @unittest.skipUnless(shutil.which('make'), 'make is not installed') + def test_make_profile_resolves(self) -> None: + result = subprocess.run(['make', '-n', 'profile'], cwd=ROOT, # nosec + capture_output=True, text=True, timeout=60) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn('test_profile.py', result.stdout) + + +class ManifestTests(unittest.TestCase): + """What :file:`MANIFEST.in` names exists, and the examples' READMEs still ship.""" + + def test_every_prune_names_a_tracked_or_ignored_directory(self) -> None: + files = tracked() + for line in (ROOT / 'MANIFEST.in').read_text(encoding='utf-8').splitlines(): + if not line.startswith('prune '): + continue + directory = line.split()[1] + with self.subTest(directory=directory): + if any(name.startswith(f'{directory}/') for name in files): + continue + ignored = subprocess.run(['git', 'check-ignore', '-q', '--no-index', # nosec + f'{directory}/'], cwd=ROOT, check=False) + self.assertEqual(ignored.returncode, 0, + f'prune {directory} names a directory git neither tracks nor ignores') + + def test_no_restructuredtext_outside_docs(self) -> None: + stray = [name for name in tracked() + if name.endswith('.rst') and not name.startswith('docs/')] + self.assertEqual(stray, []) + + def test_the_examples_markdown_ships(self) -> None: + files = tracked() + markdown = sorted(name for name in files + if name.startswith('examples/') and name.endswith('.md')) + self.assertIn('examples/benchmark/README.md', markdown) + self.assertIn('examples/legacy_smoke/README.md', markdown) + driver = textwrap.dedent(''' + import json, sys + from setuptools.command.egg_info import FileList + listing = FileList() + listing.allfiles = json.loads(sys.stdin.read()) + with open('MANIFEST.in', encoding='utf-8') as file: + for line in file: + line = line.strip() + if line and not line.startswith('#'): + listing.process_template_line(line) + print(json.dumps(sorted(set(listing.files)))) + ''') + result = subprocess.run([sys.executable, '-c', driver], cwd=ROOT, # nosec + input=json.dumps(files), capture_output=True, text=True, + check=True, timeout=120) + shipped = set(json.loads(result.stdout)) + self.assertEqual([name for name in markdown if name not in shipped], []) + + def test_no_reference_to_a_missing_examples_readme(self) -> None: + missing = [] + for name in tracked(): + path = ROOT / name + if is_history(name) or path.suffix in {'.pcap', '.pcapng', '.png', '.gz'} \ + or not path.is_file(): + continue + text = path.read_text(encoding='utf-8', errors='ignore') + cited = set(re.findall(r'examples/[\w/]+/README\.\w+', text)) + if name.startswith('examples/'): + parent = pathlib.PurePosixPath(name).parent + cited.update(str(parent / bare) for bare in + re.findall(r'(? 'tuple[str, ...]': PROSE_SCAN = ( ('.github', ('*.yml', '*.md')), ('docs', ('*.rst', '*.py')), - ('examples', ('*.rst', '*.txt', '*.py', '*.sh')), + ('examples', ('*.md', '*.txt', '*.py', '*.sh')), ('pcapkit', ('*.py',)), ('tests', ('*.py',)), ('util', ('*.py',)), diff --git a/tests/project/test_setup.py b/tests/project/test_setup.py index 494fb58703..455d739f6e 100644 --- a/tests/project/test_setup.py +++ b/tests/project/test_setup.py @@ -28,7 +28,6 @@ def fake_setup(*args, **kwargs): 'setuptools.command.build_py': 'build_py', 'setuptools.command.develop': 'develop', 'setuptools.command.install': 'install', - 'setuptools.command.bdist_wheel': 'bdist_wheel', } loaded = ['setuptools']