From 8c1fe8cf85eec11c6d714ce8a0cf0ed16fb02b64 Mon Sep 17 00:00:00 2001 From: Andrea Lacava Date: Tue, 25 Aug 2026 23:41:13 -0400 Subject: [PATCH 1/4] feat(build): make the SCTP transport an optional dependency A downstream container image that builds libe3 without the SCTP headers fails: src/connector/posix_connector.cpp:19:10: fatal error: netinet/sctp.h: No such file or directory posix_connector.cpp included unconditionally, so every consumer had to install libsctp-dev whether or not it ever used SCTP. Images that happened to install it for another reason built fine; images that did not could not build libe3 at all, for a transport they were never going to use. Gate it behind LIBE3_ENABLE_SCTP, default OFF. SCTP is the only dependency that needs a kernel module plus a distro -dev package for a single system header, while deployments typically run the E3 link over IPC or TCP. What does NOT change: E3TransportLayer::SCTP stays in the enum, at value 0. It is part of the ABI and of the wire-facing value the C API and the Python bindings mirror, so removing or renumbering it would break them. transport_layer_to_string and its test are untouched. What changes: with the flag off, a request for the SCTP transport fails at socket creation with a message naming the flag, instead of falling through to another protocol -- which would look like it worked while putting traffic on the wrong one. The four socket() sites now go through one sctp_stream_socket() shim that returns -1 with errno=EPROTONOSUPPORT, which every existing caller's error path already handles. libe3 calls no libsctp function -- only IPPROTO_SCTP and SCTP_NODELAY, both from the header -- so there is nothing to link and no find_package to add. The dependency is the header alone, checked at configure time so -DLIBE3_ENABLE_SCTP=ON without it fails naming the package, rather than partway through the build. The define is PUBLIC, so it reaches libe3.pc: a consumer can tell whether asking for SCTP will work instead of finding out at socket(). build_libe3 grows --enable-sctp/--disable-sctp and no longer installs the SCTP headers with --install-deps; README documents the flag, the packages per distro, and that the enum survives either way. Verified in a container with no libsctp-dev, which is the environment that was failing: * default (off): configures and builds to completion * -DLIBE3_ENABLE_SCTP=ON: fails at configure with the package names * preprocessing posix_connector.cpp with the flag off shows netinet/sctp.h is not reached at all Both arms also build on a host that does have the header. Assisted-by: Claude Opus 5 --- README.md | 18 ++++++++++++-- build_libe3 | 19 ++++++++++++--- cmake/libe3Dependencies.cmake | 17 +++++++++++++ cmake/libe3Options.cmake | 9 +++++++ cmake/libe3Targets.cmake | 10 ++++++++ src/connector/posix_connector.cpp | 40 +++++++++++++++++++++++++++---- 6 files changed, 104 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index d6cb7b8e..addda224 100644 --- a/README.md +++ b/README.md @@ -79,7 +79,7 @@ - `nlohmann-json3-dev` — Header-only JSON library; CMake expects the `nlohmann_json` target - `protobuf-compiler` + `libprotobuf-dev` — `protoc` and the Protocol Buffers runtime (only when `LIBE3_ENABLE_PROTOBUF=ON`) - `libzmq3-dev` for ZMQ transport -- `libsctp-dev` — SCTP development headers/libraries for POSIX/SCTP transport +- `libsctp-dev` — SCTP headers for the POSIX/SCTP transport (only when `LIBE3_ENABLE_SCTP=ON`; **not installed by `./build_libe3 -I`**) ### Install Dependencies @@ -91,7 +91,7 @@ Install all required packages using the project's installer (recommended) or man # Manual (Debian/Ubuntu) sudo apt update -sudo apt install -y build-essential cmake pkg-config libzmq3-dev ninja-build git asn1c nlohmann-json3-dev protobuf-compiler libprotobuf-dev libsctp-dev dpkg-dev debhelper fakeroot +sudo apt install -y build-essential cmake pkg-config libzmq3-dev ninja-build git asn1c nlohmann-json3-dev protobuf-compiler libprotobuf-dev dpkg-dev debhelper fakeroot ``` The packaging tools (`dpkg-dev`, `debhelper`, `fakeroot`) are only needed by `scripts/create_deb.sh`. @@ -145,6 +145,20 @@ make -j$(nproc) | `LIBE3_ENABLE_ASAN` | OFF | Enable AddressSanitizer | | `LIBE3_ENABLE_TSAN` | OFF | Enable ThreadSanitizer | | `LIBE3_ENABLE_SWIG` | OFF | Build the SWIG-generated Python bindings (`_libe3py.so` + `libe3py.py`) | +| `LIBE3_ENABLE_SCTP` | OFF | Enable the SCTP POSIX transport (needs `libsctp-dev`) | + +> **Note on SCTP:** the transport is **off by default**. It is the only dependency that needs a +> kernel module plus a distro `-dev` package for a single system header, and the deployed E3 links +> are IPC or TCP, so `./build_libe3 -I` no longer installs it. `E3TransportLayer::SCTP` still exists +> in the ABI, and in the C API and Python bindings that mirror its value, whichever way the flag is +> set — but asking a build without it for the SCTP transport fails at socket creation with a message +> telling you to rebuild, rather than silently falling back to another protocol. Configuring with +> `-DLIBE3_ENABLE_SCTP=ON` and no headers installed fails at configure time, naming the package. +> +> ```bash +> sudo apt install -y libsctp-dev # lksctp-tools-devel on Fedora/RHEL, lksctp-tools on Arch +> ./build_libe3 --enable-sctp +> ``` > **Note on encoding selection:** `LIBE3_ENABLE_ASN1`, `LIBE3_ENABLE_JSON`, and > `LIBE3_ENABLE_PROTOBUF` are independent compile-time inclusion flags — any combination can be diff --git a/build_libe3 b/build_libe3 index ccbc307e..98d1dad4 100755 --- a/build_libe3 +++ b/build_libe3 @@ -33,6 +33,7 @@ ENABLE_ZMQ=1 ENABLE_ASN1=1 ENABLE_JSON=1 ENABLE_PROTOBUF=0 +ENABLE_SCTP=0 ENABLE_ASAN=0 ENABLE_TSAN=0 ENABLE_SWIG=0 @@ -110,6 +111,12 @@ Features: --disable-json Disable JSON encoding --enable-protobuf Enable Protocol Buffers encoding support (default: OFF) --disable-protobuf Disable Protocol Buffers encoding + --enable-sctp Enable the SCTP POSIX transport (default: OFF). + Needs the SCTP headers: libsctp-dev (Debian/Ubuntu), + lksctp-tools-devel (Fedora/RHEL), lksctp-tools (Arch). + --install-deps does NOT install them; SCTP is optional + and the deployed E3 links are IPC or TCP. + --disable-sctp Disable the SCTP transport --enable-swig Build + install the Python (SWIG) bindings, libe3py (default: OFF). With --install, installs libe3py into the active interpreter's site-packages. @@ -299,7 +306,6 @@ install_dependencies() { bison \ flex \ nlohmann-json3-dev \ - libsctp-dev \ protobuf-compiler \ libprotobuf-dev \ swig \ @@ -318,7 +324,6 @@ install_dependencies() { git \ asn1c \ nlohmann-json-devel \ - lksctp-tools-devel \ protobuf-compiler \ protobuf-devel \ swig \ @@ -334,7 +339,6 @@ install_dependencies() { git \ asn1c \ nlohmann-json \ - lksctp-tools \ protobuf \ swig \ python @@ -386,6 +390,7 @@ configure_cmake() { "-DLIBE3_ENABLE_ASN1=$([ $ENABLE_ASN1 -eq 1 ] && echo ON || echo OFF)" "-DLIBE3_ENABLE_JSON=$([ $ENABLE_JSON -eq 1 ] && echo ON || echo OFF)" "-DLIBE3_ENABLE_PROTOBUF=$([ $ENABLE_PROTOBUF -eq 1 ] && echo ON || echo OFF)" + "-DLIBE3_ENABLE_SCTP=$([ $ENABLE_SCTP -eq 1 ] && echo ON || echo OFF)" "-DLIBE3_ENABLE_ASAN=$([ $ENABLE_ASAN -eq 1 ] && echo ON || echo OFF)" "-DLIBE3_ENABLE_TSAN=$([ $ENABLE_TSAN -eq 1 ] && echo ON || echo OFF)" "-DLIBE3_ENABLE_SWIG=$([ $ENABLE_SWIG -eq 1 ] && echo ON || echo OFF)" @@ -670,6 +675,14 @@ main() { ENABLE_PROTOBUF=1 shift ;; + --enable-sctp) + ENABLE_SCTP=1 + shift + ;; + --disable-sctp) + ENABLE_SCTP=0 + shift + ;; --disable-protobuf) ENABLE_PROTOBUF=0 shift diff --git a/cmake/libe3Dependencies.cmake b/cmake/libe3Dependencies.cmake index 21629a81..2ee355cb 100644 --- a/cmake/libe3Dependencies.cmake +++ b/cmake/libe3Dependencies.cmake @@ -49,6 +49,23 @@ message(STATUS "tl::expected: Fetched from GitHub") # Optional Dependencies # ============================================================================ +# Optional: SCTP. Header-only from libe3's point of view -- the connector uses +# IPPROTO_SCTP and SCTP_NODELAY from and calls no libsctp +# function, so there is nothing to link. Checked here so -DLIBE3_ENABLE_SCTP=ON +# without the -dev package fails at configure time with the package name, rather +# than partway through the build with a bare "netinet/sctp.h: No such file". +if(LIBE3_ENABLE_SCTP) + include(CheckIncludeFile) + check_include_file("netinet/sctp.h" LIBE3_HAVE_SCTP_H) + if(NOT LIBE3_HAVE_SCTP_H) + message(FATAL_ERROR + "LIBE3_ENABLE_SCTP=ON but was not found. Install the " + "SCTP headers (Debian/Ubuntu: libsctp-dev, Fedora/RHEL: " + "lksctp-tools-devel, Arch: lksctp-tools), or configure with " + "-DLIBE3_ENABLE_SCTP=OFF (the default).") + endif() +endif() + # Optional: Google Benchmark for the integration micro-benchmarks # (dev-only; never linked into the shipped library) if(LIBE3_BUILD_INTEGRATION_TESTS) diff --git a/cmake/libe3Options.cmake b/cmake/libe3Options.cmake index 801508e8..fd804777 100644 --- a/cmake/libe3Options.cmake +++ b/cmake/libe3Options.cmake @@ -9,6 +9,15 @@ option(LIBE3_BUILD_INTEGRATION_TESTS option(LIBE3_ENABLE_SWIG "Build the SWIG Python bindings" OFF) option(LIBE3_ENABLE_ZMQ "Enable ZeroMQ transport" ON) +# SCTP is an optional POSIX transport. OFF by default because it is the only +# dependency that needs a kernel module plus a distro -dev package (libsctp-dev / +# lksctp-tools-devel) for a single system header, while deployments typically run +# the E3 link over IPC or TCP and never reach for it. The E3TransportLayer::SCTP +# enumerator exists either way -- it is part of the ABI and of the wire-facing +# value that the C API and the Python bindings mirror -- but requesting it from a +# build with this OFF fails at socket creation with a clear message rather than +# silently doing something else. +option(LIBE3_ENABLE_SCTP "Enable the SCTP POSIX transport (needs libsctp-dev)" OFF) option(LIBE3_ENABLE_ASN1 "Enable ASN.1 encoding support" ON) option(LIBE3_ENABLE_JSON "Enable JSON encoding support" OFF) option(LIBE3_ENABLE_PROTOBUF "Enable Protocol Buffers encoding support" OFF) diff --git a/cmake/libe3Targets.cmake b/cmake/libe3Targets.cmake index 890305ec..65df668f 100644 --- a/cmake/libe3Targets.cmake +++ b/cmake/libe3Targets.cmake @@ -69,6 +69,12 @@ else() target_compile_definitions(libe3 PUBLIC LIBE3_HAS_ZMQ=0) endif() +if(LIBE3_ENABLE_SCTP) + # PUBLIC so it reaches libe3.pc: a consumer can then tell whether requesting + # E3TransportLayer::SCTP will work, instead of finding out at socket(). + target_compile_definitions(libe3 PUBLIC LIBE3_ENABLE_SCTP) +endif() + if(LIBE3_ENABLE_LATREC) # PUBLIC: any consumer building against this target (OAI, flexric) must # see the same define, or its own latrec_* calls silently resolve to the @@ -139,6 +145,10 @@ else() target_compile_definitions(libe3_shared PUBLIC LIBE3_HAS_ZMQ=0) endif() +if(LIBE3_ENABLE_SCTP) + target_compile_definitions(libe3_shared PUBLIC LIBE3_ENABLE_SCTP) +endif() + if(LIBE3_ENABLE_LATREC) target_compile_definitions(libe3_shared PUBLIC LIBE3_ENABLE_LATREC) if(LATREC_DEFAULT_DIR) diff --git a/src/connector/posix_connector.cpp b/src/connector/posix_connector.cpp index 5c49d344..4fa8b57e 100644 --- a/src/connector/posix_connector.cpp +++ b/src/connector/posix_connector.cpp @@ -16,7 +16,9 @@ #include #include #include +#ifdef LIBE3_ENABLE_SCTP #include +#endif #include #include #include @@ -62,13 +64,43 @@ int wait_for_socket(int sockfd, int timeout_ms) { * every accepted and connected TCP/SCTP data socket (Linux does not reliably * inherit it from the listener). No-op for UNIX-domain (IPC) sockets. */ +/** + * @brief Create the AF_INET stream socket for an SCTP endpoint. + * + * SCTP is optional at build time (LIBE3_ENABLE_SCTP, off by default): it is the + * only transport needing a kernel module and a distro -dev package for one + * header, and the deployed E3 links are IPC or TCP. The enumerator stays in the + * ABI either way, so a caller can still ask for SCTP against a build without it + * -- that has to fail loudly here rather than quietly fall through to TCP, which + * would look like it worked while putting traffic on the wrong protocol. + * + * Returns -1 with errno set, matching socket(), so every existing caller's error + * path already handles it. + */ +int sctp_stream_socket() { +#ifdef LIBE3_ENABLE_SCTP + return socket(AF_INET, SOCK_STREAM, IPPROTO_SCTP); +#else + E3_LOG_ERROR(LOG_TAG) << "SCTP transport requested but libe3 was built without it. " + "Rebuild with -DLIBE3_ENABLE_SCTP=ON (needs libsctp-dev), " + "or use the TCP or IPC transport."; + errno = EPROTONOSUPPORT; + return -1; +#endif +} + void set_nodelay(int sockfd, E3TransportLayer transport) { int one = 1; int ret = 0; if (transport == E3TransportLayer::TCP) { ret = setsockopt(sockfd, IPPROTO_TCP, TCP_NODELAY, &one, sizeof(one)); } else if (transport == E3TransportLayer::SCTP) { +#ifdef LIBE3_ENABLE_SCTP ret = setsockopt(sockfd, IPPROTO_SCTP, SCTP_NODELAY, &one, sizeof(one)); +#else + // Unreachable: sctp_stream_socket() already failed for this transport. + (void)one; +#endif } if (ret != 0) { E3_LOG_WARN(LOG_TAG) << "Failed to set NODELAY: " << strerror(errno); @@ -223,7 +255,7 @@ ErrorCode PosixE3Connector::setup_initial_connection() { } if (transport_layer_ == E3TransportLayer::SCTP) { - sock = socket(AF_INET, SOCK_STREAM, IPPROTO_SCTP); + sock = sctp_stream_socket(); int reuse = 1; setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, &reuse, sizeof(reuse)); struct sockaddr_in addr{}; @@ -370,7 +402,7 @@ ErrorCode PosixE3Connector::setup_inbound_connection() { } if (transport_layer_ == E3TransportLayer::SCTP) { - sock = socket(AF_INET, SOCK_STREAM, IPPROTO_SCTP); + sock = sctp_stream_socket(); int reuse = 1; setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, &reuse, sizeof(reuse)); struct sockaddr_in addr{}; @@ -554,7 +586,7 @@ ErrorCode PosixE3Connector::setup_outbound_connection() { } if (transport_layer_ == E3TransportLayer::SCTP) { - sock = socket(AF_INET, SOCK_STREAM, IPPROTO_SCTP); + sock = sctp_stream_socket(); struct sockaddr_in addr{}; addr.sin_family = AF_INET; addr.sin_port = htons(outbound_port_); @@ -697,7 +729,7 @@ int posix_connect_for(E3TransportLayer transport, } int sock; if (transport == E3TransportLayer::SCTP) { - sock = socket(AF_INET, SOCK_STREAM, IPPROTO_SCTP); + sock = sctp_stream_socket(); } else { sock = socket(AF_INET, SOCK_STREAM, 0); } From a8759709f059f176ac682aaacf10e3706a355a69 Mon Sep 17 00:00:00 2001 From: Andrea Lacava Date: Thu, 27 Aug 2026 15:01:40 -0400 Subject: [PATCH 2/4] test: make multi_peer_dispatch's port allocation collision-safe Two tests hardcoded a literal TCP base port (26990 / 25990) with no per-process derivation, so concurrent CTest runs on this shared, multi-session machine collide on the same port triple -- confirmed by running two copies of the binary at once, which reproduced the exact intermittent connect()/bind() failure seen in CI. Adds unique_base_port(), spread by pid (or an explicit LIBE3_TEST_PORT_OFFSET env var for CI runners that want disjoint ranges per job) with *3 spacing so no two processes' port triples overlap. Also fixes a real startup race in multi_peer_slow_peer_does_not_stall_others: wait_for_setup() only confirms the SETUP handshake socket, not the RAN's publisher socket that the test's raw connect() targets directly -- added connect_with_retry() with a bounded backoff instead of a bare, unretried connect(). Verified: the specific test passed 8/8 alone and under both 2-way and 4-way concurrent stress (the exact collision scenario) after this fix. --- .../integration/test_multi_peer_dispatch.cpp | 54 +++++++++++++++---- 1 file changed, 44 insertions(+), 10 deletions(-) diff --git a/tests/integration/test_multi_peer_dispatch.cpp b/tests/integration/test_multi_peer_dispatch.cpp index e7180fe1..883aec2b 100644 --- a/tests/integration/test_multi_peer_dispatch.cpp +++ b/tests/integration/test_multi_peer_dispatch.cpp @@ -54,6 +54,41 @@ std::string make_tmpdir() { return std::string(d); } +// A literal base port collides across concurrent CTest runs on a shared, +// multi-session machine (this repo's normal state). LIBE3_TEST_PORT_OFFSET +// lets a CI runner assign each parallel job a disjoint range explicitly; +// absent that, spread by pid so co-resident local runs don't collide either. +// *3 keeps one process's whole triple (base, base+1, base+2) from +// overlapping another process's triple. +uint16_t unique_base_port(uint16_t default_base) { + if (const char* env = std::getenv("LIBE3_TEST_PORT_OFFSET")) { + return static_cast(default_base + std::atoi(env)); + } + return static_cast(default_base + (static_cast(getpid()) % 1000) * 3); +} + +// Connects to 127.0.0.1:port, retrying with a short bounded backoff. Needed +// because a completed wait_for_setup() only confirms the SETUP handshake +// socket -- a raw connect() to a different socket (e.g. the RAN's publisher +// port) can still race its listener actually being bound. +int connect_with_retry(uint16_t port, std::chrono::milliseconds timeout) { + struct sockaddr_in addr{}; + addr.sin_family = AF_INET; + addr.sin_port = htons(port); + addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + const auto deadline = std::chrono::steady_clock::now() + timeout; + while (true) { + int fd = socket(AF_INET, SOCK_STREAM, 0); + if (fd < 0) return -1; + if (connect(fd, reinterpret_cast(&addr), sizeof(addr)) == 0) { + return fd; + } + close(fd); + if (std::chrono::steady_clock::now() >= deadline) return -1; + std::this_thread::sleep_for(20ms); + } +} + struct Endpoints { std::string setup, subscriber, publisher; }; @@ -339,9 +374,10 @@ TEST(multi_peer_two_dapps_distinct_rfs_no_crosstalk_posix_ipc) { TEST(multi_peer_two_dapps_distinct_rfs_no_crosstalk_posix_tcp) { // Unique port triple: 23990-23992 and 24990-24992 are used by other POSIX - // tests; 25990-25992 is free. + // tests; 25990-25992 is free. unique_base_port() also keeps it distinct + // across concurrent runs on a shared machine. run_multi_peer_two_dapps(E3LinkLayer::POSIX, E3TransportLayer::TCP, - /*base_port=*/25990); + /*base_port=*/unique_base_port(25990)); } // Regression test for the multi-peer broadcast blocker: a dApp that connects @@ -350,7 +386,9 @@ TEST(multi_peer_two_dapps_distinct_rfs_no_crosstalk_posix_tcp) { // RAN's outbound thread blocked forever in send() once the stalled peer's TCP // buffer filled, starving every other peer and wedging stop(). TEST(multi_peer_slow_peer_does_not_stall_others_posix_tcp) { - const uint16_t base = 26990; // distinct from the port triples above + // distinct from the port triples above; unique_base_port() also keeps it + // distinct across concurrent runs on a shared machine. + const uint16_t base = unique_base_port(26990); E3Config ran_cfg; ran_cfg.role = E3Role::RAN; @@ -381,16 +419,12 @@ TEST(multi_peer_slow_peer_does_not_stall_others_posix_tcp) { // A raw "stalled" peer: connect to the RAN's indication (publisher) port, // shrink its receive buffer, and never read. The RAN broadcasts to it too. - int stalled = socket(AF_INET, SOCK_STREAM, 0); + // Retried: wait_for_setup() above only confirms the SETUP socket, not + // this one, so the publisher's listener may not be bound yet. + int stalled = connect_with_retry(static_cast(base + 2), 2s); ASSERT_TRUE(stalled >= 0); int rcvbuf = 2048; setsockopt(stalled, SOL_SOCKET, SO_RCVBUF, &rcvbuf, sizeof(rcvbuf)); - struct sockaddr_in addr{}; - addr.sin_family = AF_INET; - addr.sin_port = htons(static_cast(base + 2)); - addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); - ASSERT_TRUE(connect(stalled, reinterpret_cast(&addr), - sizeof(addr)) == 0); // Subscribe so the SM starts broadcasting; the RAN accepts both the good // dApp and the stalled raw peer on its next send(). From de42922464c4fe649ee77cb3520f53b7cd714bb9 Mon Sep 17 00:00:00 2001 From: Andrea Lacava Date: Thu, 27 Aug 2026 15:02:01 -0400 Subject: [PATCH 3/4] build: ignore Python bytecode cache tools/__pycache__/ has been showing up as untracked in every git status this session; .gitignore had no Python entries at all. --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index ab17eaba..ef64a29c 100644 --- a/.gitignore +++ b/.gitignore @@ -78,6 +78,10 @@ conan/ docs/_build/ doxygen/ +# Python +__pycache__/ +*.pyc + # Temporary files *.tmp *.temp From 44d168446942dc4a2fa9ecac938ec38cbe24b3cc Mon Sep 17 00:00:00 2001 From: Andrea Lacava Date: Thu, 27 Aug 2026 15:13:24 -0400 Subject: [PATCH 4/4] latrec: fix underflowed E3-wire numbers in bench_full_loop_latency The posted CI report showed an "E3 wire (RAN -> dApp)" row with a mean of ~3.6e17 and a max of exactly UINT64_MAX/1000 -- a classic unsigned subtraction underflow, meaning some round trip's RECV timestamp was paired with the wrong (later) SEND_DONE. Root cause: the outbound quintuple is joined to a round trip by exact key (EMIT_ENTER.aux == business seq), but the inbound triple/pair has no such key and is joined by chronological position instead (documented in this file's own header -- the inbound side genuinely has nothing else to key on). Position-based pairing silently drifts by one for every round trip whose own inbound data didn't fully land -- most commonly at shutdown, when the SM can emit one more indication than the harness waits for a control reply to. Once drifted, every later "ii"/"ci" belongs to some other round trip, not the one it's paired with. Verified with a debug build that instrumented and printed the exact misaligned pairing, then reproduced it reliably under 20-way concurrent runs of the binary (1-4 affected round trips per run; 0/15 sequential runs hit it, matching how the original CI failure looked like a rare event). Fix: after joining both legs, check that the position-based pieces are at least causally consistent with the key-based ones (send_done <= recv, and the inbound triple's own last stamp doesn't run past the business-seq side's next stage) before accepting the round trip -- drop it otherwise, the same tolerance this file already extends to an incomplete tail. Re-ran the 20-way concurrent stress after the fix: 13/20 runs hit the drop path (1-4 round trips each), 0/20 produced a bad value. --- tests/integration/bench_full_loop_latency.cpp | 36 +++++++++++++++---- 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/tests/integration/bench_full_loop_latency.cpp b/tests/integration/bench_full_loop_latency.cpp index 03cc3be6..9ea56964 100644 --- a/tests/integration/bench_full_loop_latency.cpp +++ b/tests/integration/bench_full_loop_latency.cpp @@ -310,6 +310,7 @@ std::vector reconstruct(const std::string& dir) { }); std::vector out; + size_t skipped_misaligned = 0; size_t ind_inbound_idx = 0, ctrl_inbound_idx = 0; // std::map iterates in ascending key order, i.e. ascending business seq, // i.e. emission order -- the same order PingPong pacing guarantees the @@ -338,6 +339,29 @@ std::vector reconstruct(const std::string& dir) { if (ind_inbound_idx >= ind_inbound.size()) break; if (ctrl_inbound_idx >= ctrl_inbound.size()) break; + const Libe3Times& io = ind_out_it->second; + const Libe3Times& co = ctrl_out_it->second; + const InboundTriple& ii = ind_inbound[ind_inbound_idx++]; + const InboundPair& ci = ctrl_inbound[ctrl_inbound_idx++]; + + // ind_outbound/ctrl_outbound are joined to this round trip by exact + // key (EMIT_ENTER.aux == business seq); ii/ci are joined by + // chronological position instead, since the inbound side has no such + // key (see file header). Position-based pairing silently drifts by + // one for every round trip whose own inbound triple/pair didn't fully + // land (most commonly right at shutdown, when the SM can emit one + // more indication than the harness waits for a control reply to). + // Once drifted, ii/ci belong to some *other* round trip, not this + // one -- checking that the two joins agree on basic time ordering + // catches that and drops just the affected round trip, the same + // tolerance this file already extends to an incomplete tail. + const bool ind_ok = io.send_done <= ii.recv && ii.deliver_begin <= *t5; + const bool ctrl_ok = co.send_done <= ci.recv && ci.decode_e3ap_done <= *t8; + if (!ind_ok || !ctrl_ok) { + ++skipped_misaligned; + continue; + } + RoundTrip rt; rt.seq = seq; rt.t1 = *t1; @@ -349,31 +373,31 @@ std::vector reconstruct(const std::string& dir) { rt.t8 = *t8; rt.t9 = *t9; - const Libe3Times& io = ind_out_it->second; rt.t_ind_emit_enter = io.emit_enter; rt.t_ind_enqueue = io.enqueue; rt.t_ind_dequeue = io.dequeue; rt.t_ind_encode_e3ap_done = io.encode_e3ap_done; rt.t_ind_send_done = io.send_done; - - const InboundTriple& ii = ind_inbound[ind_inbound_idx++]; rt.t4 = ii.deliver_begin; rt.t_ind_recv = ii.recv; rt.t_ind_decode_e3ap_done = ii.decode_e3ap_done; - const Libe3Times& co = ctrl_out_it->second; rt.t_ctrl_emit_enter = co.emit_enter; rt.t_ctrl_enqueue = co.enqueue; rt.t_ctrl_dequeue = co.dequeue; rt.t_ctrl_encode_e3ap_done = co.encode_e3ap_done; rt.t_ctrl_send_done = co.send_done; - - const InboundPair& ci = ctrl_inbound[ctrl_inbound_idx++]; rt.t_ctrl_recv = ci.recv; rt.t_ctrl_decode_e3ap_done = ci.decode_e3ap_done; out.push_back(rt); } + if (skipped_misaligned > 0) { + std::fprintf(stderr, + "WARNING: dropped %zu round trip(s) with a misaligned " + "inbound position (see reconstruct()'s comment)\n", + skipped_misaligned); + } return out; }