test: harden the allocation counter, benchmarks, and golden output

CMakeLists.txt

@@ -77,6 +77,10 @@ endif()
 if((SPC_ENABLE_SANITIZERS OR SPC_ENABLE_THREAD_SANITIZER OR SPC_ENABLE_COVERAGE) AND MSVC)
     message(FATAL_ERROR "Sanitizer and coverage options support GCC and Clang only")
 endif()
+if(SPC_BUILD_BENCHMARKS AND (SPC_ENABLE_SANITIZERS OR SPC_ENABLE_THREAD_SANITIZER))
+    message(FATAL_ERROR "SPC_BUILD_BENCHMARKS cannot be combined with sanitizers: the "
+        "benchmarks replace operator new and delete, as the sanitizer runtimes do")
+endif()
 if(SPC_ENABLE_SANITIZERS)
     add_compile_options(-fsanitize=address,undefined -fno-omit-frame-pointer)
     add_link_options(-fsanitize=address,undefined)

CONTRIBUTING.md

@@ -138,6 +138,10 @@ python3 build-bench/_deps/benchmark-src/tools/compare.py benchmarks \
   build-bench/main.json build-bench/branch.json
 ```
 
+Under Valgrind, pass `--soname-synonyms=somalloc=nouserintercepts`, or
+Valgrind's own operator new replaces the counting one and the suite exits
+with an error.
+
 ## Pull requests
 
 - Branch from `main` and open a pull request against it. Every CI job must

Makefile

@@ -54,8 +54,9 @@ tidy:
 
 # Release build in build-bench, without tests and examples.
 bench:
-	@$(MAKE) --no-print-directory build BUILD_DIR=build-bench \
-		CMAKE_ARGS="-DSPC_BUILD_BENCHMARKS=ON -DSPC_BUILD_TESTS=OFF -DSPC_BUILD_EXAMPLES=OFF $(CMAKE_ARGS)"
+	@$(CMAKE) -S . -B build-bench -DCMAKE_BUILD_TYPE=Release -DSPC_BUILD_BENCHMARKS=ON \
+		-DSPC_BUILD_TESTS=OFF -DSPC_BUILD_EXAMPLES=OFF $(CMAKE_ARGS)
+	@$(CMAKE) --build build-bench -j$(NPROC)
 	@./build-bench/benchmarks/spc_benchmarks $(BENCH_ARGS)
 
 coverage:

benchmarks/benchmarks.cpp

@@ -21,11 +21,14 @@
 #include <benchmark/benchmark.h>
 #include <cstddef>
 #include <cstdint>
+#include <cstdio>
 #include <filesystem>
 #include <format>
 #include <fstream>
 #include <memory>
+#include <new>
 #include <sstream>
+#include <stdexcept>
 #include <string>
 #include <string_view>
 #include <type_traits>
@@ -295,10 +298,40 @@ void register_benchmarks() {
 	});
 }
 
+/// Whether the allocation counter sees an operator new call made here and one
+/// made inside the standard library. When it does not, the heap counters would
+/// read low without failing, so this prints the likely cause.
+bool allocation_counter_works() {
+	const spc::test::AllocationProbe probe;
+	void* block = ::operator new(64);
+	benchmark::DoNotOptimize(block);
+	::operator delete(block);
+	const std::uint64_t direct = probe.stats().allocations;
+	const std::runtime_error error("a message the standard library copies to the heap");
+	benchmark::DoNotOptimize(error.what());
+	const std::uint64_t library = probe.stats().allocations - direct;
+	if (direct != 1) {
+		std::fputs("The allocation counter misses operator new calls. Under Valgrind, run with "
+				   "--soname-synonyms=somalloc=nouserintercepts.\n",
+				   stderr);
+		return false;
+	}
+	if (library == 0) {
+		std::fputs("The allocation counter misses allocations made inside the standard library, "
+				   "which happens when LTO internalizes the replacement operator new.\n",
+				   stderr);
+		return false;
+	}
+	return true;
+}
+
 } // namespace
 
 int main(int argc, char** argv) {
 	benchmark::MaybeReenterWithoutASLR(argc, argv);
+	if (!allocation_counter_works()) {
+		return 1;
+	}
 	register_benchmarks();
 	benchmark::Initialize(&argc, argv);
 	if (benchmark::ReportUnrecognizedArguments(argc, argv)) {

tests/support/allocation_counter.cpp

@@ -20,7 +20,8 @@
 namespace {
 
 struct Counters {
-	std::atomic<bool> active{false};
+	std::atomic<bool> owned{false};	 // a probe exists
+	std::atomic<bool> active{false}; // allocations are being counted
 	std::atomic<std::uint64_t> allocations{0};
 	std::atomic<std::uint64_t> bytes{0};
 	std::atomic<std::int64_t> live{0};
@@ -118,17 +119,20 @@ std::size_t to_size(std::align_val_t alignment) noexcept {
 namespace spc::test {
 
 AllocationProbe::AllocationProbe() {
-	if (counters.active.exchange(true)) {
+	if (counters.owned.exchange(true)) {
 		throw std::logic_error("another AllocationProbe is active");
 	}
+	// Zero before counting starts, so no allocation is counted and then wiped.
 	counters.allocations.store(0);
 	counters.bytes.store(0);
 	counters.live.store(0);
 	counters.peak.store(0);
+	counters.active.store(true);
 }
 
 AllocationProbe::~AllocationProbe() {
 	counters.active.store(false);
+	counters.owned.store(false);
 }
 
 AllocationStats AllocationProbe::stats() const noexcept {
@@ -142,86 +146,96 @@ AllocationStats AllocationProbe::stats() const noexcept {
 
 } // namespace spc::test
 
-void* operator new(std::size_t size) {
+// LTO may internalize the replacements (ld64 does), and then allocations made
+// inside a shared standard library bypass them. This keeps them exported.
+#if defined(__GNUC__)
+#define SPC_REPLACEMENT __attribute__((used, visibility("default")))
+#else
+#define SPC_REPLACEMENT
+#endif
+
+SPC_REPLACEMENT void* operator new(std::size_t size) {
 	return allocate(size, 0);
 }
 
-void* operator new[](std::size_t size) {
+SPC_REPLACEMENT void* operator new[](std::size_t size) {
 	return allocate(size, 0);
 }
 
-void* operator new(std::size_t size, std::align_val_t alignment) {
+SPC_REPLACEMENT void* operator new(std::size_t size, std::align_val_t alignment) {
 	return allocate(size, to_size(alignment));
 }
 
-void* operator new[](std::size_t size, std::align_val_t alignment) {
+SPC_REPLACEMENT void* operator new[](std::size_t size, std::align_val_t alignment) {
 	return allocate(size, to_size(alignment));
 }
 
-void* operator new(std::size_t size, const std::nothrow_t& /*tag*/) noexcept {
+SPC_REPLACEMENT void* operator new(std::size_t size, const std::nothrow_t& /*tag*/) noexcept {
 	return allocate_nothrow(size, 0);
 }
 
-void* operator new[](std::size_t size, const std::nothrow_t& /*tag*/) noexcept {
+SPC_REPLACEMENT void* operator new[](std::size_t size, const std::nothrow_t& /*tag*/) noexcept {
 	return allocate_nothrow(size, 0);
 }
 
-void* operator new(std::size_t size, std::align_val_t alignment,
-				   const std::nothrow_t& /*tag*/) noexcept {
+SPC_REPLACEMENT void* operator new(std::size_t size, std::align_val_t alignment,
+								   const std::nothrow_t& /*tag*/) noexcept {
 	return allocate_nothrow(size, to_size(alignment));
 }
 
-void* operator new[](std::size_t size, std::align_val_t alignment,
-					 const std::nothrow_t& /*tag*/) noexcept {
+SPC_REPLACEMENT void* operator new[](std::size_t size, std::align_val_t alignment,
+									 const std::nothrow_t& /*tag*/) noexcept {
 	return allocate_nothrow(size, to_size(alignment));
 }
 
-void operator delete(void* ptr) noexcept {
+SPC_REPLACEMENT void operator delete(void* ptr) noexcept {
 	deallocate(ptr, 0);
 }
 
-void operator delete[](void* ptr) noexcept {
+SPC_REPLACEMENT void operator delete[](void* ptr) noexcept {
 	deallocate(ptr, 0);
 }
 
-void operator delete(void* ptr, std::size_t /*size*/) noexcept {
+SPC_REPLACEMENT void operator delete(void* ptr, std::size_t /*size*/) noexcept {
 	deallocate(ptr, 0);
 }
 
-void operator delete[](void* ptr, std::size_t /*size*/) noexcept {
+SPC_REPLACEMENT void operator delete[](void* ptr, std::size_t /*size*/) noexcept {
 	deallocate(ptr, 0);
 }
 
-void operator delete(void* ptr, std::align_val_t alignment) noexcept {
+SPC_REPLACEMENT void operator delete(void* ptr, std::align_val_t alignment) noexcept {
 	deallocate(ptr, to_size(alignment));
 }
 
-void operator delete[](void* ptr, std::align_val_t alignment) noexcept {
+SPC_REPLACEMENT void operator delete[](void* ptr, std::align_val_t alignment) noexcept {
 	deallocate(ptr, to_size(alignment));
 }
 
-void operator delete(void* ptr, std::size_t /*size*/, std::align_val_t alignment) noexcept {
+SPC_REPLACEMENT void operator delete(void* ptr, std::size_t /*size*/,
+									 std::align_val_t alignment) noexcept {
 	deallocate(ptr, to_size(alignment));
 }
 
-void operator delete[](void* ptr, std::size_t /*size*/, std::align_val_t alignment) noexcept {
+SPC_REPLACEMENT void operator delete[](void* ptr, std::size_t /*size*/,
+									   std::align_val_t alignment) noexcept {
 	deallocate(ptr, to_size(alignment));
 }
 
-void operator delete(void* ptr, const std::nothrow_t& /*tag*/) noexcept {
+SPC_REPLACEMENT void operator delete(void* ptr, const std::nothrow_t& /*tag*/) noexcept {
 	deallocate(ptr, 0);
 }
 
-void operator delete[](void* ptr, const std::nothrow_t& /*tag*/) noexcept {
+SPC_REPLACEMENT void operator delete[](void* ptr, const std::nothrow_t& /*tag*/) noexcept {
 	deallocate(ptr, 0);
 }
 
-void operator delete(void* ptr, std::align_val_t alignment,
-					 const std::nothrow_t& /*tag*/) noexcept {
+SPC_REPLACEMENT void operator delete(void* ptr, std::align_val_t alignment,
+									 const std::nothrow_t& /*tag*/) noexcept {
 	deallocate(ptr, to_size(alignment));
 }
 
-void operator delete[](void* ptr, std::align_val_t alignment,
-					   const std::nothrow_t& /*tag*/) noexcept {
+SPC_REPLACEMENT void operator delete[](void* ptr, std::align_val_t alignment,
+									   const std::nothrow_t& /*tag*/) noexcept {
 	deallocate(ptr, to_size(alignment));
 }