From 9e8c7a4e4fedc7229dcfa39f1f748bf679995c97 Mon Sep 17 00:00:00 2001 From: Jordan Peck Date: Sat, 10 Oct 2026 23:00:32 +0100 Subject: [PATCH 1/4] Rework test harness - Every register size (128, 256, 512 and 1024 bit) is tested, each built as its own pair of dispatch libraries to keep compile memory down - Every compiled feature set up to the CPU's is compared with the SCALAR reference, plain and relaxed. Non relaxed results must match bit for bit, relaxed ones within per test ULP/absolute tolerances - Plain C++ oracles check the SCALAR reference itself, and relaxed SCALAR (std:: maths) checks FastSIMD's own accuracy - Inputs start with every combination of special values (+-0, denormals, infinities, NaN payloads, rounding ties, int limits, mask patterns), then seeded random values from per test input specs - Known failures are explicit, scoped by tier, feature set range and optionally input values, and reported when no longer seen - Exit code, ctest registration and a Release ctest step in CI - Options: --seed, --count, --filter, --width, --max-feature-set, --stats --- .github/workflows/ci.yml | 3 + CMakeLists.txt | 1 + tests/CMakeLists.txt | 59 +- tests/test.cpp | 1379 ++++++++++++++++++++++++++++++++++---- tests/test.h | 220 +++++- tests/test.inl | 802 +++++++++++++++------- 6 files changed, 2067 insertions(+), 397 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bf53b89..d1999b2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -64,3 +64,6 @@ jobs: - name: 'CMake Build Release' run: cmake --build ${{ github.workspace }}/release --config Release --parallel 4 + + - name: 'Test Release' + run: ctest --test-dir ${{ github.workspace }}/release -C Release --output-on-failure diff --git a/CMakeLists.txt b/CMakeLists.txt index 0d44afc..c6be961 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -44,6 +44,7 @@ if(FASTSIMD_DISPATCH_CLASS) endif() if(FASTSIMD_TESTS) + enable_testing() add_subdirectory(tests) endif() diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 15bc605..e3abb49 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1,13 +1,60 @@ -fastsimd_create_dispatch_library(simd_test SOURCES "test.inl" FEATURE_SETS SCALAR SSE2 SSE41 AVX2 AVX512 NEON AARCH64 WASM) -fastsimd_create_dispatch_library(simd_test_relaxed RELAXED SOURCES "test.inl" FEATURE_SETS SCALAR SSE2 SSE41 AVX2 AVX512 NEON AARCH64 WASM) +set(FASTSIMD_TEST_FEATURE_SETS SCALAR SSE2 SSE41 AVX2 AVX512 NEON AARCH64 WASM) -add_executable(test "test.cpp") -target_link_libraries(test PRIVATE FastSIMD simd_test simd_test_relaxed) +# Each register size is its own pair of dispatch libraries, test.inl instantiates FASTSIMD_TEST_REGISTER_BYTES +# All sizes in one translation unit needs ~4.5GB per unit in a GCC Debug build, too much for 4 parallel jobs on a CI runner +# Keep in sync with kTestRegisterBytes in test.h +set(simd_test_libraries "") +foreach(register_bytes 16 32 64 128) + fastsimd_create_dispatch_library(simd_test_${register_bytes} SOURCES "test.inl" FEATURE_SETS ${FASTSIMD_TEST_FEATURE_SETS}) + fastsimd_create_dispatch_library(simd_test_relaxed_${register_bytes} RELAXED SOURCES "test.inl" FEATURE_SETS ${FASTSIMD_TEST_FEATURE_SETS}) + + foreach(simd_test_library simd_test_${register_bytes} simd_test_relaxed_${register_bytes}) + target_compile_definitions(${simd_test_library} PRIVATE FASTSIMD_TEST_REGISTER_BYTES=${register_bytes}) + + # Every test is too many sections for an unoptimised COFF object file + if(MSVC) + target_compile_options(${simd_test_library} PRIVATE /bigobj) + elseif(WIN32 AND CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + # GNU as (MinGW) has the same limit, clang's integrated assembler switches to big objects itself + target_compile_options(${simd_test_library} PRIVATE -Wa,-mbig-obj) + endif() + + list(APPEND simd_test_libraries ${simd_test_library}) + endforeach() +endforeach() + +# "test" is a reserved target name when CTest is enabled +add_executable(FastSIMD_tests "test.cpp") +target_link_libraries(FastSIMD_tests PRIVATE FastSIMD ${simd_test_libraries}) + +# Random inputs must be the same on every platform for a seed, stop arm64 compilers fusing the range maths into FMA +if(MSVC AND CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + target_compile_options(FastSIMD_tests PRIVATE /clang:-ffp-contract=off) +elseif(NOT CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") + target_compile_options(FastSIMD_tests PRIVATE -ffp-contract=off) +endif() + +# Fewer random inputs keeps ctest (and CI) quick, run FastSIMD_tests directly for a full run +set(FASTSIMD_CTEST_INPUT_COUNT 65536 CACHE STRING "FastSIMD random input values per test when run through ctest") +add_test(NAME FastSIMD_tests COMMAND FastSIMD_tests --count ${FASTSIMD_CTEST_INPUT_COUNT}) if(CMAKE_SYSTEM_NAME STREQUAL "Emscripten") - set(CMAKE_EXECUTABLE_SUFFIX ".html") - target_link_options(test PRIVATE -sALLOW_MEMORY_GROWTH=1 -sSINGLE_FILE) + # Runs under node, which the Emscripten toolchain sets as the ctest emulator + # EXIT_RUNTIME makes main's return value the exit code + # Debug info makes the single file output over 1GB, larger than node can load, so it's left out when linking + target_link_options(FastSIMD_tests PRIVATE -sALLOW_MEMORY_GROWTH=1 -sSINGLE_FILE -sEXIT_RUNTIME=1 $<$:-g0>) + + # The toolchain uses whichever node is first on PATH, emsdk's own node supports the relaxed SIMD the relaxed tests use + if(DEFINED ENV{EMSDK_NODE}) + set_target_properties(FastSIMD_tests PROPERTIES CROSSCOMPILING_EMULATOR "$ENV{EMSDK_NODE}") + endif() + + # Single page to run the tests in a browser + add_executable(FastSIMD_tests_web "test.cpp") + target_link_libraries(FastSIMD_tests_web PRIVATE FastSIMD ${simd_test_libraries}) + set_target_properties(FastSIMD_tests_web PROPERTIES SUFFIX ".html") + target_link_options(FastSIMD_tests_web PRIVATE -sALLOW_MEMORY_GROWTH=1 -sSINGLE_FILE $<$:-g0>) endif() # Enable Warnings diff --git a/tests/test.cpp b/tests/test.cpp index fa07caf..723746c 100644 --- a/tests/test.cpp +++ b/tests/test.cpp @@ -1,56 +1,721 @@ #include "test.h" +#include +#include +#include +#include +#include +#include #include -#include #include -#include -#include +#include +#include #include #include +#include +#include #include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +// Every register size has a test library pair, all created for the same feature sets +using TestFeatureSets = FastSIMD::simd_test_16::CompiledFeatureSets; -#include +static_assert( std::is_same_v && + std::is_same_v && + std::is_same_v && + std::is_same_v && + std::is_same_v && + std::is_same_v && + std::is_same_v, + "Every simd_test library must compile the same feature sets" ); -static constexpr size_t TestCount = 4096 * 4096; +struct TestOptions +{ + uint64_t seed = 24301; + uint64_t count = 1 << 22; // Random input values per test argument, after the special value combinations + std::string filter; // Only run tests with names containing this + size_t registerBytes = 0; // Only run this register size, 0 for all + FastSIMD::FeatureSet maxFeatureSet = FastSIMD::FeatureSet::Max; + bool stats = false; // Print the largest relaxed differences +}; + +struct TestSummary +{ + size_t passed = 0; + std::vector knownFailures; + std::vector knownNotSeen; + std::vector failed; +}; + +// Inputs are generated and tested in chunks to keep memory use flat +static constexpr size_t kChunkSize = 1 << 14; +static constexpr int kMaxFailuresPerTier = 3; -static int * rndInts; -static float* rndFloats; +static_assert( kChunkSize % kMaxTestLanes == 0, "Calls must not straddle input chunks" ); + +static uint64_t SplitMix64( uint64_t& state ) +{ + uint64_t z = ( state += 0x9E3779B97F4A7C15ull ); + z = ( z ^ ( z >> 30 ) ) * 0xBF58476D1CE4E5B9ull; + z = ( z ^ ( z >> 27 ) ) * 0x94D049BB133111EBull; + return z ^ ( z >> 31 ); +} -static float GenFiniteFloat( std::mt19937& gen ) +// FNV-1a, stable across compilers unlike std::hash +static uint64_t HashName( std::string_view name ) { - union + uint64_t hash = 0xCBF29CE484222325ull; + for( char c : name ) { - float f; - int32_t i; - } u; + hash = ( hash ^ static_cast( c ) ) * 0x100000001B3ull; + } + return hash; +} + +template +struct FloatTraits; + +template<> +struct FloatTraits +{ + using Bits = uint32_t; + static constexpr int kSignificandBits = 24; +}; + +template<> +struct FloatTraits +{ + using Bits = uint64_t; + static constexpr int kSignificandBits = 53; +}; + +template +static typename FloatTraits::Bits ToBits( T f ) +{ + typename FloatTraits::Bits bits; + std::memcpy( &bits, &f, sizeof( f ) ); + return bits; +} + +template +static T FromBits( typename FloatTraits::Bits bits ) +{ + T f; + std::memcpy( &f, &bits, sizeof( f ) ); + return f; +} - do +static float BitsToFloat( uint32_t bits ) { return FromBits( bits ); } + +static std::string FeatureSetName( FastSIMD::FeatureSet featureSet ) +{ + return FastSIMD::GetFeatureSetString( featureSet ); +} + +template +struct FeatureSetsOf; + +template +struct FeatureSetsOf> +{ + static std::vector Get() { return { SETS... }; } +}; + +static std::vector CompiledFeatureSets() +{ + return FeatureSetsOf::Get(); +} + +// Checks a test's input specs can be generated, mistakes here would silently test the wrong domain +static void ValidateInputs( std::string_view testName, const TestData& test ) +{ + auto fail = [&]( size_t argIdx, const char* problem ) { - u.i = static_cast( gen() ); + std::string message = "Invalid input spec for \"" + std::string( testName ) + "\" argument " + std::to_string( argIdx ) + ": " + problem; + std::cerr << message << std::endl; + throw std::runtime_error( message ); + }; + + constexpr double floatMax = std::numeric_limits::max(); - } while( !std::isfinite( u.f ) ); + for( size_t argIdx = 0; argIdx < kMaxTestArgs; argIdx++ ) + { + const InputSpec& spec = test.inputs[argIdx]; + bool isDefault = spec.kind == InputSpec::Kind::Any && !spec.inf && !spec.nan && spec.significandBits == 0 && spec.specials.empty(); - return u.f; + if( argIdx >= test.argCount ) + { + if( !isDefault ) + { + fail( argIdx, "test has fewer arguments than input specs" ); + } + continue; + } + + if( test.argTypes[argIdx] == TestArgType::Mask && !isDefault ) + { + fail( argIdx, "mask arguments are always generated from lane patterns, they take no input spec" ); + } + TestArgType type = test.argTypes[argIdx]; + int maxBits = type == TestArgType::Float ? 24 : type == TestArgType::Double ? 53 : 0; + if( spec.significandBits < 0 || spec.significandBits > maxBits ) + { + fail( argIdx, maxBits ? "significand bits must be in [1, 24] for floats, [1, 53] for doubles" : "significand bits only apply to floats and doubles" ); + } + for( double special : spec.specials ) + { + if( std::isnan( special ) ) + { + fail( argIdx, "special values can't be NaN, use AnyNonFinite" ); + } + } + if( spec.kind == InputSpec::Kind::Any ) + { + continue; + } + if( spec.inf || spec.nan ) + { + fail( argIdx, "inf and NaN are only supported by Any specs" ); + } + if( !std::isfinite( spec.lo ) || !std::isfinite( spec.hi ) || spec.lo > spec.hi ) + { + fail( argIdx, "range bounds must be finite with lo <= hi" ); + } + if( spec.kind == InputSpec::Kind::Magnitude && spec.lo <= 0 ) + { + fail( argIdx, "magnitude lo must be > 0" ); + } + if( test.argTypes[argIdx] == TestArgType::Float && ( std::abs( spec.lo ) > floatMax || std::abs( spec.hi ) > floatMax ) ) + { + fail( argIdx, "range bounds must be within float range" ); + } + if( test.argTypes[argIdx] == TestArgType::Int && + ( std::ceil( spec.lo ) > std::floor( spec.hi ) || std::ceil( spec.lo ) < -2147483648.0 || std::floor( spec.hi ) > 2147483647.0 ) ) + { + fail( argIdx, "int range must contain an integer and be within int32 range" ); + } + } } -static void GenerateRandomValues() +// Generates the input values for every argument of a test +// Only uses std::mt19937_64 and arithmetic without FMA contraction (see tests/CMakeLists.txt) so a seed produces the same values on any compiler +class InputGenerator { - std::cout << "Generating random values..." << std::endl; +public: + InputGenerator( uint64_t seed, std::string_view testName, const TestData& test, size_t lanes ) : + mArgCount( test.argCount ), mLanes( lanes ) + { + mSeed = seed ^ HashName( testName ); + mCombinations = mArgCount ? 1 : 0; - rndInts = new int[TestCount + 1024]; - rndFloats = new float[TestCount + 1024]; + for( size_t argIdx = 0; argIdx < mArgCount; argIdx++ ) + { + Arg& arg = mArgs[argIdx]; + arg.type = test.argTypes[argIdx]; + arg.spec = test.inputs[argIdx]; - std::random_device rd; //Will be used to obtain a seed for the random number engine - std::mt19937 gen( rd() ); //Standard mersenne_twister_engine seeded with rd() + switch( arg.type ) + { + case TestArgType::Float: + arg.floatSpecials = FloatSpecials( arg.spec ); + arg.specialCount = arg.floatSpecials.size(); + arg.floats.resize( kChunkSize ); + break; + case TestArgType::Double: + arg.doubleSpecials = FloatSpecials( arg.spec ); + arg.specialCount = arg.doubleSpecials.size(); + arg.doubles.resize( kChunkSize ); + break; + case TestArgType::Int: + arg.intSpecials = IntSpecials( arg.spec ); + arg.specialCount = arg.intSpecials.size(); + arg.ints.resize( kChunkSize ); + break; + case TestArgType::Mask: + arg.specialCount = MaskPatternCount(); + arg.ints.resize( kChunkSize ); + arg.maskFloats.resize( kChunkSize ); + arg.maskDoubles.resize( kChunkSize ); + break; + } + + mCombinations *= arg.specialCount; + } + } - for ( std::size_t i = 0; i < TestCount; i++ ) + // Inputs below this index are combinations of special values + size_t SpecialCount() const { - rndInts[i] = (int)gen(); - rndFloats[i] = GenFiniteFloat( gen ); + return mCombinations * mLanes; } - //std::sort( rndFloats, rndFloats + TestCount + 1024, std::less() ); -} + void Generate( size_t chunkStart, TestInputs& inputs ) + { + inputs = {}; + + for( size_t argIdx = 0; argIdx < mArgCount; argIdx++ ) + { + Arg& arg = mArgs[argIdx]; + uint64_t state = mSeed + chunkStart * kMaxTestArgs + argIdx; + std::mt19937_64 rng( SplitMix64( state ) ); + + for( size_t i = 0; i < kChunkSize; i++ ) + { + size_t element = chunkStart + i; + bool special = element < SpecialCount(); + + switch( arg.type ) + { + case TestArgType::Float: + arg.floats[i] = special ? arg.floatSpecials[SpecialDigit( element, argIdx )] : RandomFloat( arg.spec, rng ); + break; + case TestArgType::Double: + arg.doubles[i] = special ? arg.doubleSpecials[SpecialDigit( element, argIdx )] : RandomFloat( arg.spec, rng ); + break; + case TestArgType::Int: + arg.ints[i] = special ? arg.intSpecials[SpecialDigit( element, argIdx )] : RandomInt( arg.spec, rng ); + break; + case TestArgType::Mask: + arg.ints[i] = special ? MaskPatternLane( SpecialDigit( element, argIdx ), element % mLanes ) : static_cast( rng() & 1 ); + arg.maskFloats[i] = static_cast( arg.ints[i] ); + arg.maskDoubles[i] = static_cast( arg.ints[i] ); + break; + } + } + + switch( arg.type ) + { + case TestArgType::Float: + inputs.floats[argIdx] = arg.floats.data(); + break; + case TestArgType::Double: + inputs.doubles[argIdx] = arg.doubles.data(); + break; + case TestArgType::Int: + inputs.ints[argIdx] = arg.ints.data(); + break; + case TestArgType::Mask: + inputs.masks[argIdx] = arg.ints.data(); + inputs.maskFloats[argIdx] = arg.maskFloats.data(); + inputs.maskDoubles[argIdx] = arg.maskDoubles.data(); + break; + } + } + } + +private: + struct Arg + { + TestArgType type = TestArgType::Float; + InputSpec spec; + std::vector floatSpecials; + std::vector doubleSpecials; + std::vector intSpecials; + size_t specialCount = 0; + std::vector floats; + std::vector doubles; + std::vector ints; // Ints, or 0/1 mask lanes + std::vector maskFloats; + std::vector maskDoubles; + }; + + // Value arguments use combination (call + lane), putting every combination in every lane and in lane 0 for scalar arguments + // Mask arguments use combination (call) so a whole mask pattern lands in one call + size_t SpecialDigit( size_t element, size_t argIdx ) const + { + size_t call = element / mLanes; + size_t combination = mArgs[argIdx].type == TestArgType::Mask ? call : call + element % mLanes; + combination %= mCombinations; + + for( size_t i = 0; i < argIdx; i++ ) + { + combination /= mArgs[i].specialCount; + } + return combination % mArgs[argIdx].specialCount; + } + + // All clear, all set, each single lane set, each single lane clear, alternating, and each half set + size_t MaskPatternCount() const + { + return 2 + mLanes * 2 + 4; + } + + int32_t MaskPatternLane( size_t pattern, size_t lane ) const + { + if( pattern < 2 ) + { + return static_cast( pattern ); + } + pattern -= 2; + if( pattern < mLanes ) + { + return lane == pattern; + } + pattern -= mLanes; + if( pattern < mLanes ) + { + return lane != pattern; + } + pattern -= mLanes; + + switch( pattern ) + { + case 0: return ( lane & 1 ) == 0; + case 1: return ( lane & 1 ) == 1; + case 2: return lane < mLanes / 2; + default: return lane >= mLanes / 2; + } + } + + template + static T RangeLo( const InputSpec& spec ) { return static_cast( spec.lo ); } + template + static T RangeHi( const InputSpec& spec ) { return static_cast( spec.hi ); } + + // Int bounds are the integers inside [lo, hi], ValidateInputs checks there is at least one + static int64_t IntLo( const InputSpec& spec ) { return static_cast( std::ceil( spec.lo ) ); } + static int64_t IntHi( const InputSpec& spec ) { return static_cast( std::floor( spec.hi ) ); } + + // Drops the low significand bits, finite values only + template + static T Truncate( T value, int significandBits ) + { + constexpr int kBits = FloatTraits::kSignificandBits; + if( significandBits == 0 || significandBits >= kBits || !std::isfinite( value ) ) + { + return value; + } + using Bits = typename FloatTraits::Bits; + Bits lowBits = ( Bits( 1 ) << ( kBits - significandBits ) ) - 1; + return FromBits( ToBits( value ) & ~lowBits ); + } + + template + static bool InFloatDomain( const InputSpec& spec, T value ) + { + switch( spec.kind ) + { + case InputSpec::Kind::Any: + return ( spec.nan || !std::isnan( value ) ) && ( spec.inf || !std::isinf( value ) ); + case InputSpec::Kind::Range: + return value >= RangeLo( spec ) && value <= RangeHi( spec ); + case InputSpec::Kind::Magnitude: + return std::abs( value ) >= RangeLo( spec ) && std::abs( value ) <= RangeHi( spec ); + } + return false; + } + + // Values of every magnitude that commonly break SIMD code + static std::vector AnySpecials( const InputSpec& spec, float ) + { + constexpr float inf = std::numeric_limits::infinity(); + constexpr float floatMin = std::numeric_limits::min(); + constexpr float denormMin = std::numeric_limits::denorm_min(); + const float denormMax = BitsToFloat( 0x007FFFFF ); + + std::vector candidates = { + 0.0f, -0.0f, 1.0f, -1.0f, 0.5f, -0.5f, 1.5f, -1.5f, 2.5f, -2.5f, + 0.49999997f, -0.49999997f, 0.99999994f, -0.99999994f, 1.0000001f, -1.0000001f, + floatMin, -floatMin, denormMin, -denormMin, denormMax, -denormMax, + std::numeric_limits::max(), -std::numeric_limits::max(), + 8388607.5f, -8388607.5f, 8388608.0f, -8388608.0f, 16777216.0f, -16777216.0f, + 2147483520.0f, -2147483520.0f, 2147483648.0f, -2147483648.0f, + 3.14159274f, -3.14159274f, 1.e-10f, 1.e+10f, + }; + if( spec.inf ) + { + candidates.insert( candidates.end(), { inf, -inf } ); + } + if( spec.nan ) + { + // Quiet, negative, signalling and payload NaNs + for( uint32_t bits : { 0x7FC00000u, 0xFFC00000u, 0x7F800001u, 0xFF800001u, 0x7FFFFFFFu, 0xFFC00001u } ) + { + candidates.push_back( BitsToFloat( bits ) ); + } + } + return candidates; + } + + static std::vector AnySpecials( const InputSpec& spec, double ) + { + constexpr double inf = std::numeric_limits::infinity(); + constexpr double doubleMin = std::numeric_limits::min(); + constexpr double denormMin = std::numeric_limits::denorm_min(); + const double denormMax = FromBits( 0x000FFFFFFFFFFFFFull ); + constexpr double floatMax = std::numeric_limits::max(); + constexpr double floatMin = std::numeric_limits::min(); + constexpr double floatDenormMin = std::numeric_limits::denorm_min(); + + std::vector candidates = { + 0.0, -0.0, 1.0, -1.0, 0.5, -0.5, 1.5, -1.5, 2.5, -2.5, + std::nextafter( 0.5, 0.0 ), -std::nextafter( 0.5, 0.0 ), std::nextafter( 1.0, 0.0 ), -std::nextafter( 1.0, 0.0 ), + std::nextafter( 1.0, 2.0 ), -std::nextafter( 1.0, 2.0 ), + doubleMin, -doubleMin, denormMin, -denormMin, denormMax, -denormMax, + std::numeric_limits::max(), -std::numeric_limits::max(), + // Integer precision limits, and the int32 convert range + 4503599627370495.5, -4503599627370495.5, 4503599627370496.0, -4503599627370496.0, 9007199254740992.0, -9007199254740992.0, + 2147483647.0, -2147483648.0, 2147483647.5, -2147483648.5, 2147483648.0, -2147483649.0, + // Narrowing to float: overflow, float denormals, and rounding ties + floatMax, -floatMax, 3.4028235677973366e+38, -3.4028235677973366e+38, floatMin, -floatMin, floatDenormMin, -floatDenormMin, + floatDenormMin / 2, 1.0 + 0x1p-24, -( 1.0 + 0x1p-24 ), 1.0 + 0x1.8p-23, 1.0 + 0x1p-23 + 0x1p-25, + 3.141592653589793, -3.141592653589793, 1.e-10, 1.e+10, 1.e-300, 1.e+300, + }; + if( spec.inf ) + { + candidates.insert( candidates.end(), { inf, -inf } ); + } + if( spec.nan ) + { + // Quiet, negative, signalling and payload NaNs, including payloads narrowing to float drops + for( uint64_t bits : { 0x7FF8000000000000ull, 0xFFF8000000000000ull, 0x7FF0000000000001ull, 0xFFF0000000000001ull, + 0x7FFFFFFFFFFFFFFFull, 0xFFF8000000000001ull, 0x7FF4000000000000ull } ) + { + candidates.push_back( FromBits( bits ) ); + } + } + return candidates; + } + + template + static std::vector FloatSpecials( const InputSpec& spec ) + { + constexpr T inf = std::numeric_limits::infinity(); + constexpr T typeMin = std::numeric_limits::min(); + constexpr T denormMin = std::numeric_limits::denorm_min(); + const T denormMax = std::nextafter( typeMin, T( 0 ) ); + const T belowHalf = std::nextafter( T( 0.5 ), T( 0 ) ); + const T belowOne = std::nextafter( T( 1 ), T( 0 ) ); + const T aboveOne = std::nextafter( T( 1 ), T( 2 ) ); + std::vector candidates; + + switch( spec.kind ) + { + case InputSpec::Kind::Any: + candidates = AnySpecials( spec, T() ); + break; + + case InputSpec::Kind::Range: + candidates = { RangeLo( spec ), RangeHi( spec ), std::nextafter( RangeLo( spec ), inf ), std::nextafter( RangeHi( spec ), -inf ) }; + for( T v : { T( 0 ), T( 0.5 ), T( 1 ), T( 1.5 ), T( 2.5 ), belowHalf, belowOne, aboveOne, typeMin, denormMin, denormMax, T( 1.e-10 ), T( 1.e-30 ) } ) + { + candidates.insert( candidates.end(), { v, -v } ); + } + break; + + case InputSpec::Kind::Magnitude: + for( T v : { RangeLo( spec ), RangeHi( spec ), std::nextafter( RangeLo( spec ), inf ), std::nextafter( RangeHi( spec ), -inf ), + T( 0.5 ), T( 1 ), T( 2 ), belowOne, aboveOne, typeMin, denormMin, denormMax } ) + { + candidates.insert( candidates.end(), { v, -v } ); + } + break; + } + + for( double special : spec.specials ) + { + T v = static_cast( special ); + candidates.insert( candidates.end(), { v, std::nextafter( v, inf ), std::nextafter( v, -inf ) } ); + } + + std::vector specials; + for( T candidate : candidates ) + { + T v = Truncate( candidate, spec.significandBits ); + + // Compare bits to keep both 0 and -0, and each NaN + bool duplicate = std::any_of( specials.begin(), specials.end(), [v]( T s ) { return ToBits( s ) == ToBits( v ); } ); + + if( InFloatDomain( spec, v ) && !duplicate ) + { + specials.push_back( v ); + } + } + if( specials.empty() ) + { + throw std::runtime_error( "Input spec has no special values in its domain" ); + } + return specials; + } + + static std::vector IntSpecials( const InputSpec& spec ) + { + constexpr int32_t intMax = std::numeric_limits::max(); + constexpr int32_t intMin = std::numeric_limits::min(); + std::vector candidates; + + switch( spec.kind ) + { + case InputSpec::Kind::Any: + candidates = { + 0, 1, -1, 2, -2, 31, 32, 33, -31, -32, + intMax, intMin, intMax - 1, intMin + 1, + 0x7F800000, static_cast( 0xFF800000 ), 0x7FC00000, 0x7F800001, // inf, -inf, NaN, signalling NaN float bits + 0x3F800000, 0x00800000, 0x0000FFFF, 0x00010000, 0x55555555, static_cast( 0xAAAAAAAA ), + }; + break; + case InputSpec::Kind::Range: + candidates = { IntLo( spec ), IntHi( spec ), IntLo( spec ) + 1, IntHi( spec ) - 1, 0, 1, -1 }; + break; + case InputSpec::Kind::Magnitude: + for( int64_t v : { IntLo( spec ), IntHi( spec ), IntLo( spec ) + 1, IntHi( spec ) - 1, int64_t( 1 ) } ) + { + candidates.insert( candidates.end(), { v, -v } ); + } + break; + } + + for( double special : spec.specials ) + { + candidates.push_back( static_cast( std::round( std::min( std::max( special, -2147483648.0 ), 2147483647.0 ) ) ) ); + } + + std::vector specials; + for( int64_t v : candidates ) + { + bool inDomain = true; + switch( spec.kind ) + { + case InputSpec::Kind::Any: + inDomain = v >= intMin && v <= intMax; + break; + case InputSpec::Kind::Range: + inDomain = v >= IntLo( spec ) && v <= IntHi( spec ); + break; + case InputSpec::Kind::Magnitude: + inDomain = std::abs( v ) >= IntLo( spec ) && std::abs( v ) <= IntHi( spec ); + break; + } + + if( inDomain && std::find( specials.begin(), specials.end(), v ) == specials.end() ) + { + specials.push_back( static_cast( v ) ); + } + } + return specials; + } + + // Uniform [0, 1) + static double RandomUnit( std::mt19937_64& rng ) + { + return static_cast( rng() >> 11 ) * ( 1.0 / 9007199254740992.0 ); + } + + // Log scaled magnitude within [lo, hi], lo > 0: a uniform power of 2 binade, then uniform within its part of [lo, hi] + template + static T RandomMagnitude( double lo, double hi, std::mt19937_64& rng ) + { + int loExp, hiExp; + std::frexp( lo, &loExp ); + std::frexp( hi, &hiExp ); + + int exponent = loExp + static_cast( rng() % static_cast( hiExp - loExp + 1 ) ); + double binadeLo = std::max( lo, std::ldexp( 0.5, exponent ) ); + double binadeHi = std::min( hi, std::ldexp( 1.0, exponent ) ); + T value = static_cast( binadeLo + ( binadeHi - binadeLo ) * RandomUnit( rng ) ); + + return std::min( std::max( value, static_cast( lo ) ), static_cast( hi ) ); + } + + template + static T RandomFloat( const InputSpec& spec, std::mt19937_64& rng ) + { + constexpr double denormMin = std::numeric_limits::denorm_min(); + + for( ;; ) + { + T value = 0; + + switch( spec.kind ) + { + case InputSpec::Kind::Any: + value = FromBits( static_cast::Bits>( rng() ) ); + break; + + case InputSpec::Kind::Range: + if( rng() & 1 ) + { + value = static_cast( spec.lo + ( spec.hi - spec.lo ) * RandomUnit( rng ) ); + } + else + { + // Log scaled half covers small magnitudes, including denormals, that uniform values almost never hit + bool positive = spec.lo >= 0 || ( spec.hi > 0 && ( rng() & 1 ) ); + double maxMag = positive ? spec.hi : -spec.lo; + double minMag = positive ? std::max( spec.lo, 0.0 ) : std::max( -spec.hi, 0.0 ); + value = RandomMagnitude( std::max( minMag, denormMin ), std::max( maxMag, denormMin ), rng ); + value = positive ? value : -value; + } + break; + + case InputSpec::Kind::Magnitude: + value = RandomMagnitude( spec.lo, spec.hi, rng ); + value = ( rng() & 1 ) ? -value : value; + break; + } + + value = Truncate( value, spec.significandBits ); + + if( InFloatDomain( spec, value ) ) + { + return value; + } + } + } + + static int32_t RandomInt( const InputSpec& spec, std::mt19937_64& rng ) + { + switch( spec.kind ) + { + case InputSpec::Kind::Any: + return static_cast( rng() ); + + case InputSpec::Kind::Range: + return static_cast( IntLo( spec ) + static_cast( rng() % static_cast( IntHi( spec ) - IntLo( spec ) + 1 ) ) ); + + case InputSpec::Kind::Magnitude: + { + int64_t value = IntLo( spec ) + static_cast( rng() % static_cast( IntHi( spec ) - IntLo( spec ) + 1 ) ); + return static_cast( ( rng() & 1 ) ? -value : value ); + } + } + return 0; + } + + size_t mArgCount; + size_t mLanes; + uint64_t mSeed; + size_t mCombinations; + Arg mArgs[kMaxTestArgs]; +}; + +enum class LaneResult +{ + Equal, ZeroSign, Different +}; + +struct TierResult +{ + TestTier tier; + std::string name; + const TestData* data = nullptr; + size_t compared = 0; // Calls compared + int failures = 0; // Calls with differences not covered by a known failure + int knownCalls = 0; // Calls with only known differences + double maxUlps = 0; // Largest relaxed difference only the ULP tolerance covered, for --stats + double maxAbs = 0; // Largest relaxed difference only the absolute tolerance covered +}; template struct TestRunner @@ -60,7 +725,7 @@ struct TestRunner template struct TestOrganiser { - static TestCollection GetCollections() + static TestCollection GetCollections( FastSIMD::FeatureSet ) { return {}; } @@ -68,38 +733,52 @@ struct TestRunner template struct TestOrganiser> - { - static TestCollection GetCollections() + { + template + static TestCollection RegisterTests() { - TestCollection collections = TestOrganiser>::GetCollections(); + std::unique_ptr> testSimd( FastSIMD::NewDispatchClass>( HEAD ) ); - if( HEAD <= FastSIMD::DetectCpuMaxFeatureSet() ) + if( !testSimd ) { - std::cout << "Generating Tests: " << FastSIMD::GetFeatureSetString( HEAD ) << std::endl; - { - std::unique_ptr> testSimd( FastSIMD::NewDispatchClass>( HEAD ) ); + throw std::runtime_error( "No test class created for " + FeatureSetName( HEAD ) ); + } - TestCollection simdCollection = testSimd->RegisterTests(); + TestCollection collection = testSimd->RegisterTests(); - collections.insert( collections.begin(), simdCollection.begin(), simdCollection.end() ); - } + for( auto& test : collection ) + { + if( test.second.featureSet != HEAD || test.second.relaxed != RELAXED ) { - std::unique_ptr> testSimd( FastSIMD::NewDispatchClass>( HEAD ) ); + throw std::runtime_error( "Test class for " + FeatureSetName( HEAD ) + " registered \"" + std::string( test.first ) + "\" for " + FeatureSetName( test.second.featureSet ) ); + } + } + return collection; + } - TestCollection simdCollection = testSimd->RegisterTests(); + static TestCollection GetCollections( FastSIMD::FeatureSet maxFeatureSet ) + { + TestCollection collections = TestOrganiser>::GetCollections( maxFeatureSet ); - collections.insert( collections.begin(), simdCollection.begin(), simdCollection.end() ); - } + if( HEAD <= FastSIMD::DetectCpuMaxFeatureSet() && HEAD <= maxFeatureSet ) + { + std::cout << "Generating Tests: " << FeatureSetName( HEAD ) << std::endl; + + TestCollection relaxed = RegisterTests(); + collections.insert( collections.begin(), relaxed.begin(), relaxed.end() ); + + TestCollection nonRelaxed = RegisterTests(); + collections.insert( collections.begin(), nonRelaxed.begin(), nonRelaxed.end() ); } return collections; } - static TestSet GetSet() + static TestSet GetSet( FastSIMD::FeatureSet maxFeatureSet ) { - TestCollection collections = GetCollections(); + TestCollection collections = GetCollections( maxFeatureSet ); TestSet set; - + for( auto& collection : collections ) { std::string_view& testName = collection.first; @@ -108,9 +787,9 @@ struct TestRunner if( find == set.end() ) { - if( collection.second.featureSet != FastSIMD::FeatureSet::SCALAR ) + if( collection.second.featureSet != FastSIMD::FeatureSet::SCALAR || collection.second.relaxed ) { - throw std::runtime_error( "Scalar must be base test set" ); + throw std::runtime_error( "\"" + std::string( testName ) + "\" has no SCALAR non relaxed reference" ); } find = set.emplace( set.end(), testName, std::vector{} ); @@ -123,186 +802,594 @@ struct TestRunner } }; - // get value of least significant bit - static float DeltaUnit( float x ) + // Value of the least significant bit, relaxed differences are counted in these + // Zero and denormals use the smallest normal rather than the denormal spacing, so relaxed results may flush denormals + template + static T DeltaUnit( T x ) { - union - { - float f; - uint32_t i; - } u; x = std::fabs( x ); if( !std::isfinite( x ) ) { - return 1.f; + return T( 1 ); } - if( x == 0.f || !std::isnormal( x ) ) + if( x == T( 0 ) || !std::isnormal( x ) ) { - u.i = 0x00800000; // smallest positive normal number - return u.f; + return std::numeric_limits::min(); // smallest positive normal number } - float x1 = x; - u.f = x; - u.i++; - return u.f - x1; + + T above = std::nextafter( x, std::numeric_limits::infinity() ); + + // The largest finite value has nothing finite above it, use the gap below instead + return std::isfinite( above ) ? above - x : x - std::nextafter( x, T( 0 ) ); } template - static bool CompareTyped( std::string_view testName, FastSIMD::FeatureSet featureSet, float accuracy, size_t outputCount, void* scalarResults, void* simdResults ) + static LaneResult CompareLane( const TestData& test, bool relaxed, T expected, T actual, TierResult& result ) { - bool success = true; + if constexpr( std::is_floating_point_v ) + { + bool bitsEqual = ToBits( expected ) == ToBits( actual ); + bool bothNaN = std::isnan( expected ) && std::isnan( actual ); - const T* typedScalar = reinterpret_cast( scalarResults ); - const T* typedSimd = reinterpret_cast( simdResults ); + if( bitsEqual || ( bothNaN && !test.bitExact ) ) + { + return LaneResult::Equal; + } - for( size_t idx = 0; idx < outputCount; idx++ ) - { - if( typedScalar[idx] != typedSimd[idx] ) + // Relaxed results are compared by value, so +0 and -0 are equal + if( relaxed && !test.bitExact ) { - float relativeDif = 0; + if( expected == actual ) + { + return LaneResult::Equal; + } + + auto [allowedUlps, allowedAbs] = test.RelaxedTolerance(); + double absoluteDif = std::abs( static_cast( expected ) - static_cast( actual ) ); + double relativeDif = absoluteDif / DeltaUnit( expected ); - if constexpr( std::is_floating_point_v ) + if( relativeDif <= allowedUlps || absoluteDif <= allowedAbs ) { - if( std::isnan( typedScalar[idx] ) && std::isnan( typedSimd[idx] ) ) + // Track what each tolerance had to cover on its own + if( std::isfinite( absoluteDif ) ) { - continue; + if( absoluteDif > allowedAbs ) + { + result.maxUlps = std::max( result.maxUlps, relativeDif ); + } + if( relativeDif > allowedUlps ) + { + result.maxAbs = std::max( result.maxAbs, absoluteDif ); + } } + return LaneResult::Equal; + } + return LaneResult::Different; + } - relativeDif = std::abs( typedScalar[idx] - typedSimd[idx] ) / DeltaUnit( typedScalar[idx] ); + return expected == 0 && actual == 0 ? LaneResult::ZeroSign : LaneResult::Different; + } + else + { + return expected == actual ? LaneResult::Equal : LaneResult::Different; + } + } - if( relativeDif <= accuracy ) - { - continue; - } + template + static std::string Describe( T expected, T actual ) + { + std::ostringstream sstr; + sstr << std::setprecision( std::is_floating_point_v ? std::numeric_limits::max_digits10 : 16 ) << std::boolalpha; + sstr << "Expected \"" << expected << "\" Actual \"" << actual << "\""; + + if constexpr( std::is_floating_point_v ) + { + T absoluteDif = std::abs( expected - actual ); + sstr << " Diff \"" << absoluteDif << "\""; + + if( std::isfinite( absoluteDif ) && absoluteDif != 0 ) + { + sstr << " (" << absoluteDif / DeltaUnit( expected ) << " ULP)"; + } + if( ToBits( expected ) != ToBits( actual ) && ( absoluteDif == 0 || std::isnan( absoluteDif ) ) ) + { + sstr << std::hex << " (bits 0x" << ToBits( expected ) << " vs 0x" << ToBits( actual ) << ")" << std::dec; + } + } + else if constexpr( std::is_same_v ) + { + sstr << " Diff \"" << static_cast( expected ) - static_cast( actual ) << "\""; + } + else if constexpr( std::is_same_v ) + { + sstr << std::hex << " (bits 0x" << static_cast( expected ) << " vs 0x" << static_cast( actual ) << ")" << std::dec; + } + return sstr.str(); + } + + // Compares one call's outputs, returns true if the tier failed. Known differences mark their entry in knownSeen and set sawKnown + template + static bool CompareCall( std::string_view testName, const TestData& reference, TierResult& result, size_t outputCount, const TestInputs& inputs, size_t inIdx, + const void* expectedResults, const void* actualResults, std::vector>& knownSeen, std::string& details, bool& sawKnown ) + { + const T* expected = reinterpret_cast( expectedResults ); + const T* actual = reinterpret_cast( actualResults ); + const TestData& test = result.data ? *result.data : reference; + const auto& knownFailures = reference.knownFailures; + bool failed = false; + + for( size_t idx = 0; idx < outputCount; idx++ ) + { + LaneResult lane = CompareLane( test, result.tier.relaxed, expected[idx], actual[idx], result ); + + if( lane == LaneResult::Equal ) + { + continue; + } + + // Lane inputs for known failure conditions + TestData::LaneArgs laneArgs = {}; + for( size_t argIdx = 0; argIdx < reference.argCount; argIdx++ ) + { + if( reference.argTypes[argIdx] == TestArgType::Float ) + { + laneArgs.floats[argIdx] = inputs.floats[argIdx][inIdx + idx]; } - if( success ) + else if( reference.argTypes[argIdx] == TestArgType::Double ) { - std::cerr << std::setprecision( 16 ) << std::boolalpha; - std::cerr << "--- " << FastSIMD::GetFeatureSetString( featureSet ) << " FAILED ---" << std::endl; + laneArgs.doubles[argIdx] = inputs.doubles[argIdx][inIdx + idx]; } - std::cerr << "idx " << idx << ": " << testName - << " Expected \"" << typedScalar[idx] - << "\" Actual \"" << typedSimd[idx] - << "\" Diff \"" << std::abs( typedScalar[idx] - typedSimd[idx] ) << "\""; + } - if( relativeDif != 0.0f ) + bool known = false; + for( size_t knownIdx = 0; knownIdx < knownFailures.size(); knownIdx++ ) + { + const TestData::KnownFailureInfo& knownFailure = knownFailures[knownIdx]; + + if( knownFailure.Covers( result.tier ) && ( !knownFailure.zeroSignOnly || lane == LaneResult::ZeroSign ) && + ( !knownFailure.inputCondition || knownFailure.inputCondition( laneArgs ) ) ) { - std::cerr << " (" << relativeDif << ")"; + known = true; + sawKnown = true; + + if( std::find( knownSeen[knownIdx].begin(), knownSeen[knownIdx].end(), result.name ) == knownSeen[knownIdx].end() ) + { + knownSeen[knownIdx].push_back( result.name ); + } + break; } - std::cerr << std::endl; - success = false; } - } - return success; + if( !known ) + { + details += "idx " + std::to_string( idx ) + ": " + std::string( testName ) + ( result.tier.relaxed ? " RELAXED " : " " ) + Describe( expected[idx], actual[idx] ) + "\n"; + failed = true; + } + } + return failed; } - static bool CompareOutputs( std::string_view testName, FastSIMD::FeatureSet featureSet, TestData::ReturnType returnType, float accuracy, size_t outputCount, void* scalarResults, void* simdResults ) + static bool CompareOutputs( std::string_view testName, const TestData& reference, TierResult& result, size_t outputCount, const TestInputs& inputs, size_t inIdx, + const void* expected, const void* actual, std::vector>& knownSeen, std::string& details, bool& sawKnown ) { - switch( returnType ) + switch( reference.returnType ) { case TestData::ReturnType::boolean: - return CompareTyped( testName, featureSet, accuracy, outputCount, scalarResults, simdResults ); + return CompareCall( testName, reference, result, outputCount, inputs, inIdx, expected, actual, knownSeen, details, sawKnown ); case TestData::ReturnType::f32: - return CompareTyped( testName, featureSet, accuracy, outputCount, scalarResults, simdResults ); + return CompareCall( testName, reference, result, outputCount, inputs, inIdx, expected, actual, knownSeen, details, sawKnown ); + + case TestData::ReturnType::f64: + return CompareCall( testName, reference, result, outputCount, inputs, inIdx, expected, actual, knownSeen, details, sawKnown ); case TestData::ReturnType::i32: - return CompareTyped( testName, featureSet, accuracy, outputCount, scalarResults, simdResults ); + return CompareCall( testName, reference, result, outputCount, inputs, inIdx, expected, actual, knownSeen, details, sawKnown ); + + case TestData::ReturnType::i64: + return CompareCall( testName, reference, result, outputCount, inputs, inIdx, expected, actual, knownSeen, details, sawKnown ); } - return false; + return true; + } + + static std::string TierName( const TestTier& tier ) + { + if( tier.oracle ) + { + return "SCALAR vs ORACLE"; + } + return FeatureSetName( tier.featureSet ) + ( tier.relaxed ? " RELAXED" : "" ); + } + + static void CheckTestsMatch( std::string_view testName, const std::vector& tests ) + { + const TestData& reference = tests[0]; + + for( size_t testIdx = 1; testIdx < tests.size(); testIdx++ ) + { + const TestData& test = tests[testIdx]; + + if( test.featureSet == FastSIMD::FeatureSet::SCALAR && !test.relaxed ) + { + throw std::runtime_error( "Multiple tests with same name: " + std::string( testName ) ); + } + if( test.returnType != reference.returnType || test.lanes != reference.lanes || test.argCount != reference.argCount || test.knownFailures.size() != reference.knownFailures.size() || + !std::equal( test.argTypes, test.argTypes + kMaxTestArgs, reference.argTypes ) ) + { + throw std::runtime_error( "Tests do not match: " + std::string( testName ) + " (" + TierName( { test.featureSet, test.relaxed, false } ) + ")" ); + } + } } - static void DoTest( std::string_view testName, std::vector& tests ) + static void DoTest( std::string_view testName, std::vector& tests, const TestOptions& options, TestSummary& summary ) { std::cout << "Testing: " << testName << std::endl; - char* scalarResults = new char[RegisterBytes]; - char* simdResults = new char[RegisterBytes]; - int failed = 0; + const TestData& reference = tests[0]; + CheckTestsMatch( testName, tests ); + ValidateInputs( testName, reference ); + + InputGenerator generator( options.seed, testName, reference, reference.lanes ); + TestInputs inputs; - for( size_t idx = 0; idx < TestCount; idx += RegisterBytes / sizeof( int ) ) + // No argument tests always produce the same output + size_t inputCount = reference.argCount ? generator.SpecialCount() + static_cast( options.count ) : reference.lanes; + + std::vector tiers; + if( reference.oracleFunc ) { - for( size_t testIdx = 0; testIdx < tests.size(); testIdx++ ) - { - TestData& test = tests[testIdx]; - - char* resultsOut = testIdx ? simdResults : scalarResults; - std::memset( resultsOut, (int)testIdx, RegisterBytes ); - - size_t outputCount = test.testFunc( resultsOut, idx, rndInts, rndFloats ); + TierResult oracle; + oracle.tier = { FastSIMD::FeatureSet::SCALAR, false, true }; + oracle.name = TierName( oracle.tier ); + tiers.push_back( oracle ); + } + for( size_t testIdx = 1; testIdx < tests.size(); testIdx++ ) + { + TierResult tier; + tier.tier = { tests[testIdx].featureSet, tests[testIdx].relaxed, false }; + tier.name = TierName( tier.tier ); + tier.data = &tests[testIdx]; + tiers.push_back( tier ); + } + + // A tier stops being checked after kMaxFailuresPerTier failing calls, or known failing calls if a known failure covers every difference + auto stopped = [&]( const TierResult& tier ) + { + if( tier.failures >= kMaxFailuresPerTier ) + { + return true; + } + bool coversAll = std::any_of( reference.knownFailures.begin(), reference.knownFailures.end(), + [&]( const TestData::KnownFailureInfo& known ) { return known.CoversAnyDifference() && known.Covers( tier.tier ); } ); + return coversAll && tier.knownCalls >= kMaxFailuresPerTier; + }; + + std::vector> knownSeen( reference.knownFailures.size() ); + std::vector referenceResults( kMaxTestResultBytes ), testResults( kMaxTestResultBytes ); + + for( size_t chunkStart = 0; chunkStart < inputCount && !tiers.empty(); chunkStart += kChunkSize ) + { + if( std::all_of( tiers.begin(), tiers.end(), stopped ) ) + { + break; + } + + generator.Generate( chunkStart, inputs ); + size_t chunkCount = std::min( kChunkSize, inputCount - chunkStart ); + + for( size_t idx = 0; idx < chunkCount; idx += reference.lanes ) + { + std::memset( referenceResults.data(), 0xCD, kMaxTestResultBytes ); + size_t referenceCount = reference.testFunc( referenceResults.data(), idx, inputs ); - if( testIdx ) + for( TierResult& tier : tiers ) { - if( test.returnType != tests[0].returnType ) + if( stopped( tier ) ) { - std::cerr << "Tests do not match: " << testName; - throw std::exception(); + continue; + } + + std::memset( testResults.data(), 0xAB, kMaxTestResultBytes ); + size_t outputCount = tier.data ? tier.data->testFunc( testResults.data(), idx, inputs ) : reference.oracleFunc( testResults.data(), idx, inputs ); + + std::string details; + bool failed; + bool sawKnown = false; + + if( outputCount != referenceCount ) + { + details = "Output count " + std::to_string( outputCount ) + ", reference " + std::to_string( referenceCount ) + "\n"; + failed = true; + } + else if( tier.data ) + { + failed = CompareOutputs( testName, reference, tier, outputCount, inputs, idx, referenceResults.data(), testResults.data(), knownSeen, details, sawKnown ); } - if( test.featureSet == FastSIMD::FeatureSet::SCALAR && !test.relaxed ) + else { - std::cerr << "Multiple tests with same name: " << testName; - throw std::exception(); + // The oracle is the expected value, the SCALAR reference is checked against it + failed = CompareOutputs( testName, reference, tier, outputCount, inputs, idx, testResults.data(), referenceResults.data(), knownSeen, details, sawKnown ); } - std::string testNameRelaxed = testName.data(); - float accuracy = 0; + tier.compared++; - if( test.relaxed ) + if( failed ) { - testNameRelaxed += " RELAXED"; - accuracy = test.relaxedAccuracy; + if( !tier.failures ) + { + std::cerr << std::setprecision( 16 ) << "--- " << RegisterBytes * 8 << "bit " << testName << " " << tier.name << " FAILED ---" << std::endl; + std::cerr << details; + std::cerr << "Inputs (index " << chunkStart + idx << "): " << reference.inputsFunc( idx, inputs ) << std::endl; + } + tier.failures++; } - - if( !CompareOutputs( testNameRelaxed, test.featureSet, test.returnType, accuracy, outputCount, scalarResults, simdResults ) ) + else if( sawKnown ) { - std::cerr << "Inputs: " << tests[0].inputsFunc( idx, rndInts, rndFloats ) << std::endl; - failed++; + tier.knownCalls++; } } + } + } - if( failed >= 3 ) - { - break; - } + std::string width = std::to_string( RegisterBytes * 8 ) + "bit "; + std::vector failedTiers; + bool comparedAny = false; + + for( const TierResult& tier : tiers ) + { + comparedAny |= tier.compared > 0; + + if( tier.failures ) + { + failedTiers.push_back( tier.name ); + } + if( options.stats && tier.data && tier.tier.relaxed && ( tier.maxUlps > 0 || tier.maxAbs > 0 ) ) + { + auto [allowedUlps, allowedAbs] = tier.data->RelaxedTolerance(); + std::cout << " Stats " << width << testName << " " << tier.name << ": max " << tier.maxUlps << " / " << allowedUlps << " ULP, " + << tier.maxAbs << " / " << allowedAbs << " abs" << std::endl; } + } - if( failed >= 3 ) + auto join = []( const std::vector& names ) + { + std::string joined; + for( size_t i = 0; i < names.size(); i++ ) { - std::cerr << "Skipping test, fail limit reached" << std::endl; - break; + joined += ( i ? ", " : "" ) + names[i]; } + return joined; + }; + + for( size_t knownIdx = 0; knownIdx < reference.knownFailures.size(); knownIdx++ ) + { + const TestData::KnownFailureInfo& knownFailure = reference.knownFailures[knownIdx]; + std::string reason = knownFailure.reason ? knownFailure.reason : "(no reason given)"; + + if( !knownSeen[knownIdx].empty() ) + { + summary.knownFailures.push_back( width + std::string( testName ) + " (" + join( knownSeen[knownIdx] ) + "): " + reason ); + } + else if( std::any_of( tiers.begin(), tiers.end(), [&]( const TierResult& tier ) { return tier.compared && knownFailure.Covers( tier.tier ); } ) ) + { + summary.knownNotSeen.push_back( width + std::string( testName ) + ": " + reason ); + } + } + + if( !comparedAny ) + { + std::cout << " Warning: no tiers to compare with the reference" << std::endl; + } + + if( !failedTiers.empty() ) + { + summary.failed.push_back( width + std::string( testName ) + " (" + join( failedTiers ) + ")" ); + } + else + { + summary.passed++; } - - delete[] scalarResults; - delete[] simdResults; } - static void Run() + static void Run( const TestOptions& options, TestSummary& summary ) { + if( options.registerBytes && options.registerBytes != RegisterBytes ) + { + return; + } + std::cout << "Starting Tests - Register Size: " << RegisterBytes * 8 << " (" << RegisterBytes << "b)" << std::endl; - TestSet testSet = TestOrganiser::GetSet(); + TestSet testSet = TestOrganiser::GetSet( options.maxFeatureSet ); for( auto& test : testSet ) { - //if( test.first.find( "sqrt" ) != std::string_view::npos ) + if( test.first.find( options.filter ) != std::string_view::npos ) { - DoTest( test.first, test.second ); + DoTest( test.first, test.second, options, summary ); } } - - std::cout << "Testing Complete!" << std::endl; } }; -int main() +template +static void RunTests( const TestOptions& options, TestSummary& summary, std::index_sequence ) +{ + ( TestRunner::Run( options, summary ), ... ); +} + +static void PrintUsage( const char* exe ) +{ + std::cout << "Usage: " << exe << " [options]\n" + << " --seed Seed for random inputs (default " << TestOptions{}.seed << ")\n" + << " --count Random input values per test argument (default " << TestOptions{}.count << ")\n" + << " --filter Only run tests with names containing text\n" + << " --width Only run one register size:"; + for( size_t bytes : kTestRegisterBytes ) + { + std::cout << " " << bytes; + } + std::cout << "\n --max-feature-set Only run feature sets up to this one, to match a less capable CPU:"; + for( FastSIMD::FeatureSet featureSet : CompiledFeatureSets() ) + { + std::cout << " " << FeatureSetName( featureSet ); + } + std::cout << "\n --stats Print the largest relaxed differences within tolerance" << std::endl; +} + +static bool ParseUnsigned( const char* text, uint64_t& value ) +{ + if( !text || !std::isdigit( static_cast( text[0] ) ) ) + { + return false; + } + + char* end = nullptr; + errno = 0; + unsigned long long parsed = std::strtoull( text, &end, 0 ); + + if( errno || *end ) + { + return false; + } + value = parsed; + return true; +} + +static std::string NormaliseName( std::string_view name ) +{ + std::string normalised; + for( char c : name ) + { + if( c != '.' && c != '_' ) + { + normalised += static_cast( std::toupper( static_cast( c ) ) ); + } + } + return normalised; +} + +static bool ParseArgs( int argc, char** argv, TestOptions& options ) +{ + for( int i = 1; i < argc; i++ ) + { + std::string_view arg = argv[i]; + + if( arg == "--stats" ) + { + options.stats = true; + continue; + } + + const char* value = i + 1 < argc ? argv[++i] : nullptr; + uint64_t number = 0; + bool valid = value != nullptr; + + if( arg == "--seed" && valid ) + { + if( std::string_view( value ) == "random" ) + { + options.seed = std::random_device()(); + } + else + { + valid = ParseUnsigned( value, options.seed ); + } + } + else if( arg == "--count" && valid ) + { + // Limited so the input index can't overflow size_t on 32bit + valid = ParseUnsigned( value, number ) && number <= ( 1ull << 30 ); + options.count = number; + } + else if( arg == "--filter" && valid ) + { + options.filter = value; + } + else if( arg == "--width" && valid ) + { + valid = ParseUnsigned( value, number ) && + std::find( std::begin( kTestRegisterBytes ), std::end( kTestRegisterBytes ), number ) != std::end( kTestRegisterBytes ); + options.registerBytes = static_cast( number ); + } + else if( arg == "--max-feature-set" && valid ) + { + std::vector featureSets = CompiledFeatureSets(); + auto find = std::find_if( featureSets.begin(), featureSets.end(), [value]( FastSIMD::FeatureSet featureSet ) { return NormaliseName( FeatureSetName( featureSet ) ) == NormaliseName( value ); } ); + + valid = find != featureSets.end(); + options.maxFeatureSet = valid ? *find : options.maxFeatureSet; + } + else + { + valid = false; + } + + if( !valid ) + { + std::cout << "Invalid argument: " << arg << ( value ? std::string( " " ) + value : std::string() ) << "\n" << std::endl; + PrintUsage( argv[0] ); + return false; + } + } + return true; +} + +int main( int argc, char** argv ) { - GenerateRandomValues(); + TestOptions options; + + if( !ParseArgs( argc, argv, options ) ) + { + return 2; + } + + std::cout << "Seed: " << options.seed << ", Random inputs per test: " << options.count << std::endl; + std::cout << "CPU max feature set: " << FeatureSetName( FastSIMD::DetectCpuMaxFeatureSet() ) << ", Compiled:"; + for( FastSIMD::FeatureSet featureSet : CompiledFeatureSets() ) + { + std::cout << " " << FeatureSetName( featureSet ); + } + std::cout << std::endl; + if( options.maxFeatureSet != FastSIMD::FeatureSet::Max ) + { + std::cout << "Max feature set: " << FeatureSetName( options.maxFeatureSet ) << std::endl; + } + + TestSummary summary; - TestRunner::Run(); + try + { + RunTests( options, summary, std::make_index_sequence{} ); + } + catch( const std::exception& e ) + { + std::cerr << std::endl << "Test setup error: " << e.what() << std::endl; + return 1; + } + + std::cout << std::endl << "Passed: " << summary.passed << ", Known failures: " << summary.knownFailures.size() << ", Failed: " << summary.failed.size() << std::endl; + + for( const std::string& knownFailure : summary.knownFailures ) + { + std::cout << " Known failure: " << knownFailure << std::endl; + } + for( const std::string& notSeen : summary.knownNotSeen ) + { + std::cout << " Known failure not seen: " << notSeen << std::endl; + } + for( const std::string& failed : summary.failed ) + { + std::cout << " FAILED: " << failed << std::endl; + } + + if( !summary.passed && summary.failed.empty() ) + { + std::cout << "No tests matched filter \"" << options.filter << "\"" << std::endl; + return 1; + } - return 0; + return summary.failed.empty() ? 0 : 1; } diff --git a/tests/test.h b/tests/test.h index 24c3a19..7642ce9 100644 --- a/tests/test.h +++ b/tests/test.h @@ -1,35 +1,237 @@ #pragma once #include #include -#include +#include #include +#include #include +#include #include #include -constexpr size_t kTestBytes = 512 / 8; +// Register sizes the tests are compiled for, each is a pair of dispatch libraries in tests/CMakeLists.txt +constexpr size_t kTestRegisterBytes[] = { 16, 32, 64, 128 }; +// Lanes per call is set by a test's widest lane type, 32bit lanes give the most +constexpr size_t kMaxTestLanes = 128 / sizeof( int32_t ); +// Converts keep the lane count, so 32bit lanes widened to 64bit need twice the register size +constexpr size_t kMaxTestResultBytes = kMaxTestLanes * sizeof( double ); -struct TestResult +constexpr size_t kMaxTestArgs = 3; + +// Input values for each test argument, indexed by element +struct TestInputs +{ + const int32_t* ints[kMaxTestArgs]; + const float* floats[kMaxTestArgs]; + const double* doubles[kMaxTestArgs]; + const int32_t* masks[kMaxTestArgs]; // 0 or 1 per lane + const float* maskFloats[kMaxTestArgs]; // Same lanes as masks, 0.0f or 1.0f + const double* maskDoubles[kMaxTestArgs]; // Same lanes as masks, 0.0 or 1.0 +}; + +enum class TestArgType : uint8_t { - uint8_t returnCount; + Int, Float, Double, Mask }; -using TestFunction = size_t ( void*, size_t, int32_t*, float* ); -using InputsFunction = std::string ( size_t, int32_t*, float* ); +// How input values are generated for a test argument +// Every test first runs all combinations of the special values for its inputs, then random values +struct InputSpec +{ + enum class Kind + { + Any, // Values of every magnitude + Range, // Values within [lo, hi], half uniform, half log scaled + Magnitude, // Values with magnitude within [lo, hi] log scaled, either sign, lo > 0 + }; + + Kind kind = Kind::Any; + double lo = 0; + double hi = 0; + bool inf = false; // Any only, include +-inf + bool nan = false; // Any only, include NaNs + int significandBits = 0; // Floats are truncated to this many significand bits, 0 for all of them + std::vector specials; // Extra special values, only used when inside the spec's domain + + static InputSpec Any() { return {}; } + static InputSpec AnyInf() { InputSpec spec; spec.inf = true; return spec; } + static InputSpec AnyNonFinite() { InputSpec spec; spec.inf = spec.nan = true; return spec; } + static InputSpec Range( double lo, double hi ) { InputSpec spec; spec.kind = Kind::Range; spec.lo = lo; spec.hi = hi; return spec; } + static InputSpec Magnitude( double lo, double hi ) { InputSpec spec; spec.kind = Kind::Magnitude; spec.lo = lo; spec.hi = hi; return spec; } + + InputSpec Bits( int bits ) const { InputSpec spec = *this; spec.significandBits = bits; return spec; } + + // Adds each value and the floats (or doubles) either side of it + InputSpec Specials( std::initializer_list values ) const + { + InputSpec spec = *this; + spec.specials.insert( spec.specials.end(), values.begin(), values.end() ); + return spec; + } +}; + +using TestFunction = size_t ( void* outResult, size_t inIdx, const TestInputs& inputs ); +using InputsFunction = std::string ( size_t inIdx, const TestInputs& inputs ); + +// Identifies what a result is checked against: a tier against the SCALAR reference, or the reference against the oracle +struct TestTier +{ + FastSIMD::FeatureSet featureSet; + bool relaxed; + bool oracle; +}; struct TestData { enum class ReturnType { - boolean, f32, i32 + boolean, f32, f64, i32, i64 // i64 holds 64bit mask lanes + }; + + enum class Tiers + { + All, // Every tier compared with the reference, not the oracle + Relaxed, + NonRelaxed, + ScalarRelaxed, // Relaxed scalar uses std:: maths functions, so differs where FastSIMD is less accurate + Oracle, // The SCALAR reference against the test's oracle + }; + + // A lane's input values, arguments of other types are 0 + struct LaneArgs + { + float floats[kMaxTestArgs]; + double doubles[kMaxTestArgs]; }; - + + using LaneInputCondition = bool ( * )( const LaneArgs& args ); + + struct KnownFailureInfo + { + const char* reason; + bool zeroSignOnly; // Only results differing in the sign of zero are known, any other difference still fails + LaneInputCondition inputCondition; // Only lanes with matching inputs are known, null for every lane + Tiers tiers; + FastSIMD::FeatureSet minFeatureSet; + FastSIMD::FeatureSet maxFeatureSet; + + bool Covers( const TestTier& tier ) const + { + bool tiersMatch = false; + + switch( tiers ) + { + case Tiers::All: + tiersMatch = !tier.oracle; + break; + case Tiers::Relaxed: + tiersMatch = !tier.oracle && tier.relaxed; + break; + case Tiers::NonRelaxed: + tiersMatch = !tier.oracle && !tier.relaxed; + break; + case Tiers::ScalarRelaxed: + tiersMatch = !tier.oracle && tier.relaxed && tier.featureSet == FastSIMD::FeatureSet::SCALAR; + break; + case Tiers::Oracle: + return tier.oracle; + } + + return tiersMatch && tier.featureSet >= minFeatureSet && tier.featureSet <= maxFeatureSet; + } + + // Any difference on a covered tier is known, so comparing more of its calls can't fail + bool CoversAnyDifference() const + { + return !zeroSignOnly && !inputCondition; + } + }; + FastSIMD::FeatureSet featureSet; bool relaxed; ReturnType returnType; - float relaxedAccuracy = 0; + size_t lanes = 0; // Elements per call, the widest register in the test sets it + size_t argCount = 0; + TestArgType argTypes[kMaxTestArgs] = {}; + InputSpec inputs[kMaxTestArgs]; + double relaxedAccuracy = 0; // Allowed error in ULPs for relaxed feature sets + double relaxedAbsAccuracy = 0; // Allowed absolute error for relaxed feature sets, for results near 0 + double scalarRelaxedAccuracy = -1; // As above for relaxed SCALAR, negative uses the relaxed accuracy + double scalarRelaxedAbsAccuracy = 0; + bool bitExact = false; // Float results must match bitwise on every tier, including NaN payloads + std::vector knownFailures; std::function testFunc; + std::function oracleFunc; // Optional plain C++ version of the test, the SCALAR reference must match it exactly std::function inputsFunc; + + template + TestData& Inputs( SPECs... specs ) + { + static_assert( sizeof...( SPECs ) <= kMaxTestArgs ); + size_t idx = 0; + ( ( inputs[idx++] = specs ), ... ); + return *this; + } + + TestData& RelaxedAccuracy( double ulps, double absolute = 0 ) + { + relaxedAccuracy = ulps; + relaxedAbsAccuracy = absolute; + return *this; + } + + // Relaxed SCALAR uses std:: math for some functions, so this is the accuracy of the FS implementation itself + // Relaxed SIMD tolerances cover estimate instructions, which would hide FS accuracy regressions + TestData& ScalarRelaxedAccuracy( double ulps, double absolute = 0 ) + { + scalarRelaxedAccuracy = ulps; + scalarRelaxedAbsAccuracy = absolute; + return *this; + } + + // Allowed { ULPs, absolute } error when this tier is relaxed + std::pair RelaxedTolerance() const + { + if( featureSet == FastSIMD::FeatureSet::SCALAR && scalarRelaxedAccuracy >= 0 ) + { + return { scalarRelaxedAccuracy, scalarRelaxedAbsAccuracy }; + } + return { relaxedAccuracy, relaxedAbsAccuracy }; + } + + TestData& BitExact() + { + bitExact = true; + return *this; + } + + // Differences on tiers in scope are reported as known failures and don't fail the run + // A reason is required on every tier's TestData, scope it with tiers and the feature set range + TestData& KnownFailure( const char* reason, Tiers tiers = Tiers::All, + FastSIMD::FeatureSet minFeatureSet = FastSIMD::FeatureSet::Invalid, + FastSIMD::FeatureSet maxFeatureSet = FastSIMD::FeatureSet::Max ) + { + knownFailures.push_back( { reason, false, nullptr, tiers, minFeatureSet, maxFeatureSet } ); + return *this; + } + + // As KnownFailure, but only results that differ in the sign of zero are known + TestData& KnownZeroSign( const char* reason, Tiers tiers = Tiers::All, + FastSIMD::FeatureSet minFeatureSet = FastSIMD::FeatureSet::Invalid, + FastSIMD::FeatureSet maxFeatureSet = FastSIMD::FeatureSet::Max ) + { + knownFailures.push_back( { reason, true, nullptr, tiers, minFeatureSet, maxFeatureSet } ); + return *this; + } + + // As KnownFailure, but only lanes whose inputs match the condition are known, other inputs are still checked + TestData& KnownFailureIf( const char* reason, LaneInputCondition condition, Tiers tiers = Tiers::All, + FastSIMD::FeatureSet minFeatureSet = FastSIMD::FeatureSet::Invalid, + FastSIMD::FeatureSet maxFeatureSet = FastSIMD::FeatureSet::Max ) + { + knownFailures.push_back( { reason, false, condition, tiers, minFeatureSet, maxFeatureSet } ); + return *this; + } }; using TestCollection = std::vector>; diff --git a/tests/test.inl b/tests/test.inl index 716f40f..b66e453 100644 --- a/tests/test.inl +++ b/tests/test.inl @@ -1,11 +1,13 @@ #pragma once -#include - #include "test.h" +#include +#include +#include #include #include #include +#include #include #include @@ -51,16 +53,24 @@ struct GetReturn static constexpr auto Type = TestData::ReturnType::f32; }; +template<> +struct GetReturn +{ + static constexpr auto Type = TestData::ReturnType::f64; +}; + template<> struct GetReturn { static constexpr auto Type = TestData::ReturnType::boolean; }; +// Mask results are stored as integer lanes of the mask's width so every bit of each lane is compared template struct GetReturn> { - static constexpr auto Type = TestData::ReturnType::boolean; + static_assert( N == 32 || N == 64, "Mask results are stored as 32 or 64bit lanes" ); + static constexpr auto Type = N == 32 ? TestData::ReturnType::i32 : TestData::ReturnType::i64; }; template @@ -69,11 +79,12 @@ struct GetReturn> static constexpr auto Type = GetReturn::Type; }; +// Elements per call of a test argument or result, scalars are 1 template -static constexpr size_t GetReturnCount = 1; +static constexpr size_t LaneCount = 1; template -static constexpr size_t GetReturnCount> = N; +static constexpr size_t LaneCount> = N; template @@ -83,18 +94,11 @@ class FastSIMD::DispatchClass, SIMD> : publ using TestReg = FS::Register; - template - static std::tuple GetArg_Helper( Ret ( F::* )( Args... ) const ); - - template - using GetArg = std::tuple_element_t; - - template static std::string AsString( T arg ) { std::stringstream ss; - ss << std::setprecision( 16 ) << std::boolalpha << arg; + ss << std::setprecision( 17 ) << std::boolalpha << arg; return ss.str(); } @@ -103,14 +107,14 @@ class FastSIMD::DispatchClass, SIMD> : publ { T scalarValues[N]; FS::Store( scalarValues, arg ); - + std::ostringstream sstr; sstr << "[" << AsString( scalarValues[0] ); for( size_t idx = 1; idx < N; idx++ ) { - sstr << ", " << AsString( scalarValues[idx] ); - } + sstr << ", " << AsString( scalarValues[idx] ); + } sstr << "]"; return sstr.str(); @@ -120,7 +124,7 @@ class FastSIMD::DispatchClass, SIMD> : publ static std::string AsString( const FS::Register, N, S>& arg ) { FS::BitStorage bitMask = FS::BitMask( arg ); - + std::ostringstream sstr; sstr << "[" << AsString( static_cast( bitMask & 1 ) ); @@ -128,8 +132,8 @@ class FastSIMD::DispatchClass, SIMD> : publ { bitMask >>= 1; - sstr << ", " << AsString( static_cast( bitMask & 1 ) ); - } + sstr << ", " << AsString( static_cast( bitMask & 1 ) ); + } sstr << "]"; return sstr.str(); @@ -146,6 +150,7 @@ class FastSIMD::DispatchClass, SIMD> : publ template static size_t StoreOutput( void* outResult, FS::Register result ) { + static_assert( N * sizeof( T ) <= kMaxTestResultBytes, "Test result is larger than the result buffer" ); FS::Store( reinterpret_cast( outResult ), result ); return N; } @@ -153,63 +158,183 @@ class FastSIMD::DispatchClass, SIMD> : publ template static size_t StoreOutput( void* outResult, FS::Register, N, S> result ) { - bool* boolOut = reinterpret_cast( outResult ); - auto bitMask = FS::BitMask( result ); + static_assert( N * MN / 8 <= kMaxTestResultBytes, "Test result is larger than the result buffer" ); - for( size_t idx = 0; idx < N; idx++ ) + // Store all bits of each lane, BitMask only shows the high bit + if constexpr( MN == 32 ) { - *boolOut++ = bitMask & 1; - bitMask >>= 1; - } + FS::Store( reinterpret_cast( outResult ), FS::Masked( result, FS::Register( -1 ) ) ); + } + else + { + // No 64bit int registers, a double with every bit set keeps all the lane's bits through Masked + static_assert( MN == 64 && MB, "Mask results are stored as 32 or 64bit lanes" ); + FS::Store( reinterpret_cast( outResult ), FS::Masked( result, FS::Register( AllBitsDouble() ) ) ); + } return N; } + static double AllBitsDouble() + { + uint64_t bits = ~uint64_t( 0 ); + double value; + std::memcpy( &value, &bits, sizeof( value ) ); + return value; + } + template - struct GenArg + static const T* InputValues( const TestInputs& inputs, size_t argIdx ) { - template - static T Load( size_t inIdx, size_t argIdx, T* rnd, ARGs... ) + if constexpr( std::is_same_v ) + { + return inputs.floats[argIdx]; + } + else if constexpr( std::is_same_v ) { - return *( rnd + inIdx + argIdx ); + return inputs.doubles[argIdx]; } + else + { + static_assert( std::is_same_v, "No test inputs for type" ); + return inputs.ints[argIdx]; + } + } + + template + struct ArgType + { + static constexpr TestArgType Value = std::is_same_v ? TestArgType::Float : std::is_same_v ? TestArgType::Double : TestArgType::Int; + }; + + template + struct ArgType> : ArgType {}; + + template + struct ArgType, N, S>> + { + static constexpr TestArgType Value = TestArgType::Mask; + }; - template - static T Load( size_t inIdx, size_t argIdx, U, ARGs... args ) + template + struct GenArg + { + static T Load( size_t inIdx, size_t argIdx, const TestInputs& inputs ) { - return Load( inIdx, argIdx, args... ); + return InputValues( inputs, argIdx )[inIdx]; } }; template struct GenArg> { - template - static FS::Register Load( size_t inIdx, size_t argIdx, T* rnd, ARGs... ) + static FS::Register Load( size_t inIdx, size_t argIdx, const TestInputs& inputs ) { - return FS::Load( rnd + inIdx + argIdx * N ); + return FS::Load( InputValues( inputs, argIdx ) + inIdx ); } + }; + + // Each mask lane is generated independently, then made into a mask with a compare of the matching type + template + struct GenArg, N, S>> + { + static_assert( MN == 32 || ( MN == 64 && MB ), "Mask inputs are generated as 32bit lanes, or 64bit from doubles" ); - template - static FS::Register Load( size_t inIdx, size_t argIdx, U, ARGs... args ) + static FS::Register, N, S> Load( size_t inIdx, size_t argIdx, const TestInputs& inputs ) { - return Load( inIdx, argIdx, args... ); + if constexpr( MN == 64 ) + { + return FS::Load( inputs.maskDoubles[argIdx] + inIdx ) != FS::Register( 0.0 ); + } + else if constexpr( MB ) + { + return FS::Load( inputs.maskFloats[argIdx] + inIdx ) != FS::Register( 0.0f ); + } + else + { + return FS::Load( inputs.masks[argIdx] + inIdx ) != FS::Register( 0 ); + } + } + }; + + // Argument values for one lane, as plain C++ types for the oracle + template + struct OracleArg + { + // Scalar arguments are the same for every lane + static T Get( const TestInputs& inputs, size_t argIdx, size_t inIdx, size_t ) + { + return InputValues( inputs, argIdx )[inIdx]; + } + }; + + template + struct OracleArg> + { + static T Get( const TestInputs& inputs, size_t argIdx, size_t inIdx, size_t lane ) + { + return InputValues( inputs, argIdx )[inIdx + lane]; } }; template - struct GenArg, N, S>> + struct OracleArg, N, S>> + { + static bool Get( const TestInputs& inputs, size_t argIdx, size_t inIdx, size_t lane ) + { + return inputs.masks[argIdx][inIdx + lane] != 0; + } + }; + + // Stores an oracle lane in the same format as StoreOutput, float/double lanes can be given as their bits in an unsigned int + // Any unsigned type of the right size, uint64_t is unsigned long on LP64 so 64bit literals have a different type + template + static void StoreOracleLane( void* outResult, size_t lane, V value ) { - template - static FS::Register, N, S> Load( size_t inIdx, size_t argIdx, std::conditional_t* rnd, ARGs... ) + if constexpr( std::is_floating_point_v && std::is_integral_v && std::is_unsigned_v && sizeof( V ) == sizeof( T ) ) + { + std::memcpy( reinterpret_cast( outResult ) + lane, &value, sizeof( T ) ); + } + else { - return FS::Load( rnd + inIdx + argIdx * N ) > FS::Load( rnd + inIdx + argIdx * N + 1 ); + static_assert( std::is_same_v, "Oracle must return the test's element type" ); + reinterpret_cast( outResult )[lane] = value; } + } + + template + struct OracleOutput + { + static constexpr size_t Lanes = 1; - template - static FS::Register, N, S> Load( size_t inIdx, size_t argIdx, U, ARGs... args ) + template + static void Store( void* outResult, size_t lane, V value ) { StoreOracleLane( outResult, lane, value ); } + }; + + template + struct OracleOutput> + { + static constexpr size_t Lanes = N; + + template + static void Store( void* outResult, size_t lane, V value ) { StoreOracleLane( outResult, lane, value ); } + }; + + template + struct OracleOutput, N, S>> + { + static constexpr size_t Lanes = N; + + static void Store( void* outResult, size_t lane, bool value ) { - return Load( inIdx, argIdx, args... ); + if constexpr( MN == 32 ) + { + reinterpret_cast( outResult )[lane] = value ? -1 : 0; + } + else + { + reinterpret_cast( outResult )[lane] = value ? -1 : 0; + } } }; @@ -221,21 +346,52 @@ class FastSIMD::DispatchClass, SIMD> : publ struct TestFunctionFactory> { using ReturnType = std::invoke_result_t; + static constexpr size_t ArgCount = sizeof...( ARGs ); + static constexpr size_t Lanes = std::max( { LaneCount, LaneCount>... } ); + + static_assert( Lanes <= kMaxTestLanes, "Too many lanes per call" ); + + static_assert( ArgCount <= kMaxTestArgs, "Too many test arguments" ); + + static void ArgTypes( TestArgType* argTypes ) + { + [[maybe_unused]] size_t argIdx = 0; + ( ( argTypes[argIdx++] = ArgType>::Value ), ... ); + } static std::function Create( FUNC func ) { - return [func]( void* outResult, size_t inIdx, int32_t* rndInts, float* rndFloats ) + return [func]( void* outResult, size_t inIdx, const TestInputs& inputs ) { size_t argIdx = 0; - auto tuple = std::tuple{ GenArg>::Load( inIdx, argIdx++, rndInts, rndFloats )... }; + auto tuple = std::tuple{ GenArg>::Load( inIdx, argIdx++, inputs )... }; return StoreOutput( outResult, std::apply( func, tuple ) ); }; } + // The oracle is called once per output lane with that lane's argument values + template + static std::function CreateOracle( ORACLE oracle ) + { + return [oracle]( void* outResult, size_t inIdx, const TestInputs& inputs ) -> size_t + { + using Output = OracleOutput; + + for( size_t lane = 0; lane < Output::Lanes; lane++ ) + { + [[maybe_unused]] size_t argIdx = 0; + auto tuple = std::tuple{ OracleArg>::Get( inputs, argIdx++, inIdx, lane )... }; + + Output::Store( outResult, lane, std::apply( oracle, tuple ) ); + } + return Output::Lanes; + }; + } + static std::function GetInputs() { - return []( size_t inIdx, int32_t* rndInts, float* rndFloats ) -> std::string + return []( size_t inIdx, const TestInputs& inputs ) -> std::string { if( !sizeof...( ARGs ) ) { @@ -243,12 +399,12 @@ class FastSIMD::DispatchClass, SIMD> : publ } size_t argIdx = 0; - std::initializer_list tuple = { AsString( GenArg>::Load( inIdx, argIdx++, rndInts, rndFloats ) )... }; + std::initializer_list tuple = { AsString( GenArg>::Load( inIdx, argIdx++, inputs ) )... }; std::ostringstream inputsString; inputsString << "\n"; - std::copy( tuple.begin(), tuple.end() - 1, std::ostream_iterator( inputsString, "\n" ) ); + std::copy( tuple.begin(), tuple.end() - 1, std::ostream_iterator( inputsString, "\n" ) ); inputsString << *(tuple.end() - 1); @@ -265,6 +421,9 @@ class FastSIMD::DispatchClass, SIMD> : publ data.featureSet = SIMD; data.relaxed = FastSIMD::IsRelaxed(); data.returnType = GetReturn::ReturnType>::Type; + data.lanes = TestFunctionFactory::Lanes; + data.argCount = TestFunctionFactory::ArgCount; + TestFunctionFactory::ArgTypes( data.argTypes ); data.testFunc = TestFunctionFactory::Create( func ); data.inputsFunc = TestFunctionFactory::GetInputs(); @@ -272,6 +431,37 @@ class FastSIMD::DispatchClass, SIMD> : publ return testMap.emplace_back( name, data ).second; } + // oracle is a plain C++ version of the test, called per lane with plain values (masks as bool) + template + TestData& RegisterTest( TestCollection& testMap, const char* name, FUNC func, ORACLE oracle ) + { + TestData& data = RegisterTest( testMap, name, func ); + data.oracleFunc = TestFunctionFactory::CreateOracle( oracle ); + return data; + } + + // Oracle helpers + static uint32_t Bits( float f ) + { + uint32_t bits; + std::memcpy( &bits, &f, sizeof( bits ) ); + return bits; + } + + static uint64_t Bits( double d ) + { + uint64_t bits; + std::memcpy( &bits, &d, sizeof( bits ) ); + return bits; + } + + // int32 arithmetic wraps on every tier, done in uint32 to avoid signed overflow + static int32_t Wrap( uint32_t v ) { return static_cast( v ); } + static uint32_t U( int32_t v ) { return static_cast( v ); } + + // x + 0.0f as the masked add/decrement implementations do: -0 becomes +0, everything else is unchanged + static float PlusZero( float a ) { return a == 0.0f ? 0.0f : a; } + TestCollection RegisterTests() override { TestCollection tests; @@ -280,204 +470,344 @@ class FastSIMD::DispatchClass, SIMD> : publ using TestRegf32 = TestReg; using TestRegm32 = typename TestRegf32::MaskType; using TestRegm32i = typename TestRegi32::MaskType; - - - RegisterTest( tests, "m32 bit mask", []( TestRegm32 a ) { return a; } ); + using In = InputSpec; + using Tiers = TestData::Tiers; + using LaneArgs = TestData::LaneArgs; + + constexpr double denormMin = std::numeric_limits::denorm_min(); + constexpr double floatMax = std::numeric_limits::max(); + constexpr double pi = 3.14159265358979323846; + + // Range reduction boundaries for trig + const In trigRange = In::Range( -4096, 4096 ).Specials( { pi / 4, pi / 2, pi, 3 * pi / 2, 2 * pi, 4 * pi, 100 * pi, 1000 * pi, + -pi / 4, -pi / 2, -pi, -3 * pi / 2, -2 * pi, -4 * pi, -100 * pi, -1000 * pi } ); + const In smallTrigRange = In::Range( -2 * pi, 2 * pi ).Specials( { pi / 4, pi / 2, pi, 3 * pi / 2, 2 * pi, + -pi / 4, -pi / 2, -pi, -3 * pi / 2, -2 * pi } ); + + // Known failures shared by several tests + const char* sse2RoundZeroSign = "Below SSE4.1 rounding converts through int32 (cvtps2dq/cvttps2dq), which has no -0, so negative results that round to 0 are +0"; + const char* neonDivide = "NEON (ARMv7) has no divide, division is a reciprocal estimate plus Newton-Raphson steps which isn't correctly rounded"; + // vrecpe overflows to inf for |x| < 2^-128, then vrecps( x, inf ) is -inf, so the Newton-Raphson steps give inf of the wrong sign, or NaN for 0 / x + const char* neonDivideDenormal = "NEON (ARMv7) divide by a denormal: the reciprocal estimate overflows to inf and the Newton-Raphson steps flip its sign, giving -inf for +inf and NaN for 0"; + constexpr auto denormalArg0 = []( const LaneArgs& args ) { return std::fpclassify( args.floats[0] ) == FP_SUBNORMAL; }; + constexpr auto denormalArg1 = []( const LaneArgs& args ) { return std::fpclassify( args.floats[1] ) == FP_SUBNORMAL; }; + // The refined reciprocal can be 1 ULP low, e.g. 1 / 0.99999994 becomes 1, so FLT_MAX / 0.99999994 is FLT_MAX instead of inf + const char* neonDivideOverflow = "NEON (ARMv7) divide near overflow: the Newton-Raphson reciprocal can be 1 ULP low, so quotients that overflow to +-inf can give +-FLT_MAX"; + constexpr auto quotientOverflows = []( const LaneArgs& args ) { return std::isfinite( args.floats[0] ) && args.floats[1] != 0 && std::isinf( args.floats[0] / args.floats[1] ); }; + + + RegisterTest( tests, "m32 bit mask", []( TestRegm32 a ) { return a; }, []( bool a ) { return a; } ); RegisterTest( tests, "m32 any mask", []( TestRegm32 a ) { return FS::AnyMask( a ); } ); - - RegisterTest( tests, "m32 bit and operator", std::bit_and() ); - RegisterTest( tests, "m32 bit or operator", std::bit_or() ); - RegisterTest( tests, "m32 bit xor operator", std::bit_xor() ); - RegisterTest( tests, "m32 bit not operator", std::bit_not() ); - RegisterTest( tests, "m32 bit and not", []( TestRegm32 a, TestRegm32 b ) { return FS::BitwiseAndNot( a, b ); } ); - - RegisterTest( tests, "m32i bit mask", []( TestRegm32 a ) { return a; } ); - RegisterTest( tests, "m32i any mask", []( TestRegm32 a ) { return FS::AnyMask( a ); } ); - - RegisterTest( tests, "m32i bit and operator", std::bit_and() ); - RegisterTest( tests, "m32i bit or operator", std::bit_or() ); - RegisterTest( tests, "m32i bit xor operator", std::bit_xor() ); - RegisterTest( tests, "m32i bit not operator", std::bit_not() ); - RegisterTest( tests, "m32i bit and not", []( TestRegm32i a, TestRegm32i b ) { return FS::BitwiseAndNot( a, b ); } ); - - RegisterTest( tests, "i32 load store", []( TestRegi32 a ) { return a; } ); - RegisterTest( tests, "i32 load scalar", []( int32_t a ) { return TestRegi32( a ); } ); - RegisterTest( tests, "i32 splat", []( int32_t a ) { return FS::Splat( a ); } ); - RegisterTest( tests, "i32 extract 0", []( TestRegi32 a ) { return FS::Extract0( a ); } ); + RegisterTest( tests, "m32 bit mask to int", []( TestRegm32 a ) { return static_cast( FS::BitMask( a ) ); } ); + + RegisterTest( tests, "m32 bit and operator", std::bit_and(), []( bool a, bool b ) { return a && b; } ); + RegisterTest( tests, "m32 bit or operator", std::bit_or(), []( bool a, bool b ) { return a || b; } ); + RegisterTest( tests, "m32 bit xor operator", std::bit_xor(), []( bool a, bool b ) { return a != b; } ); + RegisterTest( tests, "m32 bit not operator", std::bit_not(), []( bool a ) { return !a; } ); + RegisterTest( tests, "m32 bit and not", []( TestRegm32 a, TestRegm32 b ) { return FS::BitwiseAndNot( a, b ); }, []( bool a, bool b ) { return a && !b; } ); + + RegisterTest( tests, "m32i bit mask", []( TestRegm32i a ) { return a; }, []( bool a ) { return a; } ); + RegisterTest( tests, "m32i any mask", []( TestRegm32i a ) { return FS::AnyMask( a ); } ); + RegisterTest( tests, "m32i bit mask to int", []( TestRegm32i a ) { return static_cast( FS::BitMask( a ) ); } ); + + RegisterTest( tests, "m32i bit and operator", std::bit_and(), []( bool a, bool b ) { return a && b; } ); + RegisterTest( tests, "m32i bit or operator", std::bit_or(), []( bool a, bool b ) { return a || b; } ); + RegisterTest( tests, "m32i bit xor operator", std::bit_xor(), []( bool a, bool b ) { return a != b; } ); + RegisterTest( tests, "m32i bit not operator", std::bit_not(), []( bool a ) { return !a; } ); + RegisterTest( tests, "m32i bit and not", []( TestRegm32i a, TestRegm32i b ) { return FS::BitwiseAndNot( a, b ); }, []( bool a, bool b ) { return a && !b; } ); + + RegisterTest( tests, "i32 load store", []( TestRegi32 a ) { return a; }, []( int32_t a ) { return a; } ); + RegisterTest( tests, "i32 load scalar", []( int32_t a ) { return TestRegi32( a ); }, []( int32_t a ) { return a; } ); + RegisterTest( tests, "i32 splat", []( int32_t a ) { return FS::Splat( a ); }, []( int32_t a ) { return a; } ); + RegisterTest( tests, "i32 extract 0", []( TestRegi32 a ) { return FS::Extract0( a ); }, []( int32_t a ) { return a; } ); RegisterTest( tests, "i32 load incremented", []() { return FS::LoadIncremented(); } ); - RegisterTest( tests, "i32 plus operator", std::plus() ); - RegisterTest( tests, "i32 minus operator", std::minus() ); - RegisterTest( tests, "i32 multiply operator", std::multiplies() ); - - RegisterTest( tests, "i32 bit and operator", std::bit_and() ); - RegisterTest( tests, "i32 bit or operator", std::bit_or() ); - RegisterTest( tests, "i32 bit xor operator", std::bit_xor() ); - RegisterTest( tests, "i32 bit not operator", std::bit_not() ); - RegisterTest( tests, "i32 negate operator", std::negate() ); - RegisterTest( tests, "i32 bit and not", []( TestRegi32 a, TestRegi32 b ) { return FS::BitwiseAndNot( a, b ); } ); - - RegisterTest( tests, "i32 increment", []( TestRegi32 a ) { return FS::Increment( a ); } ); - RegisterTest( tests, "i32 decrement", []( TestRegi32 a ) { return FS::Decrement( a ); } ); - RegisterTest( tests, "i32 abs", []( TestRegi32 a ) { return FS::Abs( a ); } ); - RegisterTest( tests, "i32 min", []( TestRegi32 a, TestRegi32 b ) { return FS::Min( a, b ); } ); - RegisterTest( tests, "i32 max", []( TestRegi32 a, TestRegi32 b ) { return FS::Max( a, b ); } ); - - RegisterTest( tests, "i32 bit shift left scalar", []( TestRegi32 a, int b ) { return a << ( b & 31 ); } ); - RegisterTest( tests, "i32 bit shift right scalar", []( TestRegi32 a, int b ) { return a >> ( b & 31 ); } ); - RegisterTest( tests, "i32 bit shift right zero extend scalar", []( TestRegi32 a, int b ) { return FS::BitShiftRightZeroExtend( a, b & 31 ); } ); + RegisterTest( tests, "i32 plus operator", std::plus(), []( int32_t a, int32_t b ) { return Wrap( U( a ) + U( b ) ); } ); + RegisterTest( tests, "i32 minus operator", std::minus(), []( int32_t a, int32_t b ) { return Wrap( U( a ) - U( b ) ); } ); + RegisterTest( tests, "i32 multiply operator", std::multiplies(), []( int32_t a, int32_t b ) { return Wrap( U( a ) * U( b ) ); } ); + + RegisterTest( tests, "i32 bit and operator", std::bit_and(), []( int32_t a, int32_t b ) { return a & b; } ); + RegisterTest( tests, "i32 bit or operator", std::bit_or(), []( int32_t a, int32_t b ) { return a | b; } ); + RegisterTest( tests, "i32 bit xor operator", std::bit_xor(), []( int32_t a, int32_t b ) { return a ^ b; } ); + RegisterTest( tests, "i32 bit not operator", std::bit_not(), []( int32_t a ) { return ~a; } ); + RegisterTest( tests, "i32 negate operator", std::negate(), []( int32_t a ) { return Wrap( 0u - U( a ) ); } ); + // MSVC x86 has miscompiled operators taking an over-aligned lhs by value, the copy was elided and the result written into the caller's a + // Select( m, a OP b, a ) reads a again after each operator, the shape that showed it + RegisterTest( tests, "i32 operators keep lhs", []( TestRegm32 m, TestRegi32 a, TestRegi32 b ) { + return FS::Select( m, a + b, a ) ^ FS::Select( m, a - b, a ) ^ FS::Select( m, a * b, a ) ^ FS::Select( m, a & b, a ) ^ + FS::Select( m, a | b, a ) ^ FS::Select( m, a ^ b, a ) ^ FS::Select( m, a << 3, a ) ^ FS::Select( m, a >> 3, a ) ^ a; + }, []( bool m, int32_t a, int32_t b ) { + int32_t results[] = { Wrap( U( a ) + U( b ) ), Wrap( U( a ) - U( b ) ), Wrap( U( a ) * U( b ) ), a & b, a | b, a ^ b, Wrap( U( a ) << 3 ), a < 0 ? ~( ~a >> 3 ) : a >> 3 }; + int32_t combined = a; + for( int32_t result : results ) + { + combined ^= m ? result : a; + } + return combined; + } ); + RegisterTest( tests, "i32 bit and not", []( TestRegi32 a, TestRegi32 b ) { return FS::BitwiseAndNot( a, b ); }, []( int32_t a, int32_t b ) { return a & ~b; } ); + + RegisterTest( tests, "i32 increment", []( TestRegi32 a ) { return FS::Increment( a ); }, []( int32_t a ) { return Wrap( U( a ) + 1u ); } ); + RegisterTest( tests, "i32 decrement", []( TestRegi32 a ) { return FS::Decrement( a ); }, []( int32_t a ) { return Wrap( U( a ) - 1u ); } ); + RegisterTest( tests, "i32 abs", []( TestRegi32 a ) { return FS::Abs( a ); }, []( int32_t a ) { return a < 0 ? Wrap( 0u - U( a ) ) : a; } ); + RegisterTest( tests, "i32 min", []( TestRegi32 a, TestRegi32 b ) { return FS::Min( a, b ); }, []( int32_t a, int32_t b ) { return std::min( a, b ); } ); + RegisterTest( tests, "i32 max", []( TestRegi32 a, TestRegi32 b ) { return FS::Max( a, b ); }, []( int32_t a, int32_t b ) { return std::max( a, b ); } ); + + RegisterTest( tests, "i32 bit shift left scalar", []( TestRegi32 a, int b ) { return a << b; }, []( int32_t a, int32_t b ) { return Wrap( U( a ) << b ); } ).Inputs( In::Any(), In::Range( 0, 31 ) ); + RegisterTest( tests, "i32 bit shift right scalar", []( TestRegi32 a, int b ) { return a >> b; }, []( int32_t a, int32_t b ) { return a < 0 ? ~( ~a >> b ) : a >> b; } ).Inputs( In::Any(), In::Range( 0, 31 ) ); + RegisterTest( tests, "i32 bit shift right zero extend scalar", []( TestRegi32 a, int b ) { return FS::BitShiftRightZeroExtend( a, b ); }, []( int32_t a, int32_t b ) { return Wrap( U( a ) >> b ); } ).Inputs( In::Any(), In::Range( 0, 31 ) ); //RegisterTest( tests, "i32 bit shift left", []( TestRegi32 a, TestRegi32 b ) { return a << FS::Min( TestRegi32( 31 ), FS::Abs( b ) ); } ); //RegisterTest( tests, "i32 bit shift right", []( TestRegi32 a, TestRegi32 b ) { return a >> FS::Min( TestRegi32( 31 ), FS::Abs( b ) ); } ); - RegisterTest( tests, "i32 equals operator", []( TestRegi32 a, TestRegi32 b ) { return a == b; } ); - RegisterTest( tests, "i32 equals operator alt", []( TestRegi32 a ) { return a == a; } ); - RegisterTest( tests, "i32 not equals operator", []( TestRegi32 a, TestRegi32 b ) { return a != b; } ); - RegisterTest( tests, "i32 not equals operator alt", []( TestRegi32 a ) { return a != a; } ); - RegisterTest( tests, "i32 less than operator", []( TestRegi32 a, TestRegi32 b ) { return a < b; } ); - RegisterTest( tests, "i32 greater than operator", []( TestRegi32 a, TestRegi32 b ) { return a > b; } ); - RegisterTest( tests, "i32 less equal than operator", []( TestRegi32 a, TestRegi32 b ) { return a <= b; } ); - RegisterTest( tests, "i32 greater equal than operator", []( TestRegi32 a, TestRegi32 b ) { return a >= b; } ); - - RegisterTest( tests, "i32 select", []( TestRegm32 m, TestRegi32 a, TestRegi32 b ) { return FS::Select( m, a, b ); } ); - RegisterTest( tests, "i32 select high bit", []( TestRegf32 m, TestRegi32 a, TestRegi32 b ) { return FS::SelectHighBit( m, a, b ); } ); - RegisterTest( tests, "i32 masked", []( TestRegm32 m, TestRegi32 a ) { return FS::Masked( m, a ); } ); - RegisterTest( tests, "i32 inv masked", []( TestRegm32 m, TestRegi32 a ) { return FS::InvMasked( m, a ); } ); - RegisterTest( tests, "i32 masked increment", []( TestRegm32 m, TestRegi32 a ) { return FS::MaskedIncrement( m, a ); } ); - RegisterTest( tests, "i32 masked decrement", []( TestRegm32 m, TestRegi32 a ) { return FS::MaskedDecrement( m, a ); } ); - RegisterTest( tests, "i32 masked add", []( TestRegm32 m, TestRegi32 a, TestRegi32 b ) { return FS::MaskedAdd( m, a, b ); } ); - RegisterTest( tests, "i32 masked sub", []( TestRegm32 m, TestRegi32 a, TestRegi32 b ) { return FS::MaskedSub( m, a, b ); } ); - RegisterTest( tests, "i32 masked mul", []( TestRegm32 m, TestRegi32 a, TestRegi32 b ) { return FS::MaskedMul( m, a, b ); } ); - RegisterTest( tests, "i32 inv masked add", []( TestRegm32 m, TestRegi32 a, TestRegi32 b ) { return FS::InvMaskedAdd( m, a, b ); } ); - RegisterTest( tests, "i32 inv masked sub", []( TestRegm32 m, TestRegi32 a, TestRegi32 b ) { return FS::InvMaskedSub( m, a, b ); } ); - RegisterTest( tests, "i32 inv masked mul", []( TestRegm32 m, TestRegi32 a, TestRegi32 b ) { return FS::InvMaskedMul( m, a, b ); } ); - - RegisterTest( tests, "f32 load store", []( TestRegf32 a ) { return a; } ); - RegisterTest( tests, "f32 load scalar", []( float a ) { return TestRegf32( a ); } ); - RegisterTest( tests, "f32 splat", []( float a ) { return FS::Splat( a ); } ); - RegisterTest( tests, "f32 extract 0", []( TestRegf32 a ) { return FS::Extract0( a ); } ); + RegisterTest( tests, "i32 equals operator", []( TestRegi32 a, TestRegi32 b ) { return a == b; }, []( int32_t a, int32_t b ) { return a == b; } ); + RegisterTest( tests, "i32 equals operator alt", []( TestRegi32 a ) { return a == a; }, []( int32_t ) { return true; } ); + RegisterTest( tests, "i32 not equals operator", []( TestRegi32 a, TestRegi32 b ) { return a != b; }, []( int32_t a, int32_t b ) { return a != b; } ); + RegisterTest( tests, "i32 not equals operator alt", []( TestRegi32 a ) { return a != a; }, []( int32_t ) { return false; } ); + RegisterTest( tests, "i32 less than operator", []( TestRegi32 a, TestRegi32 b ) { return a < b; }, []( int32_t a, int32_t b ) { return a < b; } ); + RegisterTest( tests, "i32 greater than operator", []( TestRegi32 a, TestRegi32 b ) { return a > b; }, []( int32_t a, int32_t b ) { return a > b; } ); + RegisterTest( tests, "i32 less equal than operator", []( TestRegi32 a, TestRegi32 b ) { return a <= b; }, []( int32_t a, int32_t b ) { return a <= b; } ); + RegisterTest( tests, "i32 greater equal than operator", []( TestRegi32 a, TestRegi32 b ) { return a >= b; }, []( int32_t a, int32_t b ) { return a >= b; } ); + + RegisterTest( tests, "i32 select", []( TestRegm32 m, TestRegi32 a, TestRegi32 b ) { return FS::Select( m, a, b ); }, []( bool m, int32_t a, int32_t b ) { return m ? a : b; } ); + // i32's own mask type, as int compares return + RegisterTest( tests, "i32 select int mask", []( TestRegm32i m, TestRegi32 a, TestRegi32 b ) { return FS::Select( m, a, b ); }, []( bool m, int32_t a, int32_t b ) { return m ? a : b; } ); + RegisterTest( tests, "i32 masked int mask", []( TestRegm32i m, TestRegi32 a ) { return FS::Masked( m, a ); }, []( bool m, int32_t a ) { return m ? a : 0; } ); + RegisterTest( tests, "i32 masked increment int mask", []( TestRegm32i m, TestRegi32 a ) { return FS::MaskedIncrement( m, a ); }, []( bool m, int32_t a ) { return m ? Wrap( U( a ) + 1u ) : a; } ); + RegisterTest( tests, "i32 select high bit", []( TestRegf32 m, TestRegi32 a, TestRegi32 b ) { return FS::SelectHighBit( m, a, b ); }, + []( float m, int32_t a, int32_t b ) { return ( Bits( m ) >> 31 ) ? a : b; } ).Inputs( In::AnyNonFinite() ); + RegisterTest( tests, "i32 masked", []( TestRegm32 m, TestRegi32 a ) { return FS::Masked( m, a ); }, []( bool m, int32_t a ) { return m ? a : 0; } ); + RegisterTest( tests, "i32 inv masked", []( TestRegm32 m, TestRegi32 a ) { return FS::InvMasked( m, a ); }, []( bool m, int32_t a ) { return m ? 0 : a; } ); + RegisterTest( tests, "i32 masked increment", []( TestRegm32 m, TestRegi32 a ) { return FS::MaskedIncrement( m, a ); }, []( bool m, int32_t a ) { return m ? Wrap( U( a ) + 1u ) : a; } ); + RegisterTest( tests, "i32 masked decrement", []( TestRegm32 m, TestRegi32 a ) { return FS::MaskedDecrement( m, a ); }, []( bool m, int32_t a ) { return m ? Wrap( U( a ) - 1u ) : a; } ); + RegisterTest( tests, "i32 masked add", []( TestRegm32 m, TestRegi32 a, TestRegi32 b ) { return FS::MaskedAdd( m, a, b ); }, []( bool m, int32_t a, int32_t b ) { return m ? Wrap( U( a ) + U( b ) ) : a; } ); + RegisterTest( tests, "i32 masked sub", []( TestRegm32 m, TestRegi32 a, TestRegi32 b ) { return FS::MaskedSub( m, a, b ); }, []( bool m, int32_t a, int32_t b ) { return m ? Wrap( U( a ) - U( b ) ) : a; } ); + RegisterTest( tests, "i32 masked mul", []( TestRegm32 m, TestRegi32 a, TestRegi32 b ) { return FS::MaskedMul( m, a, b ); }, []( bool m, int32_t a, int32_t b ) { return m ? Wrap( U( a ) * U( b ) ) : a; } ); + RegisterTest( tests, "i32 inv masked add", []( TestRegm32 m, TestRegi32 a, TestRegi32 b ) { return FS::InvMaskedAdd( m, a, b ); }, []( bool m, int32_t a, int32_t b ) { return m ? a : Wrap( U( a ) + U( b ) ); } ); + RegisterTest( tests, "i32 inv masked sub", []( TestRegm32 m, TestRegi32 a, TestRegi32 b ) { return FS::InvMaskedSub( m, a, b ); }, []( bool m, int32_t a, int32_t b ) { return m ? a : Wrap( U( a ) - U( b ) ); } ); + RegisterTest( tests, "i32 inv masked mul", []( TestRegm32 m, TestRegi32 a, TestRegi32 b ) { return FS::InvMaskedMul( m, a, b ); }, []( bool m, int32_t a, int32_t b ) { return m ? a : Wrap( U( a ) * U( b ) ); } ); + + RegisterTest( tests, "f32 load store", []( TestRegf32 a ) { return a; }, []( float a ) { return Bits( a ); } ).Inputs( In::AnyNonFinite() ).BitExact(); + RegisterTest( tests, "f32 load scalar", []( float a ) { return TestRegf32( a ); }, []( float a ) { return Bits( a ); } ).Inputs( In::AnyNonFinite() ).BitExact(); + RegisterTest( tests, "f32 splat", []( float a ) { return FS::Splat( a ); }, []( float a ) { return Bits( a ); } ).Inputs( In::AnyNonFinite() ).BitExact(); + RegisterTest( tests, "f32 extract 0", []( TestRegf32 a ) { return FS::Extract0( a ); }, []( float a ) { return Bits( a ); } ).Inputs( In::AnyNonFinite() ).BitExact(); RegisterTest( tests, "f32 load incremented", []() { return FS::LoadIncremented(); } ); - RegisterTest( tests, "f32 plus operator", std::plus() ); - RegisterTest( tests, "f32 minus operator", std::minus() ); - RegisterTest( tests, "f32 multiply operator", std::multiplies() ); - RegisterTest( tests, "f32 divide operator", std::divides() ).relaxedAccuracy = 64; - - RegisterTest( tests, "f32 fused multiply add", []( TestRegf32 a, TestRegf32 b ) { return FS::FMulAdd( a, TestRegf32( -1 ), b ); } ); - RegisterTest( tests, "f32 fused multiply sub", []( TestRegf32 a, TestRegf32 b ) { return FS::FMulSub( a, TestRegf32( -1 ), b ); } ); - RegisterTest( tests, "f32 fused negative multiply add", []( TestRegf32 a, TestRegf32 b ) { return FS::FNMulAdd( a, TestRegf32( -1 ), b ); } ); - RegisterTest( tests, "f32 fused negative multiply sub", []( TestRegf32 a, TestRegf32 b ) { return FS::FNMulSub( a, TestRegf32( -1 ), b ); } ); - - RegisterTest( tests, "f32 bit and operator", std::bit_and() ); - RegisterTest( tests, "f32 bit or operator", std::bit_or() ); - RegisterTest( tests, "f32 bit xor operator", std::bit_xor() ); - RegisterTest( tests, "f32 bit not operator", std::bit_not() ); - RegisterTest( tests, "f32 negate operator", std::negate() ); - RegisterTest( tests, "f32 bit and not", []( TestRegf32 a, TestRegf32 b ) { return FS::BitwiseAndNot( a, b ); } ); - - RegisterTest( tests, "f32 equals operator", []( TestRegf32 a, TestRegf32 b ) { return a == b; } ); - RegisterTest( tests, "f32 greater equal than operator", []( TestRegf32 a, TestRegf32 b ) { return a >= b; } ); - RegisterTest( tests, "f32 not equals operator", []( TestRegf32 a, TestRegf32 b ) { return a != b; } ); - RegisterTest( tests, "f32 less than operator", []( TestRegf32 a, TestRegf32 b ) { return a < b; } ); - RegisterTest( tests, "f32 greater than operator", []( TestRegf32 a, TestRegf32 b ) { return a > b; } ); - RegisterTest( tests, "f32 less equal than operator", []( TestRegf32 a, TestRegf32 b ) { return a <= b; } ); - - RegisterTest( tests, "f32 increment", []( TestRegf32 a ) { return FS::Increment( a ); } ); - RegisterTest( tests, "f32 decrement", []( TestRegf32 a ) { return FS::Decrement( a ); } ); - RegisterTest( tests, "f32 abs", []( TestRegf32 a ) { return FS::Abs( a ); } ); - RegisterTest( tests, "f32 min", []( TestRegf32 a, TestRegf32 b ) { return FS::Min( a, b ); } ); - RegisterTest( tests, "f32 max", []( TestRegf32 a, TestRegf32 b ) { return FS::Max( a, b ); } ); - - RegisterTest( tests, "f32 select", []( TestRegm32 m, TestRegf32 a, TestRegf32 b ) { return FS::Select( m, a, b ); } ); - RegisterTest( tests, "f32 select high bit", []( TestRegi32 m, TestRegf32 a, TestRegf32 b ) { return FS::SelectHighBit( m, a, b ); } ); - RegisterTest( tests, "f32 masked", []( TestRegm32 m, TestRegf32 a ) { return FS::Masked( m, a ); } ); - RegisterTest( tests, "f32 inv masked", []( TestRegm32 m, TestRegf32 a ) { return FS::InvMasked( m, a ); } ); - RegisterTest( tests, "f32 masked increment", []( TestRegm32 m, TestRegf32 a ) { return FS::MaskedIncrement( m, a ); } ); - RegisterTest( tests, "f32 masked decrement", []( TestRegm32 m, TestRegf32 a ) { return FS::MaskedDecrement( m, a ); } ); - RegisterTest( tests, "f32 masked add", []( TestRegm32 m, TestRegf32 a, TestRegf32 b ) { return FS::MaskedAdd( m, a, b ); } ); - RegisterTest( tests, "f32 masked sub", []( TestRegm32 m, TestRegf32 a, TestRegf32 b ) { return FS::MaskedSub( m, a, b ); } ); - RegisterTest( tests, "f32 masked mul", []( TestRegm32 m, TestRegf32 a, TestRegf32 b ) { return FS::MaskedMul( m, a, b ); } ); - RegisterTest( tests, "f32 inv masked add", []( TestRegm32 m, TestRegf32 a, TestRegf32 b ) { return FS::InvMaskedAdd( m, a, b ); } ); - RegisterTest( tests, "f32 inv masked sub", []( TestRegm32 m, TestRegf32 a, TestRegf32 b ) { return FS::InvMaskedSub( m, a, b ); } ); - RegisterTest( tests, "f32 inv masked mul", []( TestRegm32 m, TestRegf32 a, TestRegf32 b ) { return FS::InvMaskedMul( m, a, b ); } ); - - RegisterTest( tests, "f32 round", []( TestRegf32 a ) { return FS::Round( a ); } ); - RegisterTest( tests, "f32 ceil", []( TestRegf32 a ) { return FS::Ceil( a ); } ); - RegisterTest( tests, "f32 floor", []( TestRegf32 a ) { return FS::Floor( a ); } ); - RegisterTest( tests, "f32 trunc", []( TestRegf32 a ) { return FS::Trunc( a ); } ); - RegisterTest( tests, "f32 signbit", []( TestRegf32 a, TestRegf32 b ) { return FS::SignBit( a ) ^ b; } ); - RegisterTest( tests, "f32 modulus", []( TestRegf32 a, TestRegf32 b ) { - a = FS::Min( FS::Max( a, TestRegf32( -100.0f ) ), TestRegf32( 100.0f ) ); - b = FS::Min( FS::Max( FS::Abs( b ), TestRegf32( 0.25f ) ), TestRegf32( 100.0f ) ) | FS::SignBit( b ); - return FS::Modulus( a, b ); - } ).relaxedAccuracy = 16384; - - RegisterTest( tests, "f32 sqrt", []( TestRegf32 a ) { return FS::Sqrt( FS::Min( FS::Max( FS::Abs( a ), TestRegf32( 1.e-16f ) ), TestRegf32( 1.e+16f ) ) ); } ); - RegisterTest( tests, "f32 inv sqrt", []( TestRegf32 a ) { return FS::InvSqrt( FS::Min( FS::Max( FS::Abs( a ), TestRegf32( 1.e-16f ) ), TestRegf32( 1.e+16f ) ) ); } ).relaxedAccuracy = 8192; - RegisterTest( tests, "f32 reciprocal", []( TestRegf32 a ) { - a = FS::Min( FS::Max( FS::Abs( a ), TestRegf32( 1.e-16f ) ), TestRegf32( 1.e+16f ) ) | FS::SignBit( a ); - return FS::Reciprocal( a ); - } ).relaxedAccuracy = 8192; - - RegisterTest( tests, "f32 cos", []( TestRegf32 a ) { return FS::Cos( FS::Min( FS::Max( a, TestRegf32( -4096.0f ) ), TestRegf32( 4096.0f ) ) ); } ).relaxedAccuracy = 16384; - RegisterTest( tests, "f32 sin", []( TestRegf32 a ) { return FS::Sin( FS::Min( FS::Max( a, TestRegf32( -4096.0f ) ), TestRegf32( 4096.0f ) ) ); } ).relaxedAccuracy = 16384; - RegisterTest( tests, "f32 tan", []( TestRegf32 a ) { return FS::Tan( FS::Min( FS::Max( a, TestRegf32( -4096.0f ) ), TestRegf32( 4096.0f ) ) ); } ).relaxedAccuracy = 16384; - RegisterTest( tests, "f32 acos", []( TestRegf32 a ) { return FS::ACos( FS::Min( FS::Max( a, TestRegf32( -0.999f ) ), TestRegf32( 0.999f ) ) ); } ).relaxedAccuracy = 8192; - RegisterTest( tests, "f32 asin", []( TestRegf32 a ) { - // Clamp to avoid extreme near-zero and near-1 values - a = FS::Min( FS::Max( a, TestRegf32( -0.999f ) ), TestRegf32( 0.999f ) ); - auto tooSmall = FS::Abs( a ) < TestRegf32( 1.e-6f ); - a = FS::Select( tooSmall, TestRegf32( 0.0f ), a ); - return FS::ASin( a ); - } ).relaxedAccuracy = 8192; - RegisterTest( tests, "f32 atan", []( TestRegf32 a ) { return FS::ATan( FS::Min( FS::Max( a, TestRegf32( -100.0f ) ), TestRegf32( 100.0f ) ) ); } ).relaxedAccuracy = 16384; - RegisterTest( tests, "f32 atan2", []( TestRegf32 a, TestRegf32 b ) { - a = FS::Min( FS::Max( a, TestRegf32( -1000.0f ) ), TestRegf32( 1000.0f ) ); - b = FS::Min( FS::Max( b, TestRegf32( -1000.0f ) ), TestRegf32( 1000.0f ) ); - // Avoid very small denominators that cause precision issues - auto bTooSmall = FS::Abs( b ) < TestRegf32( 0.001f ); - b = FS::Select( bTooSmall, TestRegf32( 0.001f ) | FS::SignBit( b ), b ); - return FS::ATan2( a, b ); - } ).relaxedAccuracy = 16384; - RegisterTest( tests, "f32 exp", []( TestRegf32 a ) { return FS::Exp( FS::Min( FS::Max( a, TestRegf32( -87.0f ) ), TestRegf32( 88.0f ) ) ); } ).relaxedAccuracy = 8192; - RegisterTest( tests, "f32 exp2", []( TestRegf32 a ) { return FS::Exp2( FS::Min( FS::Max( a, TestRegf32( -126.0f ) ), TestRegf32( 127.0f ) ) ); } ).relaxedAccuracy = 8192; - RegisterTest( tests, "f32 log", []( TestRegf32 a ) { return FS::Log( FS::Min( FS::Max( FS::Abs( a ), TestRegf32( 1.e-20f ) ), TestRegf32( 1.e+20f ) ) ); } ).relaxedAccuracy = 16384; - RegisterTest( tests, "f32 log2", []( TestRegf32 a ) { - a = FS::Min( FS::Max( FS::Abs( a ), TestRegf32( 1.e-20f ) ), TestRegf32( 1.e+20f ) ); - // Avoid values very close to 1.0 which cause precision issues - auto nearOne = FS::Abs( a - TestRegf32( 1.0f ) ) < TestRegf32( 1.e-5f ); - a = FS::Select( nearOne, TestRegf32( 1.0f ), a ); - return FS::Log2( a ); - } ).relaxedAccuracy = 16384; - RegisterTest( tests, "f32 pow", []( TestRegf32 a, TestRegf32 b ) { - a = FS::Min( FS::Max( FS::Abs( a ), TestRegf32( 1.e-8f ) ), TestRegf32( 1.e+8f ) ); - b = FS::Min( FS::Max( b, TestRegf32( -8.0f ) ), TestRegf32( 8.0f ) ); - return FS::Pow( a, b ); - } ).relaxedAccuracy = 16384; - RegisterTest( tests, "f32 pow zero base", []( TestRegf32 a, TestRegf32 b ) { - // Negative inputs become a zero base, ranges avoid Exp overflow - auto zero = a < TestRegf32( 0.0f ); - a = FS::Min( FS::Max( a, TestRegf32( 1.e-4f ) ), TestRegf32( 1.e+4f ) ); - a = FS::Select( zero, TestRegf32( 0.0f ), a ); - b = FS::Min( FS::Max( FS::Abs( b ), TestRegf32( 0.1f ) ), TestRegf32( 8.0f ) ); - return FS::Pow( a, b ); - } ).relaxedAccuracy = 16384; - - RegisterTest( tests, "i32 convert to f32", []( TestRegi32 a ) { return FS::Convert( a ); } ); - RegisterTest( tests, "f32 convert to i32", []( TestRegf32 a ) { return FS::Convert( FS::Min( FS::Max( a, TestRegf32( -2147483647 - 1 ) ), TestRegf32( 2147483520 ) ) ); } ); - - RegisterTest( tests, "f32 cast to i32", []( TestRegf32 a ) { return FS::Cast( a ); } ); - RegisterTest( tests, "i32 cast to f32", []( TestRegi32 a ) { return FS::Cast( a ); } ); - - if constexpr( !( SIMD & FeatureFlag::AVX512_F ) ) + RegisterTest( tests, "f32 plus operator", std::plus(), []( float a, float b ) { return a + b; } ).Inputs( In::AnyNonFinite(), In::AnyNonFinite() ); + RegisterTest( tests, "f32 minus operator", std::minus(), []( float a, float b ) { return a - b; } ).Inputs( In::AnyNonFinite(), In::AnyNonFinite() ); + RegisterTest( tests, "f32 multiply operator", std::multiplies(), []( float a, float b ) { return a * b; } ).Inputs( In::AnyNonFinite(), In::AnyNonFinite() ); + RegisterTest( tests, "f32 divide operator", std::divides(), []( float a, float b ) { return a / b; } ).Inputs( In::AnyNonFinite(), In::AnyNonFinite() ).RelaxedAccuracy( 64 ) + .KnownFailure( neonDivide, Tiers::NonRelaxed, FeatureSet::NEON, FeatureSet::NEON ) + .KnownFailureIf( neonDivideDenormal, denormalArg1, Tiers::Relaxed, FeatureSet::NEON, FeatureSet::NEON ) + .KnownFailureIf( neonDivideOverflow, quotientOverflows, Tiers::Relaxed, FeatureSet::NEON, FeatureSet::NEON ); + + // b = -1 keeps a * b exact, so fused and unfused results match + RegisterTest( tests, "f32 fused multiply add", []( TestRegf32 a, TestRegf32 b ) { return FS::FMulAdd( a, TestRegf32( -1 ), b ); }, []( float a, float b ) { return -a + b; } ).Inputs( In::AnyNonFinite(), In::AnyNonFinite() ); + RegisterTest( tests, "f32 fused multiply sub", []( TestRegf32 a, TestRegf32 b ) { return FS::FMulSub( a, TestRegf32( -1 ), b ); }, []( float a, float b ) { return -a - b; } ).Inputs( In::AnyNonFinite(), In::AnyNonFinite() ); + RegisterTest( tests, "f32 fused negative multiply add", []( TestRegf32 a, TestRegf32 b ) { return FS::FNMulAdd( a, TestRegf32( -1 ), b ); }, []( float a, float b ) { return b + a; } ).Inputs( In::AnyNonFinite(), In::AnyNonFinite() ); + RegisterTest( tests, "f32 fused negative multiply sub", []( TestRegf32 a, TestRegf32 b ) { return FS::FNMulSub( a, TestRegf32( -1 ), b ); }, []( float a, float b ) { return -b + a; } ).Inputs( In::AnyNonFinite(), In::AnyNonFinite() ); + + // 12 bit significands keep a * b exact, so fused and unfused results match while every argument is independent + const In exactProduct = In::Magnitude( 1.e-15, 1.e+15 ).Bits( 12 ); + RegisterTest( tests, "f32 fused multiply add exact product", []( TestRegf32 a, TestRegf32 b, TestRegf32 c ) { return FS::FMulAdd( a, b, c ); }, + []( float a, float b, float c ) { float ab = a * b; return ab + c; } ).Inputs( exactProduct, exactProduct, In::Range( -1.e+30, 1.e+30 ) ); + RegisterTest( tests, "f32 fused multiply sub exact product", []( TestRegf32 a, TestRegf32 b, TestRegf32 c ) { return FS::FMulSub( a, b, c ); }, + []( float a, float b, float c ) { float ab = a * b; return ab - c; } ).Inputs( exactProduct, exactProduct, In::Range( -1.e+30, 1.e+30 ) ); + RegisterTest( tests, "f32 fused negative multiply add exact product", []( TestRegf32 a, TestRegf32 b, TestRegf32 c ) { return FS::FNMulAdd( a, b, c ); }, + []( float a, float b, float c ) { float ab = a * b; return c - ab; } ).Inputs( exactProduct, exactProduct, In::Range( -1.e+30, 1.e+30 ) ); + RegisterTest( tests, "f32 fused negative multiply sub exact product", []( TestRegf32 a, TestRegf32 b, TestRegf32 c ) { return FS::FNMulSub( a, b, c ); }, + []( float a, float b, float c ) { float ab = a * b; return -c - ab; } ).Inputs( exactProduct, exactProduct, In::Range( -1.e+30, 1.e+30 ) ); + + RegisterTest( tests, "f32 bit and operator", std::bit_and(), []( float a, float b ) { return Bits( a ) & Bits( b ); } ).Inputs( In::AnyNonFinite(), In::AnyNonFinite() ).BitExact(); + RegisterTest( tests, "f32 bit or operator", std::bit_or(), []( float a, float b ) { return Bits( a ) | Bits( b ); } ).Inputs( In::AnyNonFinite(), In::AnyNonFinite() ).BitExact(); + RegisterTest( tests, "f32 bit xor operator", std::bit_xor(), []( float a, float b ) { return Bits( a ) ^ Bits( b ); } ).Inputs( In::AnyNonFinite(), In::AnyNonFinite() ).BitExact(); + RegisterTest( tests, "f32 bit not operator", std::bit_not(), []( float a ) { return ~Bits( a ); } ).Inputs( In::AnyNonFinite() ).BitExact(); + RegisterTest( tests, "f32 negate operator", std::negate(), []( float a ) { return Bits( a ) ^ 0x80000000u; } ).Inputs( In::AnyNonFinite() ).BitExact(); + // See i32 operators keep lhs. Divide isn't exact on NEON, its lhs is read again by FS::Modulus, see f32 modulus exact + RegisterTest( tests, "f32 operators keep lhs", []( TestRegm32 m, TestRegf32 a, TestRegf32 b ) { + return FS::Select( m, a + b, a ) ^ FS::Select( m, a - b, a ) ^ FS::Select( m, a * b, a ) ^ + FS::Select( m, a & b, a ) ^ FS::Select( m, a | b, a ) ^ FS::Select( m, a ^ b, a ) ^ a; + }, []( bool m, float a, float b ) { + uint32_t results[] = { Bits( a + b ), Bits( a - b ), Bits( a * b ), Bits( a ) & Bits( b ), Bits( a ) | Bits( b ), Bits( a ) ^ Bits( b ) }; + uint32_t combined = Bits( a ); + for( uint32_t result : results ) + { + combined ^= m ? result : Bits( a ); + } + return combined; + } ).Inputs( In::Any(), In::Any(), In::Any() ).BitExact(); + RegisterTest( tests, "f32 bit and not", []( TestRegf32 a, TestRegf32 b ) { return FS::BitwiseAndNot( a, b ); }, []( float a, float b ) { return Bits( a ) & ~Bits( b ); } ).Inputs( In::AnyNonFinite(), In::AnyNonFinite() ).BitExact(); + + RegisterTest( tests, "f32 equals operator", []( TestRegf32 a, TestRegf32 b ) { return a == b; }, []( float a, float b ) { return a == b; } ).Inputs( In::AnyNonFinite(), In::AnyNonFinite() ); + RegisterTest( tests, "f32 greater equal than operator", []( TestRegf32 a, TestRegf32 b ) { return a >= b; }, []( float a, float b ) { return a >= b; } ).Inputs( In::AnyNonFinite(), In::AnyNonFinite() ); + RegisterTest( tests, "f32 not equals operator", []( TestRegf32 a, TestRegf32 b ) { return a != b; }, []( float a, float b ) { return a != b; } ).Inputs( In::AnyNonFinite(), In::AnyNonFinite() ); + RegisterTest( tests, "f32 less than operator", []( TestRegf32 a, TestRegf32 b ) { return a < b; }, []( float a, float b ) { return a < b; } ).Inputs( In::AnyNonFinite(), In::AnyNonFinite() ); + RegisterTest( tests, "f32 greater than operator", []( TestRegf32 a, TestRegf32 b ) { return a > b; }, []( float a, float b ) { return a > b; } ).Inputs( In::AnyNonFinite(), In::AnyNonFinite() ); + RegisterTest( tests, "f32 less equal than operator", []( TestRegf32 a, TestRegf32 b ) { return a <= b; }, []( float a, float b ) { return a <= b; } ).Inputs( In::AnyNonFinite(), In::AnyNonFinite() ); + + RegisterTest( tests, "f32 increment", []( TestRegf32 a ) { return FS::Increment( a ); }, []( float a ) { return a + 1.0f; } ).Inputs( In::AnyNonFinite() ); + RegisterTest( tests, "f32 decrement", []( TestRegf32 a ) { return FS::Decrement( a ); }, []( float a ) { return a - 1.0f; } ).Inputs( In::AnyNonFinite() ); + RegisterTest( tests, "f32 abs", []( TestRegf32 a ) { return FS::Abs( a ); }, []( float a ) { return Bits( a ) & 0x7FFFFFFFu; } ).Inputs( In::AnyNonFinite() ).BitExact(); +#if defined( _MSC_VER ) && !defined( __clang__ ) && defined( _DEBUG ) + // The scalar reference is wrong, so every tier and the oracle comparison see it + const char* msvcDebugAbs = "Unoptimised MSVC calls the CRT fabsf for std::abs in the scalar FS::Abs, which quiets signalling NaNs"; + tests.back().second.KnownFailure( msvcDebugAbs ).KnownFailure( msvcDebugAbs, Tiers::Oracle ); +#endif + // Relaxed min/max may use instructions with different NaN handling, those are checked non relaxed in the full range tests + RegisterTest( tests, "f32 min", []( TestRegf32 a, TestRegf32 b ) { return FS::Min( a, b ); }, []( float a, float b ) { return std::min( a, b ); } ).Inputs( In::AnyInf(), In::AnyInf() ); + RegisterTest( tests, "f32 max", []( TestRegf32 a, TestRegf32 b ) { return FS::Max( a, b ); }, []( float a, float b ) { return std::max( a, b ); } ).Inputs( In::AnyInf(), In::AnyInf() ); + + RegisterTest( tests, "f32 select", []( TestRegm32 m, TestRegf32 a, TestRegf32 b ) { return FS::Select( m, a, b ); }, []( bool m, float a, float b ) { return Bits( m ? a : b ); } ).Inputs( In::Any(), In::AnyNonFinite(), In::AnyNonFinite() ).BitExact(); + RegisterTest( tests, "f32 select high bit", []( TestRegi32 m, TestRegf32 a, TestRegf32 b ) { return FS::SelectHighBit( m, a, b ); }, []( int32_t m, float a, float b ) { return Bits( m < 0 ? a : b ); } ).Inputs( In::Any(), In::AnyNonFinite(), In::AnyNonFinite() ).BitExact(); + RegisterTest( tests, "f32 masked", []( TestRegm32 m, TestRegf32 a ) { return FS::Masked( m, a ); }, []( bool m, float a ) { return m ? Bits( a ) : 0u; } ).Inputs( In::Any(), In::AnyNonFinite() ).BitExact(); + RegisterTest( tests, "f32 inv masked", []( TestRegm32 m, TestRegf32 a ) { return FS::InvMasked( m, a ); }, []( bool m, float a ) { return m ? 0u : Bits( a ); } ).Inputs( In::Any(), In::AnyNonFinite() ).BitExact(); + RegisterTest( tests, "f32 masked increment", []( TestRegm32 m, TestRegf32 a ) { return FS::MaskedIncrement( m, a ); }, []( bool m, float a ) { return m ? a + 1.0f : a; } ).Inputs( In::Any(), In::AnyNonFinite() ); + RegisterTest( tests, "f32 masked decrement", []( TestRegm32 m, TestRegf32 a ) { return FS::MaskedDecrement( m, a ); }, []( bool m, float a ) { return m ? a - 1.0f : PlusZero( a ); } ).Inputs( In::Any(), In::AnyNonFinite() ); + + // Generic masked add is a + Masked( m, b ), so masked off lanes add +0. AVX512 512bit uses a native masked add that leaves them unchanged + const char* avx512MaskedAddZeroSign = "AVX512 512bit masked add leaves masked off lanes unchanged, the generic a + Masked( m, b ) turns -0 into +0"; + TestData& maskedAdd = RegisterTest( tests, "f32 masked add", []( TestRegm32 m, TestRegf32 a, TestRegf32 b ) { return FS::MaskedAdd( m, a, b ); }, + []( bool m, float a, float b ) { return m ? a + b : PlusZero( a ); } ).Inputs( In::Any(), In::AnyNonFinite(), In::AnyNonFinite() ); + if constexpr( RegisterBytes >= 64 ) + { + maskedAdd.KnownZeroSign( avx512MaskedAddZeroSign, Tiers::NonRelaxed, FeatureSet::AVX512, FeatureSet::AVX512 ); + } + RegisterTest( tests, "f32 masked sub", []( TestRegm32 m, TestRegf32 a, TestRegf32 b ) { return FS::MaskedSub( m, a, b ); }, []( bool m, float a, float b ) { return m ? a - b : a; } ).Inputs( In::Any(), In::AnyNonFinite(), In::AnyNonFinite() ); + RegisterTest( tests, "f32 masked mul", []( TestRegm32 m, TestRegf32 a, TestRegf32 b ) { return FS::MaskedMul( m, a, b ); }, []( bool m, float a, float b ) { return m ? a * b : a; } ).Inputs( In::Any(), In::AnyNonFinite(), In::AnyNonFinite() ); + TestData& invMaskedAdd = RegisterTest( tests, "f32 inv masked add", []( TestRegm32 m, TestRegf32 a, TestRegf32 b ) { return FS::InvMaskedAdd( m, a, b ); }, + []( bool m, float a, float b ) { return m ? PlusZero( a ) : a + b; } ).Inputs( In::Any(), In::AnyNonFinite(), In::AnyNonFinite() ); + if constexpr( RegisterBytes >= 64 ) + { + invMaskedAdd.KnownZeroSign( avx512MaskedAddZeroSign, Tiers::NonRelaxed, FeatureSet::AVX512, FeatureSet::AVX512 ); + } + RegisterTest( tests, "f32 inv masked sub", []( TestRegm32 m, TestRegf32 a, TestRegf32 b ) { return FS::InvMaskedSub( m, a, b ); }, []( bool m, float a, float b ) { return m ? a : a - b; } ).Inputs( In::Any(), In::AnyNonFinite(), In::AnyNonFinite() ); + RegisterTest( tests, "f32 inv masked mul", []( TestRegm32 m, TestRegf32 a, TestRegf32 b ) { return FS::InvMaskedMul( m, a, b ); }, []( bool m, float a, float b ) { return m ? a : a * b; } ).Inputs( In::Any(), In::AnyNonFinite(), In::AnyNonFinite() ); + + RegisterTest( tests, "f32 round", []( TestRegf32 a ) { return FS::Round( a ); }, []( float a ) { return std::rint( a ); } ).Inputs( In::AnyNonFinite() ) + .KnownZeroSign( sse2RoundZeroSign, Tiers::NonRelaxed, FeatureSet::SSE, FeatureSet::SSSE3 ); + RegisterTest( tests, "f32 ceil", []( TestRegf32 a ) { return FS::Ceil( a ); }, []( float a ) { return std::ceil( a ); } ).Inputs( In::AnyNonFinite() ) + .KnownZeroSign( sse2RoundZeroSign, Tiers::NonRelaxed, FeatureSet::SSE, FeatureSet::SSSE3 ); + RegisterTest( tests, "f32 floor", []( TestRegf32 a ) { return FS::Floor( a ); }, []( float a ) { return std::floor( a ); } ).Inputs( In::AnyNonFinite() ) + .KnownZeroSign( sse2RoundZeroSign, Tiers::NonRelaxed, FeatureSet::SSE, FeatureSet::SSSE3 ); + RegisterTest( tests, "f32 trunc", []( TestRegf32 a ) { return FS::Trunc( a ); }, []( float a ) { return std::trunc( a ); } ).Inputs( In::AnyNonFinite() ) + .KnownZeroSign( sse2RoundZeroSign, Tiers::NonRelaxed, FeatureSet::SSE, FeatureSet::SSSE3 ); + RegisterTest( tests, "f32 signbit", []( TestRegf32 a, TestRegf32 b ) { return FS::SignBit( a ) ^ b; }, []( float a, float b ) { return ( Bits( a ) & 0x80000000u ) ^ Bits( b ); } ).Inputs( In::AnyNonFinite(), In::AnyNonFinite() ).BitExact(); + RegisterTest( tests, "f32 modulus", []( TestRegf32 a, TestRegf32 b ) { return FS::Modulus( a, b ); } ) + .Inputs( In::Range( -100, 100 ), In::Magnitude( 0.25, 100 ) ).RelaxedAccuracy( 16384 ) + .KnownFailure( "Scalar uses std::fmod, SIMD and relaxed use a - trunc( a / b ) * b which isn't exact" ); + // b is a power of 2 and a is never tiny, so a - trunc( a / b ) * b is exact and must match std::fmod + RegisterTest( tests, "f32 modulus exact", []( TestRegf32 a, TestRegf32 b ) { return FS::Modulus( a, b ); }, []( float a, float b ) { return std::fmod( a, b ); } ) + .Inputs( In::Magnitude( 1.e-30, 100 ), In::Magnitude( 0.25, 64 ).Bits( 1 ) ) + .KnownZeroSign( "a - trunc( a / b ) * b gives +0 for exact negative multiples, std::fmod gives -0", Tiers::NonRelaxed ); + + RegisterTest( tests, "f32 sqrt", []( TestRegf32 a ) { return FS::Sqrt( a ); }, []( float a ) { return std::sqrt( a ); } ).Inputs( In::AnyNonFinite() ); + // Relaxed x86 uses the rcpps/rsqrtps estimates, their 1.5 * 2^-12 relative error is up to ~6144 ULP + RegisterTest( tests, "f32 inv sqrt", []( TestRegf32 a ) { return FS::InvSqrt( a ); }, []( float a ) { return 1.0f / std::sqrt( a ); } ).Inputs( In::Range( 1.e-16, 1.e+16 ) ).RelaxedAccuracy( 8192 ) + .KnownFailure( neonDivide, Tiers::NonRelaxed, FeatureSet::NEON, FeatureSet::NEON ); + RegisterTest( tests, "f32 reciprocal", []( TestRegf32 a ) { return FS::Reciprocal( a ); }, []( float a ) { return 1.0f / a; } ).Inputs( In::Magnitude( 1.e-16, 1.e+16 ) ).RelaxedAccuracy( 8192 ) + .KnownFailure( neonDivide, Tiers::NonRelaxed, FeatureSet::NEON, FeatureSet::NEON ); + + // Absolute error allowance is ~1 ULP of the largest input, float inputs that large are only that precise + RegisterTest( tests, "f32 cos", []( TestRegf32 a ) { return FS::Cos( a ); } ).Inputs( trigRange ).RelaxedAccuracy( 4, 5.e-4f ); + RegisterTest( tests, "f32 sin", []( TestRegf32 a ) { return FS::Sin( a ); } ).Inputs( trigRange ).RelaxedAccuracy( 4, 5.e-4f ) + .KnownZeroSign( "Sin_Reduce uses Trunc, see f32 trunc: Sin( -0 ) is -0 below SSE4.1 and +0 elsewhere", Tiers::NonRelaxed, FeatureSet::SSE, FeatureSet::SSSE3 ); + // Within +-2pi range reduction is exact apart from the float rounding of pi and 2pi (<= ~2.6e-7), so the polynomials are checked closely + RegisterTest( tests, "f32 cos small range", []( TestRegf32 a ) { return FS::Cos( a ); } ).Inputs( smallTrigRange ).RelaxedAccuracy( 4, 5.e-7f ); + RegisterTest( tests, "f32 sin small range", []( TestRegf32 a ) { return FS::Sin( a ); } ).Inputs( smallTrigRange ).RelaxedAccuracy( 4, 5.e-7f ) + .KnownZeroSign( "Sin_Reduce uses Trunc, see f32 trunc: Sin( -0 ) is -0 below SSE4.1 and +0 elsewhere", Tiers::NonRelaxed, FeatureSet::SSE, FeatureSet::SSSE3 ); + // The absolute allowances above hide relative accuracy near 0 + RegisterTest( tests, "f32 sin near 0", []( TestRegf32 a ) { return FS::Sin( a ); } ).Inputs( In::Magnitude( denormMin, 0.01 ) ).RelaxedAccuracy( 4 ); + // Tan is ill conditioned near its poles, so relaxed accuracy is checked between them + RegisterTest( tests, "f32 tan", []( TestRegf32 a ) { return FS::Tan( a ); } ).Inputs( In::Range( -1.5, 1.5 ).Specials( { pi / 4, -pi / 4 } ) ).RelaxedAccuracy( 64 ) + .KnownFailure( neonDivide, Tiers::NonRelaxed, FeatureSet::NEON, FeatureSet::NEON ); + // Between the poles at +-pi/2 and +-3pi/2, Tan_Reduce subtracts pi and folds the sign, near +-pi the float rounding of pi is absolute error + RegisterTest( tests, "f32 tan reduced", []( TestRegf32 a ) { return FS::Tan( a ); } ).Inputs( In::Magnitude( 1.6, 4.6 ).Specials( { pi, 3 * pi / 4, 5 * pi / 4 } ) ).RelaxedAccuracy( 128, 3.e-7f ) + .KnownFailure( neonDivide, Tiers::NonRelaxed, FeatureSet::NEON, FeatureSet::NEON ); + RegisterTest( tests, "f32 acos", []( TestRegf32 a ) { return FS::ACos( a ); } ).Inputs( In::Range( -1, 1 ) ).RelaxedAccuracy( 8 ); + // ASin is K_HALF_PI - ACos( x ), near 0 that cancels and ACos's error (~2 ULP of pi/2) becomes absolute error, +0 for |x| <= 2^-25 + RegisterTest( tests, "f32 asin", []( TestRegf32 a ) { return FS::ASin( a ); } ).Inputs( In::Range( -1, 1 ) ).RelaxedAccuracy( 32, 3.e-7f ); + // Relaxed x86 ATan uses the rcpps estimate for |x| > 1, its 1.5 * 2^-12 relative error is up to ~3072 ULP of atan near 1 + // Relaxed scalar is std::atan, so FS::ATan itself is checked much more closely + RegisterTest( tests, "f32 atan", []( TestRegf32 a ) { return FS::ATan( a ); } ).Inputs( In::AnyInf() ).RelaxedAccuracy( 4096 ).ScalarRelaxedAccuracy( 64 ) + .KnownFailure( neonDivide, Tiers::NonRelaxed, FeatureSet::NEON, FeatureSet::NEON ); + RegisterTest( tests, "f32 atan2", []( TestRegf32 a, TestRegf32 b ) { return FS::ATan2( a, b ); } ).Inputs( In::AnyInf(), In::AnyInf() ).RelaxedAccuracy( 4096 ).ScalarRelaxedAccuracy( 64 ) + .KnownFailure( "Relaxed scalar uses std::atan2: for atan2( +-0, -0 ) it returns +-pi where FS::ATan2 returns 0, and for atan2( +-inf, +-inf ) +-pi/4 or +-3pi/4 where FS::ATan2 returns NaN (inf / inf)", Tiers::ScalarRelaxed ) + .KnownFailure( neonDivide, Tiers::NonRelaxed, FeatureSet::NEON, FeatureSet::NEON ) + .KnownFailureIf( neonDivideDenormal, denormalArg1, Tiers::Relaxed, FeatureSet::NEON, FeatureSet::NEON ); + // Keep relaxed scalar checking FS::ATan2 against std::atan2 everywhere except both inputs 0 + RegisterTest( tests, "f32 atan2 nonzero y", []( TestRegf32 a, TestRegf32 b ) { return FS::ATan2( a, b ); } ).Inputs( In::Magnitude( denormMin, floatMax ), In::AnyInf() ).RelaxedAccuracy( 4096 ).ScalarRelaxedAccuracy( 64 ) + .KnownFailure( neonDivide, Tiers::NonRelaxed, FeatureSet::NEON, FeatureSet::NEON ) + .KnownFailureIf( neonDivideDenormal, denormalArg1, Tiers::Relaxed, FeatureSet::NEON, FeatureSet::NEON ); + RegisterTest( tests, "f32 atan2 nonzero x", []( TestRegf32 a, TestRegf32 b ) { return FS::ATan2( a, b ); } ).Inputs( In::AnyInf(), In::Magnitude( denormMin, floatMax ) ).RelaxedAccuracy( 4096 ).ScalarRelaxedAccuracy( 64 ) + .KnownFailure( neonDivide, Tiers::NonRelaxed, FeatureSet::NEON, FeatureSet::NEON ) + .KnownFailureIf( neonDivideDenormal, denormalArg1, Tiers::Relaxed, FeatureSet::NEON, FeatureSet::NEON ); + // FS::Exp clamps its input to +-88.3762626647949, the range reaches just below that and keeps results above FLT_MIN + RegisterTest( tests, "f32 exp", []( TestRegf32 a ) { return FS::Exp( a ); } ).Inputs( In::Range( -87.3, 88.37 ) ).RelaxedAccuracy( 8 ); + RegisterTest( tests, "f32 exp2", []( TestRegf32 a ) { return FS::Exp2( a ); } ).Inputs( In::Range( -126, 127 ) ).RelaxedAccuracy( 8 ); + RegisterTest( tests, "f32 log", []( TestRegf32 a ) { return FS::Log( a ); } ).Inputs( In::Range( 1.e-20, 1.e+20 ) ).RelaxedAccuracy( 8 ); + // Log2 reduces the mantissa to [1,2), just below 1 it cancels to ~2e-7 absolute error + RegisterTest( tests, "f32 log2", []( TestRegf32 a ) { return FS::Log2( a ); } ).Inputs( In::Range( 1.e-20, 1.e+20 ) ).RelaxedAccuracy( 8, 3.e-7f ); + // Results stay within float range, FS::Pow saturates where std::pow returns inf + // Pow is Exp( b * Log( a ) ), so Log's relative error is scaled by b * ln( a ), up to ~74 here + RegisterTest( tests, "f32 pow", []( TestRegf32 a, TestRegf32 b ) { return FS::Pow( a, b ); } ) + .Inputs( In::Range( 1.e-4, 1.e+4 ), In::Range( -8, 8 ) ).RelaxedAccuracy( 256 ); + RegisterTest( tests, "f32 pow zero base", []( TestRegf32 b ) { return FS::Pow( TestRegf32( 0.0f ), b ); }, []( float ) { return 0.0f; } ).Inputs( In::Range( 0.1, 8 ) ); + + // Relaxed x86 uses rcpps/rsqrtps, AVX512 512bit registers use rcp14/rsqrt14 which handle denormals + const char* x86EstimateDenormals = "Relaxed rcpps/rsqrtps estimates treat denormal inputs as 0 and return inf"; + // The Intel SDM allows rcpps to flush results just above FLT_MIN to 0, for |x| just below 2^126 + const char* x86ReciprocalEstimate = "Relaxed rcpps treats denormal inputs as 0 and returns inf, and may flush results near FLT_MIN to 0"; + constexpr auto x86ReciprocalInput = []( const LaneArgs& args ) { return std::fpclassify( args.floats[0] ) == FP_SUBNORMAL || std::abs( args.floats[0] ) >= 0x1.ffcp125f; }; + constexpr FastSIMD::FeatureSet x86EstimateMax = RegisterBytes >= 64 ? FeatureSet::AVX2 : FeatureSet::AVX512; + // Relaxed ARM refines vrsqrte/vrecpe with one Newton-Raphson step, which breaks where the estimate is 0 or inf + const char* armEstimateSpecials = "Relaxed ARM estimate + Newton-Raphson step: InvSqrt( +-0 ) and InvSqrt( inf ) are NaN (0 * inf), Reciprocal of a denormal is inf of the wrong sign"; + constexpr auto armEstimateInput = []( const LaneArgs& args ) { return args.floats[0] == 0 || std::isinf( args.floats[0] ) || std::fpclassify( args.floats[0] ) == FP_SUBNORMAL; }; + RegisterTest( tests, "f32 inv sqrt full range", []( TestRegf32 a ) { return FS::InvSqrt( a ); }, []( float a ) { return 1.0f / std::sqrt( a ); } ).Inputs( In::AnyNonFinite() ).RelaxedAccuracy( 8192 ) + .KnownFailure( neonDivide, Tiers::NonRelaxed, FeatureSet::NEON, FeatureSet::NEON ) + .KnownFailureIf( x86EstimateDenormals, denormalArg0, Tiers::Relaxed, FeatureSet::SSE, x86EstimateMax ) + .KnownFailureIf( armEstimateSpecials, armEstimateInput, Tiers::Relaxed, FeatureSet::NEON, FeatureSet::AARCH64 ); + RegisterTest( tests, "f32 reciprocal full range", []( TestRegf32 a ) { return FS::Reciprocal( a ); }, []( float a ) { return 1.0f / a; } ).Inputs( In::AnyNonFinite() ).RelaxedAccuracy( 8192 ) + .KnownFailure( neonDivide, Tiers::NonRelaxed, FeatureSet::NEON, FeatureSet::NEON ) + .KnownFailureIf( x86ReciprocalEstimate, x86ReciprocalInput, Tiers::Relaxed, FeatureSet::SSE, x86EstimateMax ) + .KnownFailureIf( armEstimateSpecials, armEstimateInput, Tiers::Relaxed, FeatureSet::NEON, FeatureSet::AARCH64 ); + + // Relaxed accuracy is only meaningful over the ranges above, non relaxed results must still match exactly over the whole domain + if constexpr( !FastSIMD::IsRelaxed() ) + { + RegisterTest( tests, "f32 min full range", []( TestRegf32 a, TestRegf32 b ) { return FS::Min( a, b ); }, []( float a, float b ) { return std::min( a, b ); } ).Inputs( In::AnyNonFinite(), In::AnyNonFinite() ); + RegisterTest( tests, "f32 max full range", []( TestRegf32 a, TestRegf32 b ) { return FS::Max( a, b ); }, []( float a, float b ) { return std::max( a, b ); } ).Inputs( In::AnyNonFinite(), In::AnyNonFinite() ); + RegisterTest( tests, "f32 cos full range", []( TestRegf32 a ) { return FS::Cos( a ); } ).Inputs( In::AnyNonFinite() ); + RegisterTest( tests, "f32 sin full range", []( TestRegf32 a ) { return FS::Sin( a ); } ).Inputs( In::AnyNonFinite() ) + .KnownZeroSign( "Sin_Reduce uses Trunc, see f32 trunc: Sin( -0 ) is -0 below SSE4.1 and +0 elsewhere", Tiers::NonRelaxed, FeatureSet::SSE, FeatureSet::SSSE3 ); + RegisterTest( tests, "f32 tan full range", []( TestRegf32 a ) { return FS::Tan( a ); } ).Inputs( In::AnyNonFinite() ) + .KnownFailure( neonDivide, Tiers::NonRelaxed, FeatureSet::NEON, FeatureSet::NEON ); + // Exp clamps its input, no NaN since converting it to int is undefined in the scalar reference + RegisterTest( tests, "f32 exp full range", []( TestRegf32 a ) { return FS::Exp( a ); } ) + .Inputs( In::AnyInf().Specials( { 88.3762626647949, -88.3762626647949, 88.72283935546875, -87.33654022216797, -103.97207641601562 } ) ); + RegisterTest( tests, "f32 log full range", []( TestRegf32 a ) { return FS::Log( a ); } ).Inputs( In::Range( denormMin, floatMax ) ); + RegisterTest( tests, "f32 log2 full range", []( TestRegf32 a ) { return FS::Log2( a ); } ).Inputs( In::Range( denormMin, floatMax ) ); + RegisterTest( tests, "f32 pow full range", []( TestRegf32 a, TestRegf32 b ) { return FS::Pow( a, b ); } ).Inputs( In::Range( denormMin, floatMax ), In::Any() ); + } + + RegisterTest( tests, "i32 convert to f32", []( TestRegi32 a ) { return FS::Convert( a ); }, []( int32_t a ) { return static_cast( a ); } ); + RegisterTest( tests, "f32 convert to i32", []( TestRegf32 a ) { return FS::Convert( a ); }, []( float a ) { return static_cast( std::rint( a ) ); } ) + .Inputs( In::Range( -2147483648.0, 2147483520.0 ) ); + + RegisterTest( tests, "f32 cast to i32", []( TestRegf32 a ) { return FS::Cast( a ); }, []( float a ) { return static_cast( Bits( a ) ); } ).Inputs( In::AnyNonFinite() ); + RegisterTest( tests, "i32 cast to f32", []( TestRegi32 a ) { return FS::Cast( a ); }, []( int32_t a ) { return U( a ); } ).BitExact(); + + // Cast is a raw reinterpret, a mask's bits as another type differ between SIMD levels (and k-registers can't be cast) + // Only the round trip back to the mask is defined, so that's all these test + if constexpr( !( SIMD & FeatureFlag::AVX512_F ) || RegisterBytes < 64 ) { - RegisterTest( tests, "m32 cast to i32", []( TestRegm32 a ) { return FS_BIND_INTRINSIC( FS::Cast> )( FS_BIND_INTRINSIC( FS::Cast )( a ) ); } ); - RegisterTest( tests, "m32 cast to f32", []( TestRegm32 a ) { return FS_BIND_INTRINSIC( FS::Cast> )( FS_BIND_INTRINSIC( FS::Cast )( a ) ); } ); - RegisterTest( tests, "m32i cast to i32", []( TestRegm32i a ) { return FS_BIND_INTRINSIC( FS::Cast )( FS_BIND_INTRINSIC( FS::Cast )( a ) ); } ); - RegisterTest( tests, "m32i cast to f32", []( TestRegm32i a ) { return FS_BIND_INTRINSIC( FS::Cast )( FS_BIND_INTRINSIC( FS::Cast )( a ) ); } ); + RegisterTest( tests, "m32 cast to i32", []( TestRegm32 a ) { return FS_BIND_INTRINSIC( FS::Cast> )( FS_BIND_INTRINSIC( FS::Cast )( a ) ); }, []( bool a ) { return a; } ); + RegisterTest( tests, "m32 cast to f32", []( TestRegm32 a ) { return FS_BIND_INTRINSIC( FS::Cast> )( FS_BIND_INTRINSIC( FS::Cast )( a ) ); }, []( bool a ) { return a; } ); + RegisterTest( tests, "m32i cast to i32", []( TestRegm32i a ) { return FS_BIND_INTRINSIC( FS::Cast )( FS_BIND_INTRINSIC( FS::Cast )( a ) ); }, []( bool a ) { return a; } ); + RegisterTest( tests, "m32i cast to f32", []( TestRegm32i a ) { return FS_BIND_INTRINSIC( FS::Cast )( FS_BIND_INTRINSIC( FS::Cast )( a ) ); }, []( bool a ) { return a; } ); } return tests; } }; -template class FastSIMD::RegisterDispatchClass>; +// Each register size is built as its own dispatch library, see tests/CMakeLists.txt +template class FastSIMD::RegisterDispatchClass>; From 29097761d629a99d7420934ef816912f5c973de7 Mon Sep 17 00:00:00 2001 From: Jordan Peck Date: Fri, 16 Oct 2026 15:50:18 +0100 Subject: [PATCH 2/4] Work around MSVC 19.51 miscompiling the SSE2 f32 Select MSVC 19.51 (VS 2026) optimises xor( ifFalse, and( mask, xor( ifTrue, ifFalse ) ) ) wrongly where FS::ATan2 inlines it, giving wrong results on the SSE2 feature set. The mirrored xor( ifTrue, andnot( mask, xor( ifTrue, ifFalse ) ) ) is the same 3 instructions and register use, and compiles correctly. --- include/FastSIMD/ToolSet/x86/128/f32x4.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/include/FastSIMD/ToolSet/x86/128/f32x4.h b/include/FastSIMD/ToolSet/x86/128/f32x4.h index badd308..7667a8c 100644 --- a/include/FastSIMD/ToolSet/x86/128/f32x4.h +++ b/include/FastSIMD/ToolSet/x86/128/f32x4.h @@ -224,7 +224,8 @@ namespace FS } else { - return _mm_xor_ps( ifFalse.native, _mm_and_ps( mask.native, _mm_xor_ps( ifTrue.native, ifFalse.native ) ) ); + // MSVC 19.51 miscompiles the equivalent xor( ifFalse, and( mask, xor( ifTrue, ifFalse ) ) ) inside FS::ATan2 + return _mm_xor_ps( ifTrue.native, _mm_andnot_ps( mask.native, _mm_xor_ps( ifTrue.native, ifFalse.native ) ) ); } } From daf326858a749a4261e6999db3c5cc9b006ee296 Mon Sep 17 00:00:00 2001 From: Jordan Peck Date: Fri, 16 Oct 2026 19:51:59 +0100 Subject: [PATCH 3/4] Restructure the test harness - Input generation moves to test_inputs.h/.cpp, test.cpp keeps test collection, the run loop, the summary and main - Tests are collected with a runtime loop over the compiled feature sets, only collection depends on the register size - A tier is one kind (non relaxed, relaxed or oracle) and every setting is read from the SCALAR non relaxed reference. Tiers registering different known failure counts, bit exactness or tolerances are rejected, and duplicate names are caught in every tier - Known failures are named values scoped once and registered with .Known( ... ), RegisterTests is split by category - Known failure input conditions see a lane's arguments the same way the oracle does - Each argument has one typed input buffer, mask lanes are stored as the type they're compared in - Comparing a call returns its outcome, the return type is switched on once per test - Input specs default to every value of the argument's type, Finite and NoNaN narrow it. In::Any() is gone so specs written for the old default don't compile Inputs and results are unchanged for every seed. Checked by fingerprinting every test's inputs and tier results against the previous harness on MSVC x64 (full count), Linux GCC/Clang and AArch64 GCC/Clang, GCC only differs in which NaN payload propagates. "Generating Tests" now prints in ascending feature set order and the MSVC Debug f32 abs known failure is one entry. --- tests/CMakeLists.txt | 4 +- tests/test.cpp | 1270 ++++++++++------------------------------- tests/test.h | 291 ++++++---- tests/test.inl | 439 +++++++------- tests/test_inputs.cpp | 582 +++++++++++++++++++ tests/test_inputs.h | 62 ++ 6 files changed, 1353 insertions(+), 1295 deletions(-) create mode 100644 tests/test_inputs.cpp create mode 100644 tests/test_inputs.h diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index e3abb49..b6d817a 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -25,7 +25,7 @@ foreach(register_bytes 16 32 64 128) endforeach() # "test" is a reserved target name when CTest is enabled -add_executable(FastSIMD_tests "test.cpp") +add_executable(FastSIMD_tests "test.cpp" "test_inputs.cpp") target_link_libraries(FastSIMD_tests PRIVATE FastSIMD ${simd_test_libraries}) # Random inputs must be the same on every platform for a seed, stop arm64 compilers fusing the range maths into FMA @@ -51,7 +51,7 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Emscripten") endif() # Single page to run the tests in a browser - add_executable(FastSIMD_tests_web "test.cpp") + add_executable(FastSIMD_tests_web "test.cpp" "test_inputs.cpp") target_link_libraries(FastSIMD_tests_web PRIVATE FastSIMD ${simd_test_libraries}) set_target_properties(FastSIMD_tests_web PROPERTIES SUFFIX ".html") target_link_options(FastSIMD_tests_web PRIVATE -sALLOW_MEMORY_GROWTH=1 -sSINGLE_FILE $<$:-g0>) diff --git a/tests/test.cpp b/tests/test.cpp index 723746c..a5745c6 100644 --- a/tests/test.cpp +++ b/tests/test.cpp @@ -1,4 +1,5 @@ #include "test.h" +#include "test_inputs.h" #include #include @@ -58,1166 +59,525 @@ struct TestSummary std::vector failed; }; -// Inputs are generated and tested in chunks to keep memory use flat -static constexpr size_t kChunkSize = 1 << 14; static constexpr int kMaxFailuresPerTier = 3; -static_assert( kChunkSize % kMaxTestLanes == 0, "Calls must not straddle input chunks" ); - -static uint64_t SplitMix64( uint64_t& state ) +static std::string FeatureSetName( FastSIMD::FeatureSet featureSet ) { - uint64_t z = ( state += 0x9E3779B97F4A7C15ull ); - z = ( z ^ ( z >> 30 ) ) * 0xBF58476D1CE4E5B9ull; - z = ( z ^ ( z >> 27 ) ) * 0x94D049BB133111EBull; - return z ^ ( z >> 31 ); + return FastSIMD::GetFeatureSetString( featureSet ); } -// FNV-1a, stable across compilers unlike std::hash -static uint64_t HashName( std::string_view name ) +static TestTier TierOf( const TestData& test ) { - uint64_t hash = 0xCBF29CE484222325ull; - for( char c : name ) - { - hash = ( hash ^ static_cast( c ) ) * 0x100000001B3ull; - } - return hash; + return { test.featureSet, test.relaxed ? Tiers::Relaxed : Tiers::NonRelaxed }; } -template -struct FloatTraits; - -template<> -struct FloatTraits -{ - using Bits = uint32_t; - static constexpr int kSignificandBits = 24; -}; - -template<> -struct FloatTraits +static std::string TierName( const TestTier& tier ) { - using Bits = uint64_t; - static constexpr int kSignificandBits = 53; -}; - -template -static typename FloatTraits::Bits ToBits( T f ) -{ - typename FloatTraits::Bits bits; - std::memcpy( &bits, &f, sizeof( f ) ); - return bits; + if( tier.kind == Tiers::Oracle ) + { + return "SCALAR vs ORACLE"; + } + return FeatureSetName( tier.featureSet ) + ( tier.kind == Tiers::Relaxed ? " RELAXED" : "" ); } -template -static T FromBits( typename FloatTraits::Bits bits ) +static std::string JoinNames( const std::vector& names ) { - T f; - std::memcpy( &f, &bits, sizeof( f ) ); - return f; -} - -static float BitsToFloat( uint32_t bits ) { return FromBits( bits ); } - -static std::string FeatureSetName( FastSIMD::FeatureSet featureSet ) -{ - return FastSIMD::GetFeatureSetString( featureSet ); + std::string joined; + for( size_t i = 0; i < names.size(); i++ ) + { + joined += ( i ? ", " : "" ) + names[i]; + } + return joined; } -template -struct FeatureSetsOf; +// Each test name's registrations in tier order, the first is the SCALAR non relaxed reference +using TestSet = std::vector>>; -template -struct FeatureSetsOf> +template +static void AddTier( TestSet& set, FastSIMD::FeatureSet featureSet ) { - static std::vector Get() { return { SETS... }; } -}; + std::unique_ptr> testClass( FastSIMD::NewDispatchClass>( featureSet ) ); -static std::vector CompiledFeatureSets() -{ - return FeatureSetsOf::Get(); -} - -// Checks a test's input specs can be generated, mistakes here would silently test the wrong domain -static void ValidateInputs( std::string_view testName, const TestData& test ) -{ - auto fail = [&]( size_t argIdx, const char* problem ) + if( !testClass ) { - std::string message = "Invalid input spec for \"" + std::string( testName ) + "\" argument " + std::to_string( argIdx ) + ": " + problem; - std::cerr << message << std::endl; - throw std::runtime_error( message ); - }; + throw std::runtime_error( "No test class created for " + FeatureSetName( featureSet ) ); + } - constexpr double floatMax = std::numeric_limits::max(); + bool isReference = featureSet == FastSIMD::FeatureSet::SCALAR && !Relaxed; - for( size_t argIdx = 0; argIdx < kMaxTestArgs; argIdx++ ) + for( auto& registered : testClass->RegisterTests() ) { - const InputSpec& spec = test.inputs[argIdx]; - bool isDefault = spec.kind == InputSpec::Kind::Any && !spec.inf && !spec.nan && spec.significandBits == 0 && spec.specials.empty(); + std::string_view testName = registered.first; + TestData& test = registered.second; - if( argIdx >= test.argCount ) + if( test.featureSet != featureSet || test.relaxed != Relaxed ) { - if( !isDefault ) - { - fail( argIdx, "test has fewer arguments than input specs" ); - } - continue; + throw std::runtime_error( "Test class for " + FeatureSetName( featureSet ) + " registered \"" + std::string( testName ) + "\" for " + FeatureSetName( test.featureSet ) ); } - if( test.argTypes[argIdx] == TestArgType::Mask && !isDefault ) - { - fail( argIdx, "mask arguments are always generated from lane patterns, they take no input spec" ); - } - TestArgType type = test.argTypes[argIdx]; - int maxBits = type == TestArgType::Float ? 24 : type == TestArgType::Double ? 53 : 0; - if( spec.significandBits < 0 || spec.significandBits > maxBits ) - { - fail( argIdx, maxBits ? "significand bits must be in [1, 24] for floats, [1, 53] for doubles" : "significand bits only apply to floats and doubles" ); - } - for( double special : spec.specials ) + auto find = std::find_if( set.begin(), set.end(), [testName]( const auto& pair ) { return pair.first == testName; } ); + + if( isReference ) { - if( std::isnan( special ) ) + if( find != set.end() ) { - fail( argIdx, "special values can't be NaN, use AnyNonFinite" ); + throw std::runtime_error( "Multiple tests with same name: " + std::string( testName ) ); } - } - if( spec.kind == InputSpec::Kind::Any ) - { + set.emplace_back( testName, std::vector{ std::move( test ) } ); continue; } - if( spec.inf || spec.nan ) - { - fail( argIdx, "inf and NaN are only supported by Any specs" ); - } - if( !std::isfinite( spec.lo ) || !std::isfinite( spec.hi ) || spec.lo > spec.hi ) - { - fail( argIdx, "range bounds must be finite with lo <= hi" ); - } - if( spec.kind == InputSpec::Kind::Magnitude && spec.lo <= 0 ) - { - fail( argIdx, "magnitude lo must be > 0" ); - } - if( test.argTypes[argIdx] == TestArgType::Float && ( std::abs( spec.lo ) > floatMax || std::abs( spec.hi ) > floatMax ) ) + + if( find == set.end() ) { - fail( argIdx, "range bounds must be within float range" ); + throw std::runtime_error( "\"" + std::string( testName ) + "\" has no SCALAR non relaxed reference" ); } - if( test.argTypes[argIdx] == TestArgType::Int && - ( std::ceil( spec.lo ) > std::floor( spec.hi ) || std::ceil( spec.lo ) < -2147483648.0 || std::floor( spec.hi ) > 2147483647.0 ) ) + if( std::any_of( find->second.begin(), find->second.end(), [&]( const TestData& other ) { return other.featureSet == featureSet && other.relaxed == Relaxed; } ) ) { - fail( argIdx, "int range must contain an integer and be within int32 range" ); + throw std::runtime_error( "Multiple tests with same name: " + std::string( testName ) + " (" + TierName( TierOf( test ) ) + ")" ); } + find->second.push_back( std::move( test ) ); } } -// Generates the input values for every argument of a test -// Only uses std::mt19937_64 and arithmetic without FMA contraction (see tests/CMakeLists.txt) so a seed produces the same values on any compiler -class InputGenerator +// Only creating the test classes depends on the register size +template +static TestSet CollectTests( FastSIMD::FeatureSet maxFeatureSet ) { -public: - InputGenerator( uint64_t seed, std::string_view testName, const TestData& test, size_t lanes ) : - mArgCount( test.argCount ), mLanes( lanes ) - { - mSeed = seed ^ HashName( testName ); - mCombinations = mArgCount ? 1 : 0; + TestSet set; - for( size_t argIdx = 0; argIdx < mArgCount; argIdx++ ) - { - Arg& arg = mArgs[argIdx]; - arg.type = test.argTypes[argIdx]; - arg.spec = test.inputs[argIdx]; - - switch( arg.type ) - { - case TestArgType::Float: - arg.floatSpecials = FloatSpecials( arg.spec ); - arg.specialCount = arg.floatSpecials.size(); - arg.floats.resize( kChunkSize ); - break; - case TestArgType::Double: - arg.doubleSpecials = FloatSpecials( arg.spec ); - arg.specialCount = arg.doubleSpecials.size(); - arg.doubles.resize( kChunkSize ); - break; - case TestArgType::Int: - arg.intSpecials = IntSpecials( arg.spec ); - arg.specialCount = arg.intSpecials.size(); - arg.ints.resize( kChunkSize ); - break; - case TestArgType::Mask: - arg.specialCount = MaskPatternCount(); - arg.ints.resize( kChunkSize ); - arg.maskFloats.resize( kChunkSize ); - arg.maskDoubles.resize( kChunkSize ); - break; - } - - mCombinations *= arg.specialCount; - } - } - - // Inputs below this index are combinations of special values - size_t SpecialCount() const - { - return mCombinations * mLanes; - } - - void Generate( size_t chunkStart, TestInputs& inputs ) + for( FastSIMD::FeatureSet featureSet : TestFeatureSets::AsArray ) { - inputs = {}; - - for( size_t argIdx = 0; argIdx < mArgCount; argIdx++ ) + if( featureSet <= FastSIMD::DetectCpuMaxFeatureSet() && featureSet <= maxFeatureSet ) { - Arg& arg = mArgs[argIdx]; - uint64_t state = mSeed + chunkStart * kMaxTestArgs + argIdx; - std::mt19937_64 rng( SplitMix64( state ) ); - - for( size_t i = 0; i < kChunkSize; i++ ) - { - size_t element = chunkStart + i; - bool special = element < SpecialCount(); - - switch( arg.type ) - { - case TestArgType::Float: - arg.floats[i] = special ? arg.floatSpecials[SpecialDigit( element, argIdx )] : RandomFloat( arg.spec, rng ); - break; - case TestArgType::Double: - arg.doubles[i] = special ? arg.doubleSpecials[SpecialDigit( element, argIdx )] : RandomFloat( arg.spec, rng ); - break; - case TestArgType::Int: - arg.ints[i] = special ? arg.intSpecials[SpecialDigit( element, argIdx )] : RandomInt( arg.spec, rng ); - break; - case TestArgType::Mask: - arg.ints[i] = special ? MaskPatternLane( SpecialDigit( element, argIdx ), element % mLanes ) : static_cast( rng() & 1 ); - arg.maskFloats[i] = static_cast( arg.ints[i] ); - arg.maskDoubles[i] = static_cast( arg.ints[i] ); - break; - } - } - - switch( arg.type ) - { - case TestArgType::Float: - inputs.floats[argIdx] = arg.floats.data(); - break; - case TestArgType::Double: - inputs.doubles[argIdx] = arg.doubles.data(); - break; - case TestArgType::Int: - inputs.ints[argIdx] = arg.ints.data(); - break; - case TestArgType::Mask: - inputs.masks[argIdx] = arg.ints.data(); - inputs.maskFloats[argIdx] = arg.maskFloats.data(); - inputs.maskDoubles[argIdx] = arg.maskDoubles.data(); - break; - } - } - } - -private: - struct Arg - { - TestArgType type = TestArgType::Float; - InputSpec spec; - std::vector floatSpecials; - std::vector doubleSpecials; - std::vector intSpecials; - size_t specialCount = 0; - std::vector floats; - std::vector doubles; - std::vector ints; // Ints, or 0/1 mask lanes - std::vector maskFloats; - std::vector maskDoubles; - }; - - // Value arguments use combination (call + lane), putting every combination in every lane and in lane 0 for scalar arguments - // Mask arguments use combination (call) so a whole mask pattern lands in one call - size_t SpecialDigit( size_t element, size_t argIdx ) const - { - size_t call = element / mLanes; - size_t combination = mArgs[argIdx].type == TestArgType::Mask ? call : call + element % mLanes; - combination %= mCombinations; + std::cout << "Generating Tests: " << FeatureSetName( featureSet ) << std::endl; - for( size_t i = 0; i < argIdx; i++ ) - { - combination /= mArgs[i].specialCount; + AddTier( set, featureSet ); + AddTier( set, featureSet ); } - return combination % mArgs[argIdx].specialCount; } + return set; +} - // All clear, all set, each single lane set, each single lane clear, alternating, and each half set - size_t MaskPatternCount() const - { - return 2 + mLanes * 2 + 4; - } +// Every tier's results are read as the reference's types, and only the reference's settings are used, so a tier registering others is an error +static void CheckTests( std::string_view testName, const std::vector& tests ) +{ + const TestData& reference = tests[0]; - int32_t MaskPatternLane( size_t pattern, size_t lane ) const + for( size_t testIdx = 1; testIdx < tests.size(); testIdx++ ) { - if( pattern < 2 ) - { - return static_cast( pattern ); - } - pattern -= 2; - if( pattern < mLanes ) - { - return lane == pattern; - } - pattern -= mLanes; - if( pattern < mLanes ) - { - return lane != pattern; - } - pattern -= mLanes; + const TestData& test = tests[testIdx]; - switch( pattern ) + if( test.returnType != reference.returnType || test.lanes != reference.lanes || test.argCount != reference.argCount || + !std::equal( test.argTypes, test.argTypes + kMaxTestArgs, reference.argTypes ) || + test.knownFailures.size() != reference.knownFailures.size() || test.bitExact != reference.bitExact || + test.relaxedTolerance != reference.relaxedTolerance || test.scalarRelaxedTolerance != reference.scalarRelaxedTolerance ) { - case 0: return ( lane & 1 ) == 0; - case 1: return ( lane & 1 ) == 1; - case 2: return lane < mLanes / 2; - default: return lane >= mLanes / 2; + throw std::runtime_error( "Tests do not match: " + std::string( testName ) + " (" + TierName( TierOf( test ) ) + ")" ); } } - template - static T RangeLo( const InputSpec& spec ) { return static_cast( spec.lo ); } - template - static T RangeHi( const InputSpec& spec ) { return static_cast( spec.hi ); } - - // Int bounds are the integers inside [lo, hi], ValidateInputs checks there is at least one - static int64_t IntLo( const InputSpec& spec ) { return static_cast( std::ceil( spec.lo ) ); } - static int64_t IntHi( const InputSpec& spec ) { return static_cast( std::floor( spec.hi ) ); } - - // Drops the low significand bits, finite values only - template - static T Truncate( T value, int significandBits ) + for( const KnownFailure& knownFailure : reference.knownFailures ) { - constexpr int kBits = FloatTraits::kSignificandBits; - if( significandBits == 0 || significandBits >= kBits || !std::isfinite( value ) ) + if( knownFailure.reason.empty() ) { - return value; + throw std::runtime_error( "Known failure with no reason in \"" + std::string( testName ) + "\"" ); } - using Bits = typename FloatTraits::Bits; - Bits lowBits = ( Bits( 1 ) << ( kBits - significandBits ) ) - 1; - return FromBits( ToBits( value ) & ~lowBits ); } +} - template - static bool InFloatDomain( const InputSpec& spec, T value ) - { - switch( spec.kind ) - { - case InputSpec::Kind::Any: - return ( spec.nan || !std::isnan( value ) ) && ( spec.inf || !std::isinf( value ) ); - case InputSpec::Kind::Range: - return value >= RangeLo( spec ) && value <= RangeHi( spec ); - case InputSpec::Kind::Magnitude: - return std::abs( value ) >= RangeLo( spec ) && std::abs( value ) <= RangeHi( spec ); - } - return false; - } +// Value of the least significant bit, relaxed differences are counted in these +// Zero and denormals use the smallest normal rather than the denormal spacing, so relaxed results may flush denormals +template +static T DeltaUnit( T x ) +{ + x = std::fabs( x ); - // Values of every magnitude that commonly break SIMD code - static std::vector AnySpecials( const InputSpec& spec, float ) + if( !std::isfinite( x ) ) { - constexpr float inf = std::numeric_limits::infinity(); - constexpr float floatMin = std::numeric_limits::min(); - constexpr float denormMin = std::numeric_limits::denorm_min(); - const float denormMax = BitsToFloat( 0x007FFFFF ); - - std::vector candidates = { - 0.0f, -0.0f, 1.0f, -1.0f, 0.5f, -0.5f, 1.5f, -1.5f, 2.5f, -2.5f, - 0.49999997f, -0.49999997f, 0.99999994f, -0.99999994f, 1.0000001f, -1.0000001f, - floatMin, -floatMin, denormMin, -denormMin, denormMax, -denormMax, - std::numeric_limits::max(), -std::numeric_limits::max(), - 8388607.5f, -8388607.5f, 8388608.0f, -8388608.0f, 16777216.0f, -16777216.0f, - 2147483520.0f, -2147483520.0f, 2147483648.0f, -2147483648.0f, - 3.14159274f, -3.14159274f, 1.e-10f, 1.e+10f, - }; - if( spec.inf ) - { - candidates.insert( candidates.end(), { inf, -inf } ); - } - if( spec.nan ) - { - // Quiet, negative, signalling and payload NaNs - for( uint32_t bits : { 0x7FC00000u, 0xFFC00000u, 0x7F800001u, 0xFF800001u, 0x7FFFFFFFu, 0xFFC00001u } ) - { - candidates.push_back( BitsToFloat( bits ) ); - } - } - return candidates; + return T( 1 ); } - - static std::vector AnySpecials( const InputSpec& spec, double ) + if( x == T( 0 ) || !std::isnormal( x ) ) { - constexpr double inf = std::numeric_limits::infinity(); - constexpr double doubleMin = std::numeric_limits::min(); - constexpr double denormMin = std::numeric_limits::denorm_min(); - const double denormMax = FromBits( 0x000FFFFFFFFFFFFFull ); - constexpr double floatMax = std::numeric_limits::max(); - constexpr double floatMin = std::numeric_limits::min(); - constexpr double floatDenormMin = std::numeric_limits::denorm_min(); - - std::vector candidates = { - 0.0, -0.0, 1.0, -1.0, 0.5, -0.5, 1.5, -1.5, 2.5, -2.5, - std::nextafter( 0.5, 0.0 ), -std::nextafter( 0.5, 0.0 ), std::nextafter( 1.0, 0.0 ), -std::nextafter( 1.0, 0.0 ), - std::nextafter( 1.0, 2.0 ), -std::nextafter( 1.0, 2.0 ), - doubleMin, -doubleMin, denormMin, -denormMin, denormMax, -denormMax, - std::numeric_limits::max(), -std::numeric_limits::max(), - // Integer precision limits, and the int32 convert range - 4503599627370495.5, -4503599627370495.5, 4503599627370496.0, -4503599627370496.0, 9007199254740992.0, -9007199254740992.0, - 2147483647.0, -2147483648.0, 2147483647.5, -2147483648.5, 2147483648.0, -2147483649.0, - // Narrowing to float: overflow, float denormals, and rounding ties - floatMax, -floatMax, 3.4028235677973366e+38, -3.4028235677973366e+38, floatMin, -floatMin, floatDenormMin, -floatDenormMin, - floatDenormMin / 2, 1.0 + 0x1p-24, -( 1.0 + 0x1p-24 ), 1.0 + 0x1.8p-23, 1.0 + 0x1p-23 + 0x1p-25, - 3.141592653589793, -3.141592653589793, 1.e-10, 1.e+10, 1.e-300, 1.e+300, - }; - if( spec.inf ) - { - candidates.insert( candidates.end(), { inf, -inf } ); - } - if( spec.nan ) - { - // Quiet, negative, signalling and payload NaNs, including payloads narrowing to float drops - for( uint64_t bits : { 0x7FF8000000000000ull, 0xFFF8000000000000ull, 0x7FF0000000000001ull, 0xFFF0000000000001ull, - 0x7FFFFFFFFFFFFFFFull, 0xFFF8000000000001ull, 0x7FF4000000000000ull } ) - { - candidates.push_back( FromBits( bits ) ); - } - } - return candidates; + return std::numeric_limits::min(); // smallest positive normal number } - template - static std::vector FloatSpecials( const InputSpec& spec ) - { - constexpr T inf = std::numeric_limits::infinity(); - constexpr T typeMin = std::numeric_limits::min(); - constexpr T denormMin = std::numeric_limits::denorm_min(); - const T denormMax = std::nextafter( typeMin, T( 0 ) ); - const T belowHalf = std::nextafter( T( 0.5 ), T( 0 ) ); - const T belowOne = std::nextafter( T( 1 ), T( 0 ) ); - const T aboveOne = std::nextafter( T( 1 ), T( 2 ) ); - std::vector candidates; - - switch( spec.kind ) - { - case InputSpec::Kind::Any: - candidates = AnySpecials( spec, T() ); - break; + T above = std::nextafter( x, std::numeric_limits::infinity() ); - case InputSpec::Kind::Range: - candidates = { RangeLo( spec ), RangeHi( spec ), std::nextafter( RangeLo( spec ), inf ), std::nextafter( RangeHi( spec ), -inf ) }; - for( T v : { T( 0 ), T( 0.5 ), T( 1 ), T( 1.5 ), T( 2.5 ), belowHalf, belowOne, aboveOne, typeMin, denormMin, denormMax, T( 1.e-10 ), T( 1.e-30 ) } ) - { - candidates.insert( candidates.end(), { v, -v } ); - } - break; - - case InputSpec::Kind::Magnitude: - for( T v : { RangeLo( spec ), RangeHi( spec ), std::nextafter( RangeLo( spec ), inf ), std::nextafter( RangeHi( spec ), -inf ), - T( 0.5 ), T( 1 ), T( 2 ), belowOne, aboveOne, typeMin, denormMin, denormMax } ) - { - candidates.insert( candidates.end(), { v, -v } ); - } - break; - } + // The largest finite value has nothing finite above it, use the gap below instead + return std::isfinite( above ) ? above - x : x - std::nextafter( x, T( 0 ) ); +} - for( double special : spec.specials ) - { - T v = static_cast( special ); - candidates.insert( candidates.end(), { v, std::nextafter( v, inf ), std::nextafter( v, -inf ) } ); - } +enum class LaneResult +{ + Equal, ZeroSign, Different +}; - std::vector specials; - for( T candidate : candidates ) - { - T v = Truncate( candidate, spec.significandBits ); +struct LaneComparison +{ + LaneResult result; + double ulpsCovered = 0; // Relaxed difference only the ULP tolerance covered, for --stats + double absCovered = 0; // Relaxed difference only the absolute tolerance covered +}; - // Compare bits to keep both 0 and -0, and each NaN - bool duplicate = std::any_of( specials.begin(), specials.end(), [v]( T s ) { return ToBits( s ) == ToBits( v ); } ); +template +static LaneComparison CompareLane( const TestData& test, const TestTier& tier, T expected, T actual ) +{ + if constexpr( std::is_floating_point_v ) + { + bool bitsEqual = ToBits( expected ) == ToBits( actual ); + bool bothNaN = std::isnan( expected ) && std::isnan( actual ); - if( InFloatDomain( spec, v ) && !duplicate ) - { - specials.push_back( v ); - } - } - if( specials.empty() ) + if( bitsEqual || ( bothNaN && !test.bitExact ) ) { - throw std::runtime_error( "Input spec has no special values in its domain" ); + return { LaneResult::Equal }; } - return specials; - } - - static std::vector IntSpecials( const InputSpec& spec ) - { - constexpr int32_t intMax = std::numeric_limits::max(); - constexpr int32_t intMin = std::numeric_limits::min(); - std::vector candidates; - switch( spec.kind ) + // Relaxed results are compared by value, so +0 and -0 are equal + if( tier.kind == Tiers::Relaxed && !test.bitExact ) { - case InputSpec::Kind::Any: - candidates = { - 0, 1, -1, 2, -2, 31, 32, 33, -31, -32, - intMax, intMin, intMax - 1, intMin + 1, - 0x7F800000, static_cast( 0xFF800000 ), 0x7FC00000, 0x7F800001, // inf, -inf, NaN, signalling NaN float bits - 0x3F800000, 0x00800000, 0x0000FFFF, 0x00010000, 0x55555555, static_cast( 0xAAAAAAAA ), - }; - break; - case InputSpec::Kind::Range: - candidates = { IntLo( spec ), IntHi( spec ), IntLo( spec ) + 1, IntHi( spec ) - 1, 0, 1, -1 }; - break; - case InputSpec::Kind::Magnitude: - for( int64_t v : { IntLo( spec ), IntHi( spec ), IntLo( spec ) + 1, IntHi( spec ) - 1, int64_t( 1 ) } ) + if( expected == actual ) { - candidates.insert( candidates.end(), { v, -v } ); + return { LaneResult::Equal }; } - break; - } - for( double special : spec.specials ) - { - candidates.push_back( static_cast( std::round( std::min( std::max( special, -2147483648.0 ), 2147483647.0 ) ) ) ); - } + Tolerance tolerance = test.RelaxedTolerance( tier.featureSet ); + double absoluteDif = std::abs( static_cast( expected ) - static_cast( actual ) ); + double relativeDif = absoluteDif / DeltaUnit( expected ); - std::vector specials; - for( int64_t v : candidates ) - { - bool inDomain = true; - switch( spec.kind ) + if( relativeDif <= tolerance.ulps || absoluteDif <= tolerance.absolute ) { - case InputSpec::Kind::Any: - inDomain = v >= intMin && v <= intMax; - break; - case InputSpec::Kind::Range: - inDomain = v >= IntLo( spec ) && v <= IntHi( spec ); - break; - case InputSpec::Kind::Magnitude: - inDomain = std::abs( v ) >= IntLo( spec ) && std::abs( v ) <= IntHi( spec ); - break; - } + LaneComparison equal = { LaneResult::Equal }; - if( inDomain && std::find( specials.begin(), specials.end(), v ) == specials.end() ) - { - specials.push_back( static_cast( v ) ); + // Track what each tolerance had to cover on its own + if( std::isfinite( absoluteDif ) ) + { + if( absoluteDif > tolerance.absolute ) + { + equal.ulpsCovered = relativeDif; + } + if( relativeDif > tolerance.ulps ) + { + equal.absCovered = absoluteDif; + } + } + return equal; } + return { LaneResult::Different }; } - return specials; - } - // Uniform [0, 1) - static double RandomUnit( std::mt19937_64& rng ) - { - return static_cast( rng() >> 11 ) * ( 1.0 / 9007199254740992.0 ); + return { expected == 0 && actual == 0 ? LaneResult::ZeroSign : LaneResult::Different }; } - - // Log scaled magnitude within [lo, hi], lo > 0: a uniform power of 2 binade, then uniform within its part of [lo, hi] - template - static T RandomMagnitude( double lo, double hi, std::mt19937_64& rng ) + else { - int loExp, hiExp; - std::frexp( lo, &loExp ); - std::frexp( hi, &hiExp ); - - int exponent = loExp + static_cast( rng() % static_cast( hiExp - loExp + 1 ) ); - double binadeLo = std::max( lo, std::ldexp( 0.5, exponent ) ); - double binadeHi = std::min( hi, std::ldexp( 1.0, exponent ) ); - T value = static_cast( binadeLo + ( binadeHi - binadeLo ) * RandomUnit( rng ) ); - - return std::min( std::max( value, static_cast( lo ) ), static_cast( hi ) ); + return { expected == actual ? LaneResult::Equal : LaneResult::Different }; } +} - template - static T RandomFloat( const InputSpec& spec, std::mt19937_64& rng ) - { - constexpr double denormMin = std::numeric_limits::denorm_min(); - - for( ;; ) - { - T value = 0; - - switch( spec.kind ) - { - case InputSpec::Kind::Any: - value = FromBits( static_cast::Bits>( rng() ) ); - break; - - case InputSpec::Kind::Range: - if( rng() & 1 ) - { - value = static_cast( spec.lo + ( spec.hi - spec.lo ) * RandomUnit( rng ) ); - } - else - { - // Log scaled half covers small magnitudes, including denormals, that uniform values almost never hit - bool positive = spec.lo >= 0 || ( spec.hi > 0 && ( rng() & 1 ) ); - double maxMag = positive ? spec.hi : -spec.lo; - double minMag = positive ? std::max( spec.lo, 0.0 ) : std::max( -spec.hi, 0.0 ); - value = RandomMagnitude( std::max( minMag, denormMin ), std::max( maxMag, denormMin ), rng ); - value = positive ? value : -value; - } - break; - - case InputSpec::Kind::Magnitude: - value = RandomMagnitude( spec.lo, spec.hi, rng ); - value = ( rng() & 1 ) ? -value : value; - break; - } - - value = Truncate( value, spec.significandBits ); - - if( InFloatDomain( spec, value ) ) - { - return value; - } - } - } +template +static std::string Describe( T expected, T actual ) +{ + std::ostringstream sstr; + sstr << std::setprecision( std::is_floating_point_v ? std::numeric_limits::max_digits10 : 16 ) << std::boolalpha; + sstr << "Expected \"" << expected << "\" Actual \"" << actual << "\""; - static int32_t RandomInt( const InputSpec& spec, std::mt19937_64& rng ) + if constexpr( std::is_floating_point_v ) { - switch( spec.kind ) - { - case InputSpec::Kind::Any: - return static_cast( rng() ); - - case InputSpec::Kind::Range: - return static_cast( IntLo( spec ) + static_cast( rng() % static_cast( IntHi( spec ) - IntLo( spec ) + 1 ) ) ); + T absoluteDif = std::abs( expected - actual ); + sstr << " Diff \"" << absoluteDif << "\""; - case InputSpec::Kind::Magnitude: + if( std::isfinite( absoluteDif ) && absoluteDif != 0 ) { - int64_t value = IntLo( spec ) + static_cast( rng() % static_cast( IntHi( spec ) - IntLo( spec ) + 1 ) ); - return static_cast( ( rng() & 1 ) ? -value : value ); + sstr << " (" << absoluteDif / DeltaUnit( expected ) << " ULP)"; } + if( ToBits( expected ) != ToBits( actual ) && ( absoluteDif == 0 || std::isnan( absoluteDif ) ) ) + { + sstr << std::hex << " (bits 0x" << ToBits( expected ) << " vs 0x" << ToBits( actual ) << ")" << std::dec; } - return 0; } + else if constexpr( std::is_same_v ) + { + sstr << " Diff \"" << static_cast( expected ) - static_cast( actual ) << "\""; + } + else if constexpr( std::is_same_v ) + { + sstr << std::hex << " (bits 0x" << static_cast( expected ) << " vs 0x" << static_cast( actual ) << ")" << std::dec; + } + return sstr.str(); +} - size_t mArgCount; - size_t mLanes; - uint64_t mSeed; - size_t mCombinations; - Arg mArgs[kMaxTestArgs]; -}; - -enum class LaneResult -{ - Equal, ZeroSign, Different -}; - -struct TierResult +// A tier's run state while one test runs +struct Tier : TestTier { - TestTier tier; std::string name; - const TestData* data = nullptr; + const std::function& testFunc; // The tier's test, or the oracle for the oracle tier + bool allDifferencesKnown; // A known failure covers any difference on this tier, so comparing more of its calls can't fail size_t compared = 0; // Calls compared int failures = 0; // Calls with differences not covered by a known failure int knownCalls = 0; // Calls with only known differences double maxUlps = 0; // Largest relaxed difference only the ULP tolerance covered, for --stats double maxAbs = 0; // Largest relaxed difference only the absolute tolerance covered + + // A tier stops being checked after kMaxFailuresPerTier failing calls, or known failing calls if a known failure covers every difference + bool Stopped() const + { + return failures >= kMaxFailuresPerTier || ( allDifferencesKnown && knownCalls >= kMaxFailuresPerTier ); + } }; -template -struct TestRunner +// Ordered, a call's outcome is its worst lane's +enum class CallOutcome { - using TestSet = std::vector>>; + Pass, Known, Fail +}; - template - struct TestOrganiser - { - static TestCollection GetCollections( FastSIMD::FeatureSet ) - { - return {}; - } - }; +struct CallResult +{ + CallOutcome outcome = CallOutcome::Pass; + std::string details; // A line for each difference no known failure covers +}; - template - struct TestOrganiser> +// Compares every tier of one test on every call, then reports what they found +class TestRun +{ +public: + TestRun( size_t registerBytes, std::string_view testName, const std::vector& tests ) : + mWidth( std::to_string( registerBytes * 8 ) + "bit " ), mTestName( testName ), mReference( tests[0] ), mKnownSeen( tests[0].knownFailures.size() ) { - template - static TestCollection RegisterTests() - { - std::unique_ptr> testSimd( FastSIMD::NewDispatchClass>( HEAD ) ); - - if( !testSimd ) - { - throw std::runtime_error( "No test class created for " + FeatureSetName( HEAD ) ); - } - - TestCollection collection = testSimd->RegisterTests(); - - for( auto& test : collection ) - { - if( test.second.featureSet != HEAD || test.second.relaxed != RELAXED ) - { - throw std::runtime_error( "Test class for " + FeatureSetName( HEAD ) + " registered \"" + std::string( test.first ) + "\" for " + FeatureSetName( test.second.featureSet ) ); - } - } - return collection; - } - - static TestCollection GetCollections( FastSIMD::FeatureSet maxFeatureSet ) - { - TestCollection collections = TestOrganiser>::GetCollections( maxFeatureSet ); - - if( HEAD <= FastSIMD::DetectCpuMaxFeatureSet() && HEAD <= maxFeatureSet ) - { - std::cout << "Generating Tests: " << FeatureSetName( HEAD ) << std::endl; - - TestCollection relaxed = RegisterTests(); - collections.insert( collections.begin(), relaxed.begin(), relaxed.end() ); - - TestCollection nonRelaxed = RegisterTests(); - collections.insert( collections.begin(), nonRelaxed.begin(), nonRelaxed.end() ); - } - - return collections; - } - - static TestSet GetSet( FastSIMD::FeatureSet maxFeatureSet ) + auto addComparison = [this]( TestTier tier, const std::function& testFunc ) { - TestCollection collections = GetCollections( maxFeatureSet ); - TestSet set; + bool allDifferencesKnown = std::any_of( mReference.knownFailures.begin(), mReference.knownFailures.end(), + [&]( const KnownFailure& knownFailure ) { return knownFailure.CoversAnyDifference() && knownFailure.Covers( tier ); } ); - for( auto& collection : collections ) - { - std::string_view& testName = collection.first; - - auto find = std::find_if( set.begin(), set.end(), [testName]( const auto& pair ){ return pair.first == testName; } ); - - if( find == set.end() ) - { - if( collection.second.featureSet != FastSIMD::FeatureSet::SCALAR || collection.second.relaxed ) - { - throw std::runtime_error( "\"" + std::string( testName ) + "\" has no SCALAR non relaxed reference" ); - } - - find = set.emplace( set.end(), testName, std::vector{} ); - } - - find->second.emplace_back( collection.second ); - } - - return set; - } - }; - - // Value of the least significant bit, relaxed differences are counted in these - // Zero and denormals use the smallest normal rather than the denormal spacing, so relaxed results may flush denormals - template - static T DeltaUnit( T x ) - { - x = std::fabs( x ); + mTiers.push_back( { tier, TierName( tier ), testFunc, allDifferencesKnown } ); + }; - if( !std::isfinite( x ) ) + if( mReference.oracleFunc ) { - return T( 1 ); + addComparison( { FastSIMD::FeatureSet::SCALAR, Tiers::Oracle }, mReference.oracleFunc ); } - if( x == T( 0 ) || !std::isnormal( x ) ) + for( size_t testIdx = 1; testIdx < tests.size(); testIdx++ ) { - return std::numeric_limits::min(); // smallest positive normal number + addComparison( TierOf( tests[testIdx] ), tests[testIdx].testFunc ); } - - T above = std::nextafter( x, std::numeric_limits::infinity() ); - - // The largest finite value has nothing finite above it, use the gap below instead - return std::isfinite( above ) ? above - x : x - std::nextafter( x, T( 0 ) ); } - template - static LaneResult CompareLane( const TestData& test, bool relaxed, T expected, T actual, TierResult& result ) + void Run( const TestOptions& options ) { - if constexpr( std::is_floating_point_v ) + switch( mReference.returnType ) { - bool bitsEqual = ToBits( expected ) == ToBits( actual ); - bool bothNaN = std::isnan( expected ) && std::isnan( actual ); - - if( bitsEqual || ( bothNaN && !test.bitExact ) ) - { - return LaneResult::Equal; - } + case TestData::ReturnType::boolean: + return RunCalls( options ); - // Relaxed results are compared by value, so +0 and -0 are equal - if( relaxed && !test.bitExact ) - { - if( expected == actual ) - { - return LaneResult::Equal; - } + case TestData::ReturnType::f32: + return RunCalls( options ); - auto [allowedUlps, allowedAbs] = test.RelaxedTolerance(); - double absoluteDif = std::abs( static_cast( expected ) - static_cast( actual ) ); - double relativeDif = absoluteDif / DeltaUnit( expected ); + case TestData::ReturnType::f64: + return RunCalls( options ); - if( relativeDif <= allowedUlps || absoluteDif <= allowedAbs ) - { - // Track what each tolerance had to cover on its own - if( std::isfinite( absoluteDif ) ) - { - if( absoluteDif > allowedAbs ) - { - result.maxUlps = std::max( result.maxUlps, relativeDif ); - } - if( relativeDif > allowedUlps ) - { - result.maxAbs = std::max( result.maxAbs, absoluteDif ); - } - } - return LaneResult::Equal; - } - return LaneResult::Different; - } + case TestData::ReturnType::i32: + return RunCalls( options ); - return expected == 0 && actual == 0 ? LaneResult::ZeroSign : LaneResult::Different; - } - else - { - return expected == actual ? LaneResult::Equal : LaneResult::Different; + case TestData::ReturnType::i64: + return RunCalls( options ); } } - template - static std::string Describe( T expected, T actual ) + void Report( const TestOptions& options, TestSummary& summary ) const { - std::ostringstream sstr; - sstr << std::setprecision( std::is_floating_point_v ? std::numeric_limits::max_digits10 : 16 ) << std::boolalpha; - sstr << "Expected \"" << expected << "\" Actual \"" << actual << "\""; + std::vector failedTiers; + bool comparedAny = false; - if constexpr( std::is_floating_point_v ) + for( const Tier& tier : mTiers ) { - T absoluteDif = std::abs( expected - actual ); - sstr << " Diff \"" << absoluteDif << "\""; + comparedAny |= tier.compared > 0; - if( std::isfinite( absoluteDif ) && absoluteDif != 0 ) + if( tier.failures ) { - sstr << " (" << absoluteDif / DeltaUnit( expected ) << " ULP)"; + failedTiers.push_back( tier.name ); } - if( ToBits( expected ) != ToBits( actual ) && ( absoluteDif == 0 || std::isnan( absoluteDif ) ) ) + if( options.stats && tier.kind == Tiers::Relaxed && ( tier.maxUlps > 0 || tier.maxAbs > 0 ) ) { - sstr << std::hex << " (bits 0x" << ToBits( expected ) << " vs 0x" << ToBits( actual ) << ")" << std::dec; + Tolerance tolerance = mReference.RelaxedTolerance( tier.featureSet ); + std::cout << " Stats " << mWidth << mTestName << " " << tier.name << ": max " << tier.maxUlps << " / " << tolerance.ulps << " ULP, " + << tier.maxAbs << " / " << tolerance.absolute << " abs" << std::endl; } } - else if constexpr( std::is_same_v ) - { - sstr << " Diff \"" << static_cast( expected ) - static_cast( actual ) << "\""; - } - else if constexpr( std::is_same_v ) - { - sstr << std::hex << " (bits 0x" << static_cast( expected ) << " vs 0x" << static_cast( actual ) << ")" << std::dec; - } - return sstr.str(); - } - // Compares one call's outputs, returns true if the tier failed. Known differences mark their entry in knownSeen and set sawKnown - template - static bool CompareCall( std::string_view testName, const TestData& reference, TierResult& result, size_t outputCount, const TestInputs& inputs, size_t inIdx, - const void* expectedResults, const void* actualResults, std::vector>& knownSeen, std::string& details, bool& sawKnown ) - { - const T* expected = reinterpret_cast( expectedResults ); - const T* actual = reinterpret_cast( actualResults ); - const TestData& test = result.data ? *result.data : reference; - const auto& knownFailures = reference.knownFailures; - bool failed = false; - - for( size_t idx = 0; idx < outputCount; idx++ ) + for( size_t knownIdx = 0; knownIdx < mReference.knownFailures.size(); knownIdx++ ) { - LaneResult lane = CompareLane( test, result.tier.relaxed, expected[idx], actual[idx], result ); - - if( lane == LaneResult::Equal ) - { - continue; - } - - // Lane inputs for known failure conditions - TestData::LaneArgs laneArgs = {}; - for( size_t argIdx = 0; argIdx < reference.argCount; argIdx++ ) - { - if( reference.argTypes[argIdx] == TestArgType::Float ) - { - laneArgs.floats[argIdx] = inputs.floats[argIdx][inIdx + idx]; - } - else if( reference.argTypes[argIdx] == TestArgType::Double ) - { - laneArgs.doubles[argIdx] = inputs.doubles[argIdx][inIdx + idx]; - } - } + const KnownFailure& knownFailure = mReference.knownFailures[knownIdx]; + std::string reason( knownFailure.reason ); - bool known = false; - for( size_t knownIdx = 0; knownIdx < knownFailures.size(); knownIdx++ ) + if( !mKnownSeen[knownIdx].empty() ) { - const TestData::KnownFailureInfo& knownFailure = knownFailures[knownIdx]; - - if( knownFailure.Covers( result.tier ) && ( !knownFailure.zeroSignOnly || lane == LaneResult::ZeroSign ) && - ( !knownFailure.inputCondition || knownFailure.inputCondition( laneArgs ) ) ) - { - known = true; - sawKnown = true; - - if( std::find( knownSeen[knownIdx].begin(), knownSeen[knownIdx].end(), result.name ) == knownSeen[knownIdx].end() ) - { - knownSeen[knownIdx].push_back( result.name ); - } - break; - } + summary.knownFailures.push_back( mWidth + std::string( mTestName ) + " (" + JoinNames( mKnownSeen[knownIdx] ) + "): " + reason ); } - - if( !known ) + else if( std::any_of( mTiers.begin(), mTiers.end(), [&]( const Tier& tier ) { return tier.compared && knownFailure.Covers( tier ); } ) ) { - details += "idx " + std::to_string( idx ) + ": " + std::string( testName ) + ( result.tier.relaxed ? " RELAXED " : " " ) + Describe( expected[idx], actual[idx] ) + "\n"; - failed = true; + summary.knownNotSeen.push_back( mWidth + std::string( mTestName ) + ": " + reason ); } } - return failed; - } - static bool CompareOutputs( std::string_view testName, const TestData& reference, TierResult& result, size_t outputCount, const TestInputs& inputs, size_t inIdx, - const void* expected, const void* actual, std::vector>& knownSeen, std::string& details, bool& sawKnown ) - { - switch( reference.returnType ) + if( !comparedAny ) { - case TestData::ReturnType::boolean: - return CompareCall( testName, reference, result, outputCount, inputs, inIdx, expected, actual, knownSeen, details, sawKnown ); - - case TestData::ReturnType::f32: - return CompareCall( testName, reference, result, outputCount, inputs, inIdx, expected, actual, knownSeen, details, sawKnown ); - - case TestData::ReturnType::f64: - return CompareCall( testName, reference, result, outputCount, inputs, inIdx, expected, actual, knownSeen, details, sawKnown ); - - case TestData::ReturnType::i32: - return CompareCall( testName, reference, result, outputCount, inputs, inIdx, expected, actual, knownSeen, details, sawKnown ); - - case TestData::ReturnType::i64: - return CompareCall( testName, reference, result, outputCount, inputs, inIdx, expected, actual, knownSeen, details, sawKnown ); + std::cout << " Warning: no tiers to compare with the reference" << std::endl; } - return true; - } - - static std::string TierName( const TestTier& tier ) - { - if( tier.oracle ) + if( !failedTiers.empty() ) { - return "SCALAR vs ORACLE"; + summary.failed.push_back( mWidth + std::string( mTestName ) + " (" + JoinNames( failedTiers ) + ")" ); } - return FeatureSetName( tier.featureSet ) + ( tier.relaxed ? " RELAXED" : "" ); - } - - static void CheckTestsMatch( std::string_view testName, const std::vector& tests ) - { - const TestData& reference = tests[0]; - - for( size_t testIdx = 1; testIdx < tests.size(); testIdx++ ) + else { - const TestData& test = tests[testIdx]; - - if( test.featureSet == FastSIMD::FeatureSet::SCALAR && !test.relaxed ) - { - throw std::runtime_error( "Multiple tests with same name: " + std::string( testName ) ); - } - if( test.returnType != reference.returnType || test.lanes != reference.lanes || test.argCount != reference.argCount || test.knownFailures.size() != reference.knownFailures.size() || - !std::equal( test.argTypes, test.argTypes + kMaxTestArgs, reference.argTypes ) ) - { - throw std::runtime_error( "Tests do not match: " + std::string( testName ) + " (" + TierName( { test.featureSet, test.relaxed, false } ) + ")" ); - } + summary.passed++; } } - static void DoTest( std::string_view testName, std::vector& tests, const TestOptions& options, TestSummary& summary ) +private: + template + void RunCalls( const TestOptions& options ) { - std::cout << "Testing: " << testName << std::endl; - - const TestData& reference = tests[0]; - CheckTestsMatch( testName, tests ); - ValidateInputs( testName, reference ); - - InputGenerator generator( options.seed, testName, reference, reference.lanes ); - TestInputs inputs; + InputGenerator generator( options.seed, mTestName, mReference ); // No argument tests always produce the same output - size_t inputCount = reference.argCount ? generator.SpecialCount() + static_cast( options.count ) : reference.lanes; - - std::vector tiers; - if( reference.oracleFunc ) - { - TierResult oracle; - oracle.tier = { FastSIMD::FeatureSet::SCALAR, false, true }; - oracle.name = TierName( oracle.tier ); - tiers.push_back( oracle ); - } - for( size_t testIdx = 1; testIdx < tests.size(); testIdx++ ) - { - TierResult tier; - tier.tier = { tests[testIdx].featureSet, tests[testIdx].relaxed, false }; - tier.name = TierName( tier.tier ); - tier.data = &tests[testIdx]; - tiers.push_back( tier ); - } - - // A tier stops being checked after kMaxFailuresPerTier failing calls, or known failing calls if a known failure covers every difference - auto stopped = [&]( const TierResult& tier ) - { - if( tier.failures >= kMaxFailuresPerTier ) - { - return true; - } - bool coversAll = std::any_of( reference.knownFailures.begin(), reference.knownFailures.end(), - [&]( const TestData::KnownFailureInfo& known ) { return known.CoversAnyDifference() && known.Covers( tier.tier ); } ); - return coversAll && tier.knownCalls >= kMaxFailuresPerTier; - }; + size_t inputCount = mReference.argCount ? generator.SpecialCount() + static_cast( options.count ) : mReference.lanes; - std::vector> knownSeen( reference.knownFailures.size() ); - std::vector referenceResults( kMaxTestResultBytes ), testResults( kMaxTestResultBytes ); + auto allStopped = [this]() { return std::all_of( mTiers.begin(), mTiers.end(), []( const Tier& tier ) { return tier.Stopped(); } ); }; + std::vector referenceResults( kMaxTestResultBytes ), tierResults( kMaxTestResultBytes ); + const T* reference = reinterpret_cast( referenceResults.data() ); + const T* results = reinterpret_cast( tierResults.data() ); - for( size_t chunkStart = 0; chunkStart < inputCount && !tiers.empty(); chunkStart += kChunkSize ) + for( size_t chunkStart = 0; chunkStart < inputCount && !allStopped(); chunkStart += kChunkSize ) { - if( std::all_of( tiers.begin(), tiers.end(), stopped ) ) - { - break; - } - - generator.Generate( chunkStart, inputs ); + const TestInputs& inputs = generator.Generate( chunkStart ); size_t chunkCount = std::min( kChunkSize, inputCount - chunkStart ); - for( size_t idx = 0; idx < chunkCount; idx += reference.lanes ) + for( size_t idx = 0; idx < chunkCount; idx += mReference.lanes ) { std::memset( referenceResults.data(), 0xCD, kMaxTestResultBytes ); - size_t referenceCount = reference.testFunc( referenceResults.data(), idx, inputs ); + size_t referenceCount = mReference.testFunc( referenceResults.data(), idx, inputs ); - for( TierResult& tier : tiers ) + for( Tier& tier : mTiers ) { - if( stopped( tier ) ) + if( tier.Stopped() ) { continue; } - std::memset( testResults.data(), 0xAB, kMaxTestResultBytes ); - size_t outputCount = tier.data ? tier.data->testFunc( testResults.data(), idx, inputs ) : reference.oracleFunc( testResults.data(), idx, inputs ); + std::memset( tierResults.data(), 0xAB, kMaxTestResultBytes ); + size_t outputCount = tier.testFunc( tierResults.data(), idx, inputs ); - std::string details; - bool failed; - bool sawKnown = false; - - if( outputCount != referenceCount ) - { - details = "Output count " + std::to_string( outputCount ) + ", reference " + std::to_string( referenceCount ) + "\n"; - failed = true; - } - else if( tier.data ) - { - failed = CompareOutputs( testName, reference, tier, outputCount, inputs, idx, referenceResults.data(), testResults.data(), knownSeen, details, sawKnown ); - } - else - { - // The oracle is the expected value, the SCALAR reference is checked against it - failed = CompareOutputs( testName, reference, tier, outputCount, inputs, idx, testResults.data(), referenceResults.data(), knownSeen, details, sawKnown ); - } + // The oracle is the expected value, the SCALAR reference is checked against it + bool oracle = tier.kind == Tiers::Oracle; + CallResult call = outputCount != referenceCount + ? CallResult{ CallOutcome::Fail, "Output count " + std::to_string( outputCount ) + ", reference " + std::to_string( referenceCount ) + "\n" } + : CompareCall( tier, inputs, idx, outputCount, oracle ? results : reference, oracle ? reference : results ); tier.compared++; - if( failed ) + if( call.outcome == CallOutcome::Fail ) { if( !tier.failures ) { - std::cerr << std::setprecision( 16 ) << "--- " << RegisterBytes * 8 << "bit " << testName << " " << tier.name << " FAILED ---" << std::endl; - std::cerr << details; - std::cerr << "Inputs (index " << chunkStart + idx << "): " << reference.inputsFunc( idx, inputs ) << std::endl; + std::cerr << std::setprecision( 16 ) << "--- " << mWidth << mTestName << " " << tier.name << " FAILED ---" << std::endl; + std::cerr << call.details; + std::cerr << "Inputs (index " << chunkStart + idx << "): " << mReference.inputsFunc( idx, inputs ) << std::endl; } tier.failures++; } - else if( sawKnown ) + else if( call.outcome == CallOutcome::Known ) { tier.knownCalls++; } } } } + } - std::string width = std::to_string( RegisterBytes * 8 ) + "bit "; - std::vector failedTiers; - bool comparedAny = false; + template + CallResult CompareCall( Tier& tier, const TestInputs& inputs, size_t inIdx, size_t outputCount, const T* expected, const T* actual ) + { + const std::vector& knownFailures = mReference.knownFailures; + CallResult call; - for( const TierResult& tier : tiers ) + for( size_t idx = 0; idx < outputCount; idx++ ) { - comparedAny |= tier.compared > 0; + LaneComparison lane = CompareLane( mReference, tier, expected[idx], actual[idx] ); + tier.maxUlps = std::max( tier.maxUlps, lane.ulpsCovered ); + tier.maxAbs = std::max( tier.maxAbs, lane.absCovered ); - if( tier.failures ) - { - failedTiers.push_back( tier.name ); - } - if( options.stats && tier.data && tier.tier.relaxed && ( tier.maxUlps > 0 || tier.maxAbs > 0 ) ) + if( lane.result == LaneResult::Equal ) { - auto [allowedUlps, allowedAbs] = tier.data->RelaxedTolerance(); - std::cout << " Stats " << width << testName << " " << tier.name << ": max " << tier.maxUlps << " / " << allowedUlps << " ULP, " - << tier.maxAbs << " / " << allowedAbs << " abs" << std::endl; + continue; } - } - auto join = []( const std::vector& names ) - { - std::string joined; - for( size_t i = 0; i < names.size(); i++ ) + LaneArgs laneArgs = mReference.laneArgsFunc( inIdx, idx, inputs ); + auto known = std::find_if( knownFailures.begin(), knownFailures.end(), [&]( const KnownFailure& knownFailure ) { - joined += ( i ? ", " : "" ) + names[i]; - } - return joined; - }; + return knownFailure.Matches( tier, lane.result == LaneResult::ZeroSign, laneArgs ); + } ); - for( size_t knownIdx = 0; knownIdx < reference.knownFailures.size(); knownIdx++ ) - { - const TestData::KnownFailureInfo& knownFailure = reference.knownFailures[knownIdx]; - std::string reason = knownFailure.reason ? knownFailure.reason : "(no reason given)"; - - if( !knownSeen[knownIdx].empty() ) + if( known != knownFailures.end() ) { - summary.knownFailures.push_back( width + std::string( testName ) + " (" + join( knownSeen[knownIdx] ) + "): " + reason ); + std::vector& seen = mKnownSeen[known - knownFailures.begin()]; + + if( std::find( seen.begin(), seen.end(), tier.name ) == seen.end() ) + { + seen.push_back( tier.name ); + } + call.outcome = std::max( call.outcome, CallOutcome::Known ); } - else if( std::any_of( tiers.begin(), tiers.end(), [&]( const TierResult& tier ) { return tier.compared && knownFailure.Covers( tier.tier ); } ) ) + else { - summary.knownNotSeen.push_back( width + std::string( testName ) + ": " + reason ); + call.details += "idx " + std::to_string( idx ) + ": " + std::string( mTestName ) + ( tier.kind == Tiers::Relaxed ? " RELAXED " : " " ) + Describe( expected[idx], actual[idx] ) + "\n"; + call.outcome = CallOutcome::Fail; } } + return call; + } - if( !comparedAny ) - { - std::cout << " Warning: no tiers to compare with the reference" << std::endl; - } + std::string mWidth; + std::string_view mTestName; + const TestData& mReference; + std::vector mTiers; + std::vector> mKnownSeen; // Tiers each known failure was seen on +}; - if( !failedTiers.empty() ) - { - summary.failed.push_back( width + std::string( testName ) + " (" + join( failedTiers ) + ")" ); - } - else - { - summary.passed++; - } - } +static void DoTest( size_t registerBytes, std::string_view testName, const std::vector& tests, const TestOptions& options, TestSummary& summary ) +{ + std::cout << "Testing: " << testName << std::endl; - static void Run( const TestOptions& options, TestSummary& summary ) - { - if( options.registerBytes && options.registerBytes != RegisterBytes ) - { - return; - } + CheckTests( testName, tests ); + ValidateInputs( testName, tests[0] ); - std::cout << "Starting Tests - Register Size: " << RegisterBytes * 8 << " (" << RegisterBytes << "b)" << std::endl; + TestRun run( registerBytes, testName, tests ); + run.Run( options ); + run.Report( options, summary ); +} - TestSet testSet = TestOrganiser::GetSet( options.maxFeatureSet ); +template +static void RunRegisterSize( const TestOptions& options, TestSummary& summary ) +{ + if( options.registerBytes && options.registerBytes != RegisterBytes ) + { + return; + } - for( auto& test : testSet ) + std::cout << "Starting Tests - Register Size: " << RegisterBytes * 8 << " (" << RegisterBytes << "b)" << std::endl; + + for( const auto& [testName, tests] : CollectTests( options.maxFeatureSet ) ) + { + if( testName.find( options.filter ) != std::string_view::npos ) { - if( test.first.find( options.filter ) != std::string_view::npos ) - { - DoTest( test.first, test.second, options, summary ); - } + DoTest( RegisterBytes, testName, tests, options, summary ); } } -}; +} template static void RunTests( const TestOptions& options, TestSummary& summary, std::index_sequence ) { - ( TestRunner::Run( options, summary ), ... ); + ( RunRegisterSize( options, summary ), ... ); } static void PrintUsage( const char* exe ) @@ -1232,7 +592,7 @@ static void PrintUsage( const char* exe ) std::cout << " " << bytes; } std::cout << "\n --max-feature-set Only run feature sets up to this one, to match a less capable CPU:"; - for( FastSIMD::FeatureSet featureSet : CompiledFeatureSets() ) + for( FastSIMD::FeatureSet featureSet : TestFeatureSets::AsArray ) { std::cout << " " << FeatureSetName( featureSet ); } @@ -1316,10 +676,10 @@ static bool ParseArgs( int argc, char** argv, TestOptions& options ) } else if( arg == "--max-feature-set" && valid ) { - std::vector featureSets = CompiledFeatureSets(); - auto find = std::find_if( featureSets.begin(), featureSets.end(), [value]( FastSIMD::FeatureSet featureSet ) { return NormaliseName( FeatureSetName( featureSet ) ) == NormaliseName( value ); } ); + const auto& featureSets = TestFeatureSets::AsArray; + auto find = std::find_if( std::begin( featureSets ), std::end( featureSets ), [value]( FastSIMD::FeatureSet featureSet ) { return NormaliseName( FeatureSetName( featureSet ) ) == NormaliseName( value ); } ); - valid = find != featureSets.end(); + valid = find != std::end( featureSets ); options.maxFeatureSet = valid ? *find : options.maxFeatureSet; } else @@ -1348,7 +708,7 @@ int main( int argc, char** argv ) std::cout << "Seed: " << options.seed << ", Random inputs per test: " << options.count << std::endl; std::cout << "CPU max feature set: " << FeatureSetName( FastSIMD::DetectCpuMaxFeatureSet() ) << ", Compiled:"; - for( FastSIMD::FeatureSet featureSet : CompiledFeatureSets() ) + for( FastSIMD::FeatureSet featureSet : TestFeatureSets::AsArray ) { std::cout << " " << FeatureSetName( featureSet ); } diff --git a/tests/test.h b/tests/test.h index 7642ce9..c2c35e8 100644 --- a/tests/test.h +++ b/tests/test.h @@ -2,9 +2,12 @@ #include #include #include +#include #include #include +#include #include +#include #include #include #include @@ -18,29 +21,87 @@ constexpr size_t kMaxTestResultBytes = kMaxTestLanes * sizeof( double ); constexpr size_t kMaxTestArgs = 3; -// Input values for each test argument, indexed by element +template +struct FloatTraits; + +template<> +struct FloatTraits +{ + using Bits = uint32_t; + static constexpr int kSignificandBits = 24; +}; + +template<> +struct FloatTraits +{ + using Bits = uint64_t; + static constexpr int kSignificandBits = 53; +}; + +template +typename FloatTraits::Bits ToBits( T f ) +{ + typename FloatTraits::Bits bits; + std::memcpy( &bits, &f, sizeof( f ) ); + return bits; +} + +template +T FromBits( typename FloatTraits::Bits bits ) +{ + T f; + std::memcpy( &f, &bits, sizeof( f ) ); + return f; +} + +// The type an argument's input values are stored as, one buffer per argument +enum class TestElement : uint8_t +{ + Int32, Float, Double +}; + +struct TestArgType +{ + TestElement element; + bool mask; // Lanes are 0 or 1 in the element type, from lane patterns rather than an input spec, compared != 0 to make the mask + + bool operator==( const TestArgType& other ) const { return element == other.element && mask == other.mask; } +}; + +// One pointer per argument to its values for the current chunk, indexed by input index and stored as the argument's TestElement struct TestInputs { - const int32_t* ints[kMaxTestArgs]; - const float* floats[kMaxTestArgs]; - const double* doubles[kMaxTestArgs]; - const int32_t* masks[kMaxTestArgs]; // 0 or 1 per lane - const float* maskFloats[kMaxTestArgs]; // Same lanes as masks, 0.0f or 1.0f - const double* maskDoubles[kMaxTestArgs]; // Same lanes as masks, 0.0 or 1.0 + const void* values[kMaxTestArgs]; + + template + const T* Get( size_t argIdx ) const + { + return static_cast( values[argIdx] ); + } }; -enum class TestArgType : uint8_t +// A lane's input values, for known failure conditions. Arguments of other types are 0 +struct LaneArgs { - Int, Float, Double, Mask + int32_t ints[kMaxTestArgs]; // Masks are 0 or 1 + float floats[kMaxTestArgs]; + double doubles[kMaxTestArgs]; + + void Set( size_t argIdx, int32_t value ) { ints[argIdx] = value; } + void Set( size_t argIdx, bool value ) { ints[argIdx] = value; } + void Set( size_t argIdx, float value ) { floats[argIdx] = value; } + void Set( size_t argIdx, double value ) { doubles[argIdx] = value; } }; -// How input values are generated for a test argument +// How input values are generated for a test argument, the default spec is every value of the argument's type // Every test first runs all combinations of the special values for its inputs, then random values struct InputSpec { enum class Kind { - Any, // Values of every magnitude + Any, // Values of every magnitude, +-inf and NaNs + NoNaN, // Values of every magnitude and +-inf + Finite, // Values of every magnitude Range, // Values within [lo, hi], half uniform, half log scaled Magnitude, // Values with magnitude within [lo, hi] log scaled, either sign, lo > 0 }; @@ -48,14 +109,11 @@ struct InputSpec Kind kind = Kind::Any; double lo = 0; double hi = 0; - bool inf = false; // Any only, include +-inf - bool nan = false; // Any only, include NaNs int significandBits = 0; // Floats are truncated to this many significand bits, 0 for all of them std::vector specials; // Extra special values, only used when inside the spec's domain - static InputSpec Any() { return {}; } - static InputSpec AnyInf() { InputSpec spec; spec.inf = true; return spec; } - static InputSpec AnyNonFinite() { InputSpec spec; spec.inf = spec.nan = true; return spec; } + static InputSpec NoNaN() { InputSpec spec; spec.kind = Kind::NoNaN; return spec; } + static InputSpec Finite() { InputSpec spec; spec.kind = Kind::Finite; return spec; } static InputSpec Range( double lo, double hi ) { InputSpec spec; spec.kind = Kind::Range; spec.lo = lo; spec.hi = hi; return spec; } static InputSpec Magnitude( double lo, double hi ) { InputSpec spec; spec.kind = Kind::Magnitude; spec.lo = lo; spec.hi = hi; return spec; } @@ -68,83 +126,111 @@ struct InputSpec spec.specials.insert( spec.specials.end(), values.begin(), values.end() ); return spec; } + + bool IsDefault() const + { + return kind == Kind::Any && significandBits == 0 && specials.empty(); + } }; using TestFunction = size_t ( void* outResult, size_t inIdx, const TestInputs& inputs ); using InputsFunction = std::string ( size_t inIdx, const TestInputs& inputs ); +using LaneArgsFunction = LaneArgs ( size_t inIdx, size_t lane, const TestInputs& inputs ); + +// Kinds of tier, each tier is one kind and known failures are scoped to a set of them +enum class Tiers : uint8_t +{ + None = 0, + NonRelaxed = 1 << 0, // A non relaxed feature set against the SCALAR reference, bit for bit + Relaxed = 1 << 1, // A relaxed feature set against the SCALAR reference, within the test's tolerance + Oracle = 1 << 2, // The SCALAR reference against the test's oracle + All = NonRelaxed | Relaxed, // Every tier compared with the reference, not the oracle +}; + +constexpr Tiers operator|( Tiers a, Tiers b ) +{ + return static_cast( static_cast( a ) | static_cast( b ) ); +} -// Identifies what a result is checked against: a tier against the SCALAR reference, or the reference against the oracle +constexpr bool operator&( Tiers a, Tiers b ) +{ + return static_cast( a ) & static_cast( b ); +} + +// What a result is checked against: a feature set against the SCALAR reference, or the reference against the oracle struct TestTier { FastSIMD::FeatureSet featureSet; - bool relaxed; - bool oracle; + Tiers kind; // One of NonRelaxed, Relaxed or Oracle }; -struct TestData +using LaneInputCondition = bool ( * )( const LaneArgs& args ); + +// Differences on tiers in scope are reported as known failures and don't fail the run, scoped by tier kind and feature set range +struct KnownFailure { - enum class ReturnType + std::string_view reason; + Tiers tiers; + FastSIMD::FeatureSet minFeatureSet; + FastSIMD::FeatureSet maxFeatureSet; + bool zeroSignOnly = false; // Only results differing in the sign of zero are known, any other difference still fails + LaneInputCondition inputCondition = nullptr; // Only lanes with matching inputs are known, null for every lane + + constexpr explicit KnownFailure( std::string_view knownReason, Tiers knownTiers = Tiers::All, + FastSIMD::FeatureSet minKnown = FastSIMD::FeatureSet::Invalid, + FastSIMD::FeatureSet maxKnown = FastSIMD::FeatureSet::Max ) : + reason( knownReason ), tiers( knownTiers ), minFeatureSet( minKnown ), maxFeatureSet( maxKnown ) {} + + // Only results that differ in the sign of zero are known + constexpr KnownFailure ZeroSignOnly() const { - boolean, f32, f64, i32, i64 // i64 holds 64bit mask lanes - }; + KnownFailure known = *this; + known.zeroSignOnly = true; + return known; + } - enum class Tiers + // Only lanes whose inputs match the condition are known, other inputs are still checked + constexpr KnownFailure ForInputs( LaneInputCondition condition ) const { - All, // Every tier compared with the reference, not the oracle - Relaxed, - NonRelaxed, - ScalarRelaxed, // Relaxed scalar uses std:: maths functions, so differs where FastSIMD is less accurate - Oracle, // The SCALAR reference against the test's oracle - }; + KnownFailure known = *this; + known.inputCondition = condition; + return known; + } - // A lane's input values, arguments of other types are 0 - struct LaneArgs + bool Covers( const TestTier& tier ) const { - float floats[kMaxTestArgs]; - double doubles[kMaxTestArgs]; - }; + return ( tiers & tier.kind ) && tier.featureSet >= minFeatureSet && tier.featureSet <= maxFeatureSet; + } - using LaneInputCondition = bool ( * )( const LaneArgs& args ); - - struct KnownFailureInfo - { - const char* reason; - bool zeroSignOnly; // Only results differing in the sign of zero are known, any other difference still fails - LaneInputCondition inputCondition; // Only lanes with matching inputs are known, null for every lane - Tiers tiers; - FastSIMD::FeatureSet minFeatureSet; - FastSIMD::FeatureSet maxFeatureSet; - - bool Covers( const TestTier& tier ) const - { - bool tiersMatch = false; - - switch( tiers ) - { - case Tiers::All: - tiersMatch = !tier.oracle; - break; - case Tiers::Relaxed: - tiersMatch = !tier.oracle && tier.relaxed; - break; - case Tiers::NonRelaxed: - tiersMatch = !tier.oracle && !tier.relaxed; - break; - case Tiers::ScalarRelaxed: - tiersMatch = !tier.oracle && tier.relaxed && tier.featureSet == FastSIMD::FeatureSet::SCALAR; - break; - case Tiers::Oracle: - return tier.oracle; - } - - return tiersMatch && tier.featureSet >= minFeatureSet && tier.featureSet <= maxFeatureSet; - } - - // Any difference on a covered tier is known, so comparing more of its calls can't fail - bool CoversAnyDifference() const - { - return !zeroSignOnly && !inputCondition; - } + // Any difference on a covered tier is known, so comparing more of its calls can't fail + bool CoversAnyDifference() const + { + return !zeroSignOnly && !inputCondition; + } + + bool Matches( const TestTier& tier, bool zeroSignDifference, const LaneArgs& args ) const + { + return Covers( tier ) && ( !zeroSignOnly || zeroSignDifference ) && ( !inputCondition || inputCondition( args ) ); + } +}; + +// Allowed error for relaxed results +struct Tolerance +{ + double ulps = 0; + double absolute = 0; // For results near 0 + + bool operator==( const Tolerance& other ) const { return ulps == other.ulps && absolute == other.absolute; } + bool operator!=( const Tolerance& other ) const { return !( *this == other ); } +}; + +// A test as registered by one feature set, plain or relaxed +// Only the SCALAR non relaxed registration's settings (inputs, tolerances, known failures, oracle) are used, every tier must register the same ones +struct TestData +{ + enum class ReturnType + { + boolean, f32, f64, i32, i64 // i64 holds 64bit mask lanes }; FastSIMD::FeatureSet featureSet; @@ -154,15 +240,14 @@ struct TestData size_t argCount = 0; TestArgType argTypes[kMaxTestArgs] = {}; InputSpec inputs[kMaxTestArgs]; - double relaxedAccuracy = 0; // Allowed error in ULPs for relaxed feature sets - double relaxedAbsAccuracy = 0; // Allowed absolute error for relaxed feature sets, for results near 0 - double scalarRelaxedAccuracy = -1; // As above for relaxed SCALAR, negative uses the relaxed accuracy - double scalarRelaxedAbsAccuracy = 0; + Tolerance relaxedTolerance; + std::optional scalarRelaxedTolerance; // Relaxed SCALAR, when it differs from the relaxed tolerance bool bitExact = false; // Float results must match bitwise on every tier, including NaN payloads - std::vector knownFailures; + std::vector knownFailures; std::function testFunc; std::function oracleFunc; // Optional plain C++ version of the test, the SCALAR reference must match it exactly std::function inputsFunc; + std::function laneArgsFunc; template TestData& Inputs( SPECs... specs ) @@ -175,8 +260,7 @@ struct TestData TestData& RelaxedAccuracy( double ulps, double absolute = 0 ) { - relaxedAccuracy = ulps; - relaxedAbsAccuracy = absolute; + relaxedTolerance = { ulps, absolute }; return *this; } @@ -184,19 +268,13 @@ struct TestData // Relaxed SIMD tolerances cover estimate instructions, which would hide FS accuracy regressions TestData& ScalarRelaxedAccuracy( double ulps, double absolute = 0 ) { - scalarRelaxedAccuracy = ulps; - scalarRelaxedAbsAccuracy = absolute; + scalarRelaxedTolerance = Tolerance{ ulps, absolute }; return *this; } - // Allowed { ULPs, absolute } error when this tier is relaxed - std::pair RelaxedTolerance() const + Tolerance RelaxedTolerance( FastSIMD::FeatureSet tierFeatureSet ) const { - if( featureSet == FastSIMD::FeatureSet::SCALAR && scalarRelaxedAccuracy >= 0 ) - { - return { scalarRelaxedAccuracy, scalarRelaxedAbsAccuracy }; - } - return { relaxedAccuracy, relaxedAbsAccuracy }; + return tierFeatureSet == FastSIMD::FeatureSet::SCALAR && scalarRelaxedTolerance ? *scalarRelaxedTolerance : relaxedTolerance; } TestData& BitExact() @@ -205,31 +283,12 @@ struct TestData return *this; } - // Differences on tiers in scope are reported as known failures and don't fail the run - // A reason is required on every tier's TestData, scope it with tiers and the feature set range - TestData& KnownFailure( const char* reason, Tiers tiers = Tiers::All, - FastSIMD::FeatureSet minFeatureSet = FastSIMD::FeatureSet::Invalid, - FastSIMD::FeatureSet maxFeatureSet = FastSIMD::FeatureSet::Max ) - { - knownFailures.push_back( { reason, false, nullptr, tiers, minFeatureSet, maxFeatureSet } ); - return *this; - } - - // As KnownFailure, but only results that differ in the sign of zero are known - TestData& KnownZeroSign( const char* reason, Tiers tiers = Tiers::All, - FastSIMD::FeatureSet minFeatureSet = FastSIMD::FeatureSet::Invalid, - FastSIMD::FeatureSet maxFeatureSet = FastSIMD::FeatureSet::Max ) - { - knownFailures.push_back( { reason, true, nullptr, tiers, minFeatureSet, maxFeatureSet } ); - return *this; - } - - // As KnownFailure, but only lanes whose inputs match the condition are known, other inputs are still checked - TestData& KnownFailureIf( const char* reason, LaneInputCondition condition, Tiers tiers = Tiers::All, - FastSIMD::FeatureSet minFeatureSet = FastSIMD::FeatureSet::Invalid, - FastSIMD::FeatureSet maxFeatureSet = FastSIMD::FeatureSet::Max ) + // Differences these cover are reported as known failures and don't fail the run + template + TestData& Known( const KNOWNs&... known ) { - knownFailures.push_back( { reason, false, condition, tiers, minFeatureSet, maxFeatureSet } ); + static_assert( ( std::is_same_v && ... ) ); + ( knownFailures.push_back( known ), ... ); return *this; } }; diff --git a/tests/test.inl b/tests/test.inl index b66e453..b6a205d 100644 --- a/tests/test.inl +++ b/tests/test.inl @@ -169,42 +169,27 @@ class FastSIMD::DispatchClass, SIMD> : publ { // No 64bit int registers, a double with every bit set keeps all the lane's bits through Masked static_assert( MN == 64 && MB, "Mask results are stored as 32 or 64bit lanes" ); - FS::Store( reinterpret_cast( outResult ), FS::Masked( result, FS::Register( AllBitsDouble() ) ) ); + FS::Store( reinterpret_cast( outResult ), FS::Masked( result, FS::Register( FromBits( ~uint64_t( 0 ) ) ) ) ); } return N; } - static double AllBitsDouble() - { - uint64_t bits = ~uint64_t( 0 ); - double value; - std::memcpy( &value, &bits, sizeof( value ) ); - return value; - } - template - static const T* InputValues( const TestInputs& inputs, size_t argIdx ) + static constexpr TestElement ElementOf() { - if constexpr( std::is_same_v ) - { - return inputs.floats[argIdx]; - } - else if constexpr( std::is_same_v ) - { - return inputs.doubles[argIdx]; - } - else - { - static_assert( std::is_same_v, "No test inputs for type" ); - return inputs.ints[argIdx]; - } + static_assert( std::is_same_v || std::is_same_v || std::is_same_v, "No test inputs for type" ); + return std::is_same_v ? TestElement::Float : std::is_same_v ? TestElement::Double : TestElement::Int32; } + // A mask argument's 0/1 lanes are stored as, and compared != 0 in: double for 64bit masks, float for float masks, int32 for int masks + template + using MaskElement = std::conditional_t>; + template struct ArgType { - static constexpr TestArgType Value = std::is_same_v ? TestArgType::Float : std::is_same_v ? TestArgType::Double : TestArgType::Int; + static constexpr TestArgType Value = { ElementOf(), false }; }; template @@ -213,7 +198,9 @@ class FastSIMD::DispatchClass, SIMD> : publ template struct ArgType, N, S>> { - static constexpr TestArgType Value = TestArgType::Mask; + static_assert( MN == 32 || ( MN == 64 && MB ), "Mask inputs are generated as 32bit lanes, or 64bit from doubles" ); + + static constexpr TestArgType Value = { ElementOf>(), true }; }; template @@ -221,7 +208,7 @@ class FastSIMD::DispatchClass, SIMD> : publ { static T Load( size_t inIdx, size_t argIdx, const TestInputs& inputs ) { - return InputValues( inputs, argIdx )[inIdx]; + return inputs.Get( argIdx )[inIdx]; } }; @@ -230,30 +217,18 @@ class FastSIMD::DispatchClass, SIMD> : publ { static FS::Register Load( size_t inIdx, size_t argIdx, const TestInputs& inputs ) { - return FS::Load( InputValues( inputs, argIdx ) + inIdx ); + return FS::Load( inputs.Get( argIdx ) + inIdx ); } }; - // Each mask lane is generated independently, then made into a mask with a compare of the matching type + // Each mask lane is generated independently, then compared != 0 in its stored type to make the mask template struct GenArg, N, S>> { - static_assert( MN == 32 || ( MN == 64 && MB ), "Mask inputs are generated as 32bit lanes, or 64bit from doubles" ); - static FS::Register, N, S> Load( size_t inIdx, size_t argIdx, const TestInputs& inputs ) { - if constexpr( MN == 64 ) - { - return FS::Load( inputs.maskDoubles[argIdx] + inIdx ) != FS::Register( 0.0 ); - } - else if constexpr( MB ) - { - return FS::Load( inputs.maskFloats[argIdx] + inIdx ) != FS::Register( 0.0f ); - } - else - { - return FS::Load( inputs.masks[argIdx] + inIdx ) != FS::Register( 0 ); - } + using Element = MaskElement; + return FS::Load( inputs.Get( argIdx ) + inIdx ) != FS::Register( Element( 0 ) ); } }; @@ -264,7 +239,7 @@ class FastSIMD::DispatchClass, SIMD> : publ // Scalar arguments are the same for every lane static T Get( const TestInputs& inputs, size_t argIdx, size_t inIdx, size_t ) { - return InputValues( inputs, argIdx )[inIdx]; + return inputs.Get( argIdx )[inIdx]; } }; @@ -273,7 +248,7 @@ class FastSIMD::DispatchClass, SIMD> : publ { static T Get( const TestInputs& inputs, size_t argIdx, size_t inIdx, size_t lane ) { - return InputValues( inputs, argIdx )[inIdx + lane]; + return inputs.Get( argIdx )[inIdx + lane]; } }; @@ -282,7 +257,7 @@ class FastSIMD::DispatchClass, SIMD> : publ { static bool Get( const TestInputs& inputs, size_t argIdx, size_t inIdx, size_t lane ) { - return inputs.masks[argIdx][inIdx + lane] != 0; + return inputs.Get>( argIdx )[inIdx + lane] != 0; } }; @@ -389,6 +364,15 @@ class FastSIMD::DispatchClass, SIMD> : publ }; } + // A lane's argument values as the oracle sees them, for known failure conditions + static LaneArgs GetLaneArgs( [[maybe_unused]] size_t inIdx, [[maybe_unused]] size_t lane, [[maybe_unused]] const TestInputs& inputs ) + { + LaneArgs args = {}; + [[maybe_unused]] size_t argIdx = 0; + ( ( args.Set( argIdx, OracleArg>::Get( inputs, argIdx, inIdx, lane ) ), argIdx++ ), ... ); + return args; + } + static std::function GetInputs() { return []( size_t inIdx, const TestInputs& inputs ) -> std::string @@ -427,6 +411,7 @@ class FastSIMD::DispatchClass, SIMD> : publ data.testFunc = TestFunctionFactory::Create( func ); data.inputsFunc = TestFunctionFactory::GetInputs(); + data.laneArgsFunc = TestFunctionFactory::GetLaneArgs; return testMap.emplace_back( name, data ).second; } @@ -440,21 +425,6 @@ class FastSIMD::DispatchClass, SIMD> : publ return data; } - // Oracle helpers - static uint32_t Bits( float f ) - { - uint32_t bits; - std::memcpy( &bits, &f, sizeof( bits ) ); - return bits; - } - - static uint64_t Bits( double d ) - { - uint64_t bits; - std::memcpy( &bits, &d, sizeof( bits ) ); - return bits; - } - // int32 arithmetic wraps on every tier, done in uint32 to avoid signed overflow static int32_t Wrap( uint32_t v ) { return static_cast( v ); } static uint32_t U( int32_t v ) { return static_cast( v ); } @@ -462,40 +432,80 @@ class FastSIMD::DispatchClass, SIMD> : publ // x + 0.0f as the masked add/decrement implementations do: -0 becomes +0, everything else is unchanged static float PlusZero( float a ) { return a == 0.0f ? 0.0f : a; } + using TestRegi32 = TestReg; + using TestRegf32 = TestReg; + using TestRegm32 = typename TestRegf32::MaskType; + using TestRegm32i = typename TestRegi32::MaskType; + using In = InputSpec; + + static constexpr double denormMin = std::numeric_limits::denorm_min(); + static constexpr double floatMax = std::numeric_limits::max(); + static constexpr double pi = 3.14159265358979323846; + + // Lane input conditions for known failures + static bool DenormalArg0( const LaneArgs& args ) { return std::fpclassify( args.floats[0] ) == FP_SUBNORMAL; } + static bool DenormalArg1( const LaneArgs& args ) { return std::fpclassify( args.floats[1] ) == FP_SUBNORMAL; } + static bool QuotientOverflows( const LaneArgs& args ) { return std::isfinite( args.floats[0] ) && args.floats[1] != 0 && std::isinf( args.floats[0] / args.floats[1] ); } + static bool X86ReciprocalInput( const LaneArgs& args ) { return std::fpclassify( args.floats[0] ) == FP_SUBNORMAL || std::abs( args.floats[0] ) >= 0x1.ffcp125f; } + static bool ArmEstimateInput( const LaneArgs& args ) { return args.floats[0] == 0 || std::isinf( args.floats[0] ) || std::fpclassify( args.floats[0] ) == FP_SUBNORMAL; } + + // Known library behaviour, each scoped to the tiers that show it + static constexpr KnownFailure sse2RoundZeroSign = KnownFailure( + "Below SSE4.1 rounding converts through int32 (cvtps2dq/cvttps2dq), which has no -0, so negative results that round to 0 are +0", + Tiers::NonRelaxed, FeatureSet::SSE, FeatureSet::SSSE3 ).ZeroSignOnly(); + static constexpr KnownFailure sinReduceZeroSign = KnownFailure( + "Sin_Reduce uses Trunc, see f32 trunc: Sin( -0 ) is -0 below SSE4.1 and +0 elsewhere", + Tiers::NonRelaxed, FeatureSet::SSE, FeatureSet::SSSE3 ).ZeroSignOnly(); + static constexpr KnownFailure neonDivide = KnownFailure( + "NEON (ARMv7) has no divide, division is a reciprocal estimate plus Newton-Raphson steps which isn't correctly rounded", + Tiers::NonRelaxed, FeatureSet::NEON, FeatureSet::NEON ); + // vrecpe overflows to inf for |x| < 2^-128, then vrecps( x, inf ) is -inf, so the Newton-Raphson steps give inf of the wrong sign, or NaN for 0 / x + static constexpr KnownFailure neonDivideDenormal = KnownFailure( + "NEON (ARMv7) divide by a denormal: the reciprocal estimate overflows to inf and the Newton-Raphson steps flip its sign, giving -inf for +inf and NaN for 0", + Tiers::Relaxed, FeatureSet::NEON, FeatureSet::NEON ).ForInputs( DenormalArg1 ); + // The refined reciprocal can be 1 ULP low, e.g. 1 / 0.99999994 becomes 1, so FLT_MAX / 0.99999994 is FLT_MAX instead of inf + static constexpr KnownFailure neonDivideOverflow = KnownFailure( + "NEON (ARMv7) divide near overflow: the Newton-Raphson reciprocal can be 1 ULP low, so quotients that overflow to +-inf can give +-FLT_MAX", + Tiers::Relaxed, FeatureSet::NEON, FeatureSet::NEON ).ForInputs( QuotientOverflows ); + // Generic masked add is a + Masked( m, b ), so masked off lanes add +0. AVX512 512bit uses a native masked add that leaves them unchanged + static constexpr KnownFailure avx512MaskedAddZeroSign = KnownFailure( + "AVX512 512bit masked add leaves masked off lanes unchanged, the generic a + Masked( m, b ) turns -0 into +0", + RegisterBytes >= 64 ? Tiers::NonRelaxed : Tiers::None, FeatureSet::AVX512, FeatureSet::AVX512 ).ZeroSignOnly(); + // Relaxed x86 uses rcpps/rsqrtps, AVX512 512bit registers use rcp14/rsqrt14 which handle denormals + static constexpr FastSIMD::FeatureSet x86EstimateMax = RegisterBytes >= 64 ? FeatureSet::AVX2 : FeatureSet::AVX512; + static constexpr KnownFailure x86EstimateDenormals = KnownFailure( + "Relaxed rcpps/rsqrtps estimates treat denormal inputs as 0 and return inf", + Tiers::Relaxed, FeatureSet::SSE, x86EstimateMax ).ForInputs( DenormalArg0 ); + // The Intel SDM allows rcpps to flush results just above FLT_MIN to 0, for |x| just below 2^126 + static constexpr KnownFailure x86ReciprocalEstimate = KnownFailure( + "Relaxed rcpps treats denormal inputs as 0 and returns inf, and may flush results near FLT_MIN to 0", + Tiers::Relaxed, FeatureSet::SSE, x86EstimateMax ).ForInputs( X86ReciprocalInput ); + // Relaxed ARM refines vrsqrte/vrecpe with one Newton-Raphson step, which breaks where the estimate is 0 or inf + static constexpr KnownFailure armEstimateSpecials = KnownFailure( + "Relaxed ARM estimate + Newton-Raphson step: InvSqrt( +-0 ) and InvSqrt( inf ) are NaN (0 * inf), Reciprocal of a denormal is inf of the wrong sign", + Tiers::Relaxed, FeatureSet::NEON, FeatureSet::AARCH64 ).ForInputs( ArmEstimateInput ); +#if defined( _MSC_VER ) && !defined( __clang__ ) && defined( _DEBUG ) + // The scalar reference is wrong, so every tier and the oracle comparison see it + static constexpr Tiers msvcDebugAbsTiers = Tiers::All | Tiers::Oracle; +#else + static constexpr Tiers msvcDebugAbsTiers = Tiers::None; +#endif + static constexpr KnownFailure msvcDebugAbs = KnownFailure( + "Unoptimised MSVC calls the CRT fabsf for std::abs in the scalar FS::Abs, which quiets signalling NaNs", msvcDebugAbsTiers ); + TestCollection RegisterTests() override { TestCollection tests; + RegisterMaskTests( tests ); + RegisterIntTests( tests ); + RegisterFloatTests( tests ); + RegisterFloatMathTests( tests ); + RegisterConvertTests( tests ); + return tests; + } - using TestRegi32 = TestReg; - using TestRegf32 = TestReg; - using TestRegm32 = typename TestRegf32::MaskType; - using TestRegm32i = typename TestRegi32::MaskType; - using In = InputSpec; - using Tiers = TestData::Tiers; - using LaneArgs = TestData::LaneArgs; - - constexpr double denormMin = std::numeric_limits::denorm_min(); - constexpr double floatMax = std::numeric_limits::max(); - constexpr double pi = 3.14159265358979323846; - - // Range reduction boundaries for trig - const In trigRange = In::Range( -4096, 4096 ).Specials( { pi / 4, pi / 2, pi, 3 * pi / 2, 2 * pi, 4 * pi, 100 * pi, 1000 * pi, - -pi / 4, -pi / 2, -pi, -3 * pi / 2, -2 * pi, -4 * pi, -100 * pi, -1000 * pi } ); - const In smallTrigRange = In::Range( -2 * pi, 2 * pi ).Specials( { pi / 4, pi / 2, pi, 3 * pi / 2, 2 * pi, - -pi / 4, -pi / 2, -pi, -3 * pi / 2, -2 * pi } ); - - // Known failures shared by several tests - const char* sse2RoundZeroSign = "Below SSE4.1 rounding converts through int32 (cvtps2dq/cvttps2dq), which has no -0, so negative results that round to 0 are +0"; - const char* neonDivide = "NEON (ARMv7) has no divide, division is a reciprocal estimate plus Newton-Raphson steps which isn't correctly rounded"; - // vrecpe overflows to inf for |x| < 2^-128, then vrecps( x, inf ) is -inf, so the Newton-Raphson steps give inf of the wrong sign, or NaN for 0 / x - const char* neonDivideDenormal = "NEON (ARMv7) divide by a denormal: the reciprocal estimate overflows to inf and the Newton-Raphson steps flip its sign, giving -inf for +inf and NaN for 0"; - constexpr auto denormalArg0 = []( const LaneArgs& args ) { return std::fpclassify( args.floats[0] ) == FP_SUBNORMAL; }; - constexpr auto denormalArg1 = []( const LaneArgs& args ) { return std::fpclassify( args.floats[1] ) == FP_SUBNORMAL; }; - // The refined reciprocal can be 1 ULP low, e.g. 1 / 0.99999994 becomes 1, so FLT_MAX / 0.99999994 is FLT_MAX instead of inf - const char* neonDivideOverflow = "NEON (ARMv7) divide near overflow: the Newton-Raphson reciprocal can be 1 ULP low, so quotients that overflow to +-inf can give +-FLT_MAX"; - constexpr auto quotientOverflows = []( const LaneArgs& args ) { return std::isfinite( args.floats[0] ) && args.floats[1] != 0 && std::isinf( args.floats[0] / args.floats[1] ); }; - - + void RegisterMaskTests( TestCollection& tests ) + { RegisterTest( tests, "m32 bit mask", []( TestRegm32 a ) { return a; }, []( bool a ) { return a; } ); RegisterTest( tests, "m32 any mask", []( TestRegm32 a ) { return FS::AnyMask( a ); } ); RegisterTest( tests, "m32 bit mask to int", []( TestRegm32 a ) { return static_cast( FS::BitMask( a ) ); } ); @@ -515,7 +525,10 @@ class FastSIMD::DispatchClass, SIMD> : publ RegisterTest( tests, "m32i bit xor operator", std::bit_xor(), []( bool a, bool b ) { return a != b; } ); RegisterTest( tests, "m32i bit not operator", std::bit_not(), []( bool a ) { return !a; } ); RegisterTest( tests, "m32i bit and not", []( TestRegm32i a, TestRegm32i b ) { return FS::BitwiseAndNot( a, b ); }, []( bool a, bool b ) { return a && !b; } ); + } + void RegisterIntTests( TestCollection& tests ) + { RegisterTest( tests, "i32 load store", []( TestRegi32 a ) { return a; }, []( int32_t a ) { return a; } ); RegisterTest( tests, "i32 load scalar", []( int32_t a ) { return TestRegi32( a ); }, []( int32_t a ) { return a; } ); RegisterTest( tests, "i32 splat", []( int32_t a ) { return FS::Splat( a ); }, []( int32_t a ) { return a; } ); @@ -553,9 +566,9 @@ class FastSIMD::DispatchClass, SIMD> : publ RegisterTest( tests, "i32 min", []( TestRegi32 a, TestRegi32 b ) { return FS::Min( a, b ); }, []( int32_t a, int32_t b ) { return std::min( a, b ); } ); RegisterTest( tests, "i32 max", []( TestRegi32 a, TestRegi32 b ) { return FS::Max( a, b ); }, []( int32_t a, int32_t b ) { return std::max( a, b ); } ); - RegisterTest( tests, "i32 bit shift left scalar", []( TestRegi32 a, int b ) { return a << b; }, []( int32_t a, int32_t b ) { return Wrap( U( a ) << b ); } ).Inputs( In::Any(), In::Range( 0, 31 ) ); - RegisterTest( tests, "i32 bit shift right scalar", []( TestRegi32 a, int b ) { return a >> b; }, []( int32_t a, int32_t b ) { return a < 0 ? ~( ~a >> b ) : a >> b; } ).Inputs( In::Any(), In::Range( 0, 31 ) ); - RegisterTest( tests, "i32 bit shift right zero extend scalar", []( TestRegi32 a, int b ) { return FS::BitShiftRightZeroExtend( a, b ); }, []( int32_t a, int32_t b ) { return Wrap( U( a ) >> b ); } ).Inputs( In::Any(), In::Range( 0, 31 ) ); + RegisterTest( tests, "i32 bit shift left scalar", []( TestRegi32 a, int b ) { return a << b; }, []( int32_t a, int32_t b ) { return Wrap( U( a ) << b ); } ).Inputs( In(), In::Range( 0, 31 ) ); + RegisterTest( tests, "i32 bit shift right scalar", []( TestRegi32 a, int b ) { return a >> b; }, []( int32_t a, int32_t b ) { return a < 0 ? ~( ~a >> b ) : a >> b; } ).Inputs( In(), In::Range( 0, 31 ) ); + RegisterTest( tests, "i32 bit shift right zero extend scalar", []( TestRegi32 a, int b ) { return FS::BitShiftRightZeroExtend( a, b ); }, []( int32_t a, int32_t b ) { return Wrap( U( a ) >> b ); } ).Inputs( In(), In::Range( 0, 31 ) ); //RegisterTest( tests, "i32 bit shift left", []( TestRegi32 a, TestRegi32 b ) { return a << FS::Min( TestRegi32( 31 ), FS::Abs( b ) ); } ); //RegisterTest( tests, "i32 bit shift right", []( TestRegi32 a, TestRegi32 b ) { return a >> FS::Min( TestRegi32( 31 ), FS::Abs( b ) ); } ); @@ -575,7 +588,7 @@ class FastSIMD::DispatchClass, SIMD> : publ RegisterTest( tests, "i32 masked int mask", []( TestRegm32i m, TestRegi32 a ) { return FS::Masked( m, a ); }, []( bool m, int32_t a ) { return m ? a : 0; } ); RegisterTest( tests, "i32 masked increment int mask", []( TestRegm32i m, TestRegi32 a ) { return FS::MaskedIncrement( m, a ); }, []( bool m, int32_t a ) { return m ? Wrap( U( a ) + 1u ) : a; } ); RegisterTest( tests, "i32 select high bit", []( TestRegf32 m, TestRegi32 a, TestRegi32 b ) { return FS::SelectHighBit( m, a, b ); }, - []( float m, int32_t a, int32_t b ) { return ( Bits( m ) >> 31 ) ? a : b; } ).Inputs( In::AnyNonFinite() ); + []( float m, int32_t a, int32_t b ) { return ( ToBits( m ) >> 31 ) ? a : b; } ); RegisterTest( tests, "i32 masked", []( TestRegm32 m, TestRegi32 a ) { return FS::Masked( m, a ); }, []( bool m, int32_t a ) { return m ? a : 0; } ); RegisterTest( tests, "i32 inv masked", []( TestRegm32 m, TestRegi32 a ) { return FS::InvMasked( m, a ); }, []( bool m, int32_t a ) { return m ? 0 : a; } ); RegisterTest( tests, "i32 masked increment", []( TestRegm32 m, TestRegi32 a ) { return FS::MaskedIncrement( m, a ); }, []( bool m, int32_t a ) { return m ? Wrap( U( a ) + 1u ) : a; } ); @@ -586,26 +599,27 @@ class FastSIMD::DispatchClass, SIMD> : publ RegisterTest( tests, "i32 inv masked add", []( TestRegm32 m, TestRegi32 a, TestRegi32 b ) { return FS::InvMaskedAdd( m, a, b ); }, []( bool m, int32_t a, int32_t b ) { return m ? a : Wrap( U( a ) + U( b ) ); } ); RegisterTest( tests, "i32 inv masked sub", []( TestRegm32 m, TestRegi32 a, TestRegi32 b ) { return FS::InvMaskedSub( m, a, b ); }, []( bool m, int32_t a, int32_t b ) { return m ? a : Wrap( U( a ) - U( b ) ); } ); RegisterTest( tests, "i32 inv masked mul", []( TestRegm32 m, TestRegi32 a, TestRegi32 b ) { return FS::InvMaskedMul( m, a, b ); }, []( bool m, int32_t a, int32_t b ) { return m ? a : Wrap( U( a ) * U( b ) ); } ); + } - RegisterTest( tests, "f32 load store", []( TestRegf32 a ) { return a; }, []( float a ) { return Bits( a ); } ).Inputs( In::AnyNonFinite() ).BitExact(); - RegisterTest( tests, "f32 load scalar", []( float a ) { return TestRegf32( a ); }, []( float a ) { return Bits( a ); } ).Inputs( In::AnyNonFinite() ).BitExact(); - RegisterTest( tests, "f32 splat", []( float a ) { return FS::Splat( a ); }, []( float a ) { return Bits( a ); } ).Inputs( In::AnyNonFinite() ).BitExact(); - RegisterTest( tests, "f32 extract 0", []( TestRegf32 a ) { return FS::Extract0( a ); }, []( float a ) { return Bits( a ); } ).Inputs( In::AnyNonFinite() ).BitExact(); + void RegisterFloatTests( TestCollection& tests ) + { + RegisterTest( tests, "f32 load store", []( TestRegf32 a ) { return a; }, []( float a ) { return ToBits( a ); } ).BitExact(); + RegisterTest( tests, "f32 load scalar", []( float a ) { return TestRegf32( a ); }, []( float a ) { return ToBits( a ); } ).BitExact(); + RegisterTest( tests, "f32 splat", []( float a ) { return FS::Splat( a ); }, []( float a ) { return ToBits( a ); } ).BitExact(); + RegisterTest( tests, "f32 extract 0", []( TestRegf32 a ) { return FS::Extract0( a ); }, []( float a ) { return ToBits( a ); } ).BitExact(); RegisterTest( tests, "f32 load incremented", []() { return FS::LoadIncremented(); } ); - RegisterTest( tests, "f32 plus operator", std::plus(), []( float a, float b ) { return a + b; } ).Inputs( In::AnyNonFinite(), In::AnyNonFinite() ); - RegisterTest( tests, "f32 minus operator", std::minus(), []( float a, float b ) { return a - b; } ).Inputs( In::AnyNonFinite(), In::AnyNonFinite() ); - RegisterTest( tests, "f32 multiply operator", std::multiplies(), []( float a, float b ) { return a * b; } ).Inputs( In::AnyNonFinite(), In::AnyNonFinite() ); - RegisterTest( tests, "f32 divide operator", std::divides(), []( float a, float b ) { return a / b; } ).Inputs( In::AnyNonFinite(), In::AnyNonFinite() ).RelaxedAccuracy( 64 ) - .KnownFailure( neonDivide, Tiers::NonRelaxed, FeatureSet::NEON, FeatureSet::NEON ) - .KnownFailureIf( neonDivideDenormal, denormalArg1, Tiers::Relaxed, FeatureSet::NEON, FeatureSet::NEON ) - .KnownFailureIf( neonDivideOverflow, quotientOverflows, Tiers::Relaxed, FeatureSet::NEON, FeatureSet::NEON ); + RegisterTest( tests, "f32 plus operator", std::plus(), []( float a, float b ) { return a + b; } ); + RegisterTest( tests, "f32 minus operator", std::minus(), []( float a, float b ) { return a - b; } ); + RegisterTest( tests, "f32 multiply operator", std::multiplies(), []( float a, float b ) { return a * b; } ); + RegisterTest( tests, "f32 divide operator", std::divides(), []( float a, float b ) { return a / b; } ).RelaxedAccuracy( 64 ) + .Known( neonDivide, neonDivideDenormal, neonDivideOverflow ); // b = -1 keeps a * b exact, so fused and unfused results match - RegisterTest( tests, "f32 fused multiply add", []( TestRegf32 a, TestRegf32 b ) { return FS::FMulAdd( a, TestRegf32( -1 ), b ); }, []( float a, float b ) { return -a + b; } ).Inputs( In::AnyNonFinite(), In::AnyNonFinite() ); - RegisterTest( tests, "f32 fused multiply sub", []( TestRegf32 a, TestRegf32 b ) { return FS::FMulSub( a, TestRegf32( -1 ), b ); }, []( float a, float b ) { return -a - b; } ).Inputs( In::AnyNonFinite(), In::AnyNonFinite() ); - RegisterTest( tests, "f32 fused negative multiply add", []( TestRegf32 a, TestRegf32 b ) { return FS::FNMulAdd( a, TestRegf32( -1 ), b ); }, []( float a, float b ) { return b + a; } ).Inputs( In::AnyNonFinite(), In::AnyNonFinite() ); - RegisterTest( tests, "f32 fused negative multiply sub", []( TestRegf32 a, TestRegf32 b ) { return FS::FNMulSub( a, TestRegf32( -1 ), b ); }, []( float a, float b ) { return -b + a; } ).Inputs( In::AnyNonFinite(), In::AnyNonFinite() ); + RegisterTest( tests, "f32 fused multiply add", []( TestRegf32 a, TestRegf32 b ) { return FS::FMulAdd( a, TestRegf32( -1 ), b ); }, []( float a, float b ) { return -a + b; } ); + RegisterTest( tests, "f32 fused multiply sub", []( TestRegf32 a, TestRegf32 b ) { return FS::FMulSub( a, TestRegf32( -1 ), b ); }, []( float a, float b ) { return -a - b; } ); + RegisterTest( tests, "f32 fused negative multiply add", []( TestRegf32 a, TestRegf32 b ) { return FS::FNMulAdd( a, TestRegf32( -1 ), b ); }, []( float a, float b ) { return b + a; } ); + RegisterTest( tests, "f32 fused negative multiply sub", []( TestRegf32 a, TestRegf32 b ) { return FS::FNMulSub( a, TestRegf32( -1 ), b ); }, []( float a, float b ) { return -b + a; } ); // 12 bit significands keep a * b exact, so fused and unfused results match while every argument is independent const In exactProduct = In::Magnitude( 1.e-15, 1.e+15 ).Bits( 12 ); @@ -618,129 +632,121 @@ class FastSIMD::DispatchClass, SIMD> : publ RegisterTest( tests, "f32 fused negative multiply sub exact product", []( TestRegf32 a, TestRegf32 b, TestRegf32 c ) { return FS::FNMulSub( a, b, c ); }, []( float a, float b, float c ) { float ab = a * b; return -c - ab; } ).Inputs( exactProduct, exactProduct, In::Range( -1.e+30, 1.e+30 ) ); - RegisterTest( tests, "f32 bit and operator", std::bit_and(), []( float a, float b ) { return Bits( a ) & Bits( b ); } ).Inputs( In::AnyNonFinite(), In::AnyNonFinite() ).BitExact(); - RegisterTest( tests, "f32 bit or operator", std::bit_or(), []( float a, float b ) { return Bits( a ) | Bits( b ); } ).Inputs( In::AnyNonFinite(), In::AnyNonFinite() ).BitExact(); - RegisterTest( tests, "f32 bit xor operator", std::bit_xor(), []( float a, float b ) { return Bits( a ) ^ Bits( b ); } ).Inputs( In::AnyNonFinite(), In::AnyNonFinite() ).BitExact(); - RegisterTest( tests, "f32 bit not operator", std::bit_not(), []( float a ) { return ~Bits( a ); } ).Inputs( In::AnyNonFinite() ).BitExact(); - RegisterTest( tests, "f32 negate operator", std::negate(), []( float a ) { return Bits( a ) ^ 0x80000000u; } ).Inputs( In::AnyNonFinite() ).BitExact(); + RegisterTest( tests, "f32 bit and operator", std::bit_and(), []( float a, float b ) { return ToBits( a ) & ToBits( b ); } ).BitExact(); + RegisterTest( tests, "f32 bit or operator", std::bit_or(), []( float a, float b ) { return ToBits( a ) | ToBits( b ); } ).BitExact(); + RegisterTest( tests, "f32 bit xor operator", std::bit_xor(), []( float a, float b ) { return ToBits( a ) ^ ToBits( b ); } ).BitExact(); + RegisterTest( tests, "f32 bit not operator", std::bit_not(), []( float a ) { return ~ToBits( a ); } ).BitExact(); + RegisterTest( tests, "f32 negate operator", std::negate(), []( float a ) { return ToBits( a ) ^ 0x80000000u; } ).BitExact(); // See i32 operators keep lhs. Divide isn't exact on NEON, its lhs is read again by FS::Modulus, see f32 modulus exact RegisterTest( tests, "f32 operators keep lhs", []( TestRegm32 m, TestRegf32 a, TestRegf32 b ) { return FS::Select( m, a + b, a ) ^ FS::Select( m, a - b, a ) ^ FS::Select( m, a * b, a ) ^ FS::Select( m, a & b, a ) ^ FS::Select( m, a | b, a ) ^ FS::Select( m, a ^ b, a ) ^ a; }, []( bool m, float a, float b ) { - uint32_t results[] = { Bits( a + b ), Bits( a - b ), Bits( a * b ), Bits( a ) & Bits( b ), Bits( a ) | Bits( b ), Bits( a ) ^ Bits( b ) }; - uint32_t combined = Bits( a ); + uint32_t results[] = { ToBits( a + b ), ToBits( a - b ), ToBits( a * b ), ToBits( a ) & ToBits( b ), ToBits( a ) | ToBits( b ), ToBits( a ) ^ ToBits( b ) }; + uint32_t combined = ToBits( a ); for( uint32_t result : results ) { - combined ^= m ? result : Bits( a ); + combined ^= m ? result : ToBits( a ); } return combined; - } ).Inputs( In::Any(), In::Any(), In::Any() ).BitExact(); - RegisterTest( tests, "f32 bit and not", []( TestRegf32 a, TestRegf32 b ) { return FS::BitwiseAndNot( a, b ); }, []( float a, float b ) { return Bits( a ) & ~Bits( b ); } ).Inputs( In::AnyNonFinite(), In::AnyNonFinite() ).BitExact(); - - RegisterTest( tests, "f32 equals operator", []( TestRegf32 a, TestRegf32 b ) { return a == b; }, []( float a, float b ) { return a == b; } ).Inputs( In::AnyNonFinite(), In::AnyNonFinite() ); - RegisterTest( tests, "f32 greater equal than operator", []( TestRegf32 a, TestRegf32 b ) { return a >= b; }, []( float a, float b ) { return a >= b; } ).Inputs( In::AnyNonFinite(), In::AnyNonFinite() ); - RegisterTest( tests, "f32 not equals operator", []( TestRegf32 a, TestRegf32 b ) { return a != b; }, []( float a, float b ) { return a != b; } ).Inputs( In::AnyNonFinite(), In::AnyNonFinite() ); - RegisterTest( tests, "f32 less than operator", []( TestRegf32 a, TestRegf32 b ) { return a < b; }, []( float a, float b ) { return a < b; } ).Inputs( In::AnyNonFinite(), In::AnyNonFinite() ); - RegisterTest( tests, "f32 greater than operator", []( TestRegf32 a, TestRegf32 b ) { return a > b; }, []( float a, float b ) { return a > b; } ).Inputs( In::AnyNonFinite(), In::AnyNonFinite() ); - RegisterTest( tests, "f32 less equal than operator", []( TestRegf32 a, TestRegf32 b ) { return a <= b; }, []( float a, float b ) { return a <= b; } ).Inputs( In::AnyNonFinite(), In::AnyNonFinite() ); - - RegisterTest( tests, "f32 increment", []( TestRegf32 a ) { return FS::Increment( a ); }, []( float a ) { return a + 1.0f; } ).Inputs( In::AnyNonFinite() ); - RegisterTest( tests, "f32 decrement", []( TestRegf32 a ) { return FS::Decrement( a ); }, []( float a ) { return a - 1.0f; } ).Inputs( In::AnyNonFinite() ); - RegisterTest( tests, "f32 abs", []( TestRegf32 a ) { return FS::Abs( a ); }, []( float a ) { return Bits( a ) & 0x7FFFFFFFu; } ).Inputs( In::AnyNonFinite() ).BitExact(); -#if defined( _MSC_VER ) && !defined( __clang__ ) && defined( _DEBUG ) - // The scalar reference is wrong, so every tier and the oracle comparison see it - const char* msvcDebugAbs = "Unoptimised MSVC calls the CRT fabsf for std::abs in the scalar FS::Abs, which quiets signalling NaNs"; - tests.back().second.KnownFailure( msvcDebugAbs ).KnownFailure( msvcDebugAbs, Tiers::Oracle ); -#endif + } ).Inputs( In(), In::Finite(), In::Finite() ).BitExact(); + RegisterTest( tests, "f32 bit and not", []( TestRegf32 a, TestRegf32 b ) { return FS::BitwiseAndNot( a, b ); }, []( float a, float b ) { return ToBits( a ) & ~ToBits( b ); } ).BitExact(); + + RegisterTest( tests, "f32 equals operator", []( TestRegf32 a, TestRegf32 b ) { return a == b; }, []( float a, float b ) { return a == b; } ); + RegisterTest( tests, "f32 greater equal than operator", []( TestRegf32 a, TestRegf32 b ) { return a >= b; }, []( float a, float b ) { return a >= b; } ); + RegisterTest( tests, "f32 not equals operator", []( TestRegf32 a, TestRegf32 b ) { return a != b; }, []( float a, float b ) { return a != b; } ); + RegisterTest( tests, "f32 less than operator", []( TestRegf32 a, TestRegf32 b ) { return a < b; }, []( float a, float b ) { return a < b; } ); + RegisterTest( tests, "f32 greater than operator", []( TestRegf32 a, TestRegf32 b ) { return a > b; }, []( float a, float b ) { return a > b; } ); + RegisterTest( tests, "f32 less equal than operator", []( TestRegf32 a, TestRegf32 b ) { return a <= b; }, []( float a, float b ) { return a <= b; } ); + + RegisterTest( tests, "f32 increment", []( TestRegf32 a ) { return FS::Increment( a ); }, []( float a ) { return a + 1.0f; } ); + RegisterTest( tests, "f32 decrement", []( TestRegf32 a ) { return FS::Decrement( a ); }, []( float a ) { return a - 1.0f; } ); + RegisterTest( tests, "f32 abs", []( TestRegf32 a ) { return FS::Abs( a ); }, []( float a ) { return ToBits( a ) & 0x7FFFFFFFu; } ).BitExact().Known( msvcDebugAbs ); // Relaxed min/max may use instructions with different NaN handling, those are checked non relaxed in the full range tests - RegisterTest( tests, "f32 min", []( TestRegf32 a, TestRegf32 b ) { return FS::Min( a, b ); }, []( float a, float b ) { return std::min( a, b ); } ).Inputs( In::AnyInf(), In::AnyInf() ); - RegisterTest( tests, "f32 max", []( TestRegf32 a, TestRegf32 b ) { return FS::Max( a, b ); }, []( float a, float b ) { return std::max( a, b ); } ).Inputs( In::AnyInf(), In::AnyInf() ); - - RegisterTest( tests, "f32 select", []( TestRegm32 m, TestRegf32 a, TestRegf32 b ) { return FS::Select( m, a, b ); }, []( bool m, float a, float b ) { return Bits( m ? a : b ); } ).Inputs( In::Any(), In::AnyNonFinite(), In::AnyNonFinite() ).BitExact(); - RegisterTest( tests, "f32 select high bit", []( TestRegi32 m, TestRegf32 a, TestRegf32 b ) { return FS::SelectHighBit( m, a, b ); }, []( int32_t m, float a, float b ) { return Bits( m < 0 ? a : b ); } ).Inputs( In::Any(), In::AnyNonFinite(), In::AnyNonFinite() ).BitExact(); - RegisterTest( tests, "f32 masked", []( TestRegm32 m, TestRegf32 a ) { return FS::Masked( m, a ); }, []( bool m, float a ) { return m ? Bits( a ) : 0u; } ).Inputs( In::Any(), In::AnyNonFinite() ).BitExact(); - RegisterTest( tests, "f32 inv masked", []( TestRegm32 m, TestRegf32 a ) { return FS::InvMasked( m, a ); }, []( bool m, float a ) { return m ? 0u : Bits( a ); } ).Inputs( In::Any(), In::AnyNonFinite() ).BitExact(); - RegisterTest( tests, "f32 masked increment", []( TestRegm32 m, TestRegf32 a ) { return FS::MaskedIncrement( m, a ); }, []( bool m, float a ) { return m ? a + 1.0f : a; } ).Inputs( In::Any(), In::AnyNonFinite() ); - RegisterTest( tests, "f32 masked decrement", []( TestRegm32 m, TestRegf32 a ) { return FS::MaskedDecrement( m, a ); }, []( bool m, float a ) { return m ? a - 1.0f : PlusZero( a ); } ).Inputs( In::Any(), In::AnyNonFinite() ); - - // Generic masked add is a + Masked( m, b ), so masked off lanes add +0. AVX512 512bit uses a native masked add that leaves them unchanged - const char* avx512MaskedAddZeroSign = "AVX512 512bit masked add leaves masked off lanes unchanged, the generic a + Masked( m, b ) turns -0 into +0"; - TestData& maskedAdd = RegisterTest( tests, "f32 masked add", []( TestRegm32 m, TestRegf32 a, TestRegf32 b ) { return FS::MaskedAdd( m, a, b ); }, - []( bool m, float a, float b ) { return m ? a + b : PlusZero( a ); } ).Inputs( In::Any(), In::AnyNonFinite(), In::AnyNonFinite() ); - if constexpr( RegisterBytes >= 64 ) - { - maskedAdd.KnownZeroSign( avx512MaskedAddZeroSign, Tiers::NonRelaxed, FeatureSet::AVX512, FeatureSet::AVX512 ); - } - RegisterTest( tests, "f32 masked sub", []( TestRegm32 m, TestRegf32 a, TestRegf32 b ) { return FS::MaskedSub( m, a, b ); }, []( bool m, float a, float b ) { return m ? a - b : a; } ).Inputs( In::Any(), In::AnyNonFinite(), In::AnyNonFinite() ); - RegisterTest( tests, "f32 masked mul", []( TestRegm32 m, TestRegf32 a, TestRegf32 b ) { return FS::MaskedMul( m, a, b ); }, []( bool m, float a, float b ) { return m ? a * b : a; } ).Inputs( In::Any(), In::AnyNonFinite(), In::AnyNonFinite() ); - TestData& invMaskedAdd = RegisterTest( tests, "f32 inv masked add", []( TestRegm32 m, TestRegf32 a, TestRegf32 b ) { return FS::InvMaskedAdd( m, a, b ); }, - []( bool m, float a, float b ) { return m ? PlusZero( a ) : a + b; } ).Inputs( In::Any(), In::AnyNonFinite(), In::AnyNonFinite() ); - if constexpr( RegisterBytes >= 64 ) - { - invMaskedAdd.KnownZeroSign( avx512MaskedAddZeroSign, Tiers::NonRelaxed, FeatureSet::AVX512, FeatureSet::AVX512 ); - } - RegisterTest( tests, "f32 inv masked sub", []( TestRegm32 m, TestRegf32 a, TestRegf32 b ) { return FS::InvMaskedSub( m, a, b ); }, []( bool m, float a, float b ) { return m ? a : a - b; } ).Inputs( In::Any(), In::AnyNonFinite(), In::AnyNonFinite() ); - RegisterTest( tests, "f32 inv masked mul", []( TestRegm32 m, TestRegf32 a, TestRegf32 b ) { return FS::InvMaskedMul( m, a, b ); }, []( bool m, float a, float b ) { return m ? a : a * b; } ).Inputs( In::Any(), In::AnyNonFinite(), In::AnyNonFinite() ); - - RegisterTest( tests, "f32 round", []( TestRegf32 a ) { return FS::Round( a ); }, []( float a ) { return std::rint( a ); } ).Inputs( In::AnyNonFinite() ) - .KnownZeroSign( sse2RoundZeroSign, Tiers::NonRelaxed, FeatureSet::SSE, FeatureSet::SSSE3 ); - RegisterTest( tests, "f32 ceil", []( TestRegf32 a ) { return FS::Ceil( a ); }, []( float a ) { return std::ceil( a ); } ).Inputs( In::AnyNonFinite() ) - .KnownZeroSign( sse2RoundZeroSign, Tiers::NonRelaxed, FeatureSet::SSE, FeatureSet::SSSE3 ); - RegisterTest( tests, "f32 floor", []( TestRegf32 a ) { return FS::Floor( a ); }, []( float a ) { return std::floor( a ); } ).Inputs( In::AnyNonFinite() ) - .KnownZeroSign( sse2RoundZeroSign, Tiers::NonRelaxed, FeatureSet::SSE, FeatureSet::SSSE3 ); - RegisterTest( tests, "f32 trunc", []( TestRegf32 a ) { return FS::Trunc( a ); }, []( float a ) { return std::trunc( a ); } ).Inputs( In::AnyNonFinite() ) - .KnownZeroSign( sse2RoundZeroSign, Tiers::NonRelaxed, FeatureSet::SSE, FeatureSet::SSSE3 ); - RegisterTest( tests, "f32 signbit", []( TestRegf32 a, TestRegf32 b ) { return FS::SignBit( a ) ^ b; }, []( float a, float b ) { return ( Bits( a ) & 0x80000000u ) ^ Bits( b ); } ).Inputs( In::AnyNonFinite(), In::AnyNonFinite() ).BitExact(); + RegisterTest( tests, "f32 min", []( TestRegf32 a, TestRegf32 b ) { return FS::Min( a, b ); }, []( float a, float b ) { return std::min( a, b ); } ).Inputs( In::NoNaN(), In::NoNaN() ); + RegisterTest( tests, "f32 max", []( TestRegf32 a, TestRegf32 b ) { return FS::Max( a, b ); }, []( float a, float b ) { return std::max( a, b ); } ).Inputs( In::NoNaN(), In::NoNaN() ); + + RegisterTest( tests, "f32 select", []( TestRegm32 m, TestRegf32 a, TestRegf32 b ) { return FS::Select( m, a, b ); }, []( bool m, float a, float b ) { return ToBits( m ? a : b ); } ).BitExact(); + RegisterTest( tests, "f32 select high bit", []( TestRegi32 m, TestRegf32 a, TestRegf32 b ) { return FS::SelectHighBit( m, a, b ); }, []( int32_t m, float a, float b ) { return ToBits( m < 0 ? a : b ); } ).BitExact(); + RegisterTest( tests, "f32 masked", []( TestRegm32 m, TestRegf32 a ) { return FS::Masked( m, a ); }, []( bool m, float a ) { return m ? ToBits( a ) : 0u; } ).BitExact(); + RegisterTest( tests, "f32 inv masked", []( TestRegm32 m, TestRegf32 a ) { return FS::InvMasked( m, a ); }, []( bool m, float a ) { return m ? 0u : ToBits( a ); } ).BitExact(); + RegisterTest( tests, "f32 masked increment", []( TestRegm32 m, TestRegf32 a ) { return FS::MaskedIncrement( m, a ); }, []( bool m, float a ) { return m ? a + 1.0f : a; } ); + RegisterTest( tests, "f32 masked decrement", []( TestRegm32 m, TestRegf32 a ) { return FS::MaskedDecrement( m, a ); }, []( bool m, float a ) { return m ? a - 1.0f : PlusZero( a ); } ); + + RegisterTest( tests, "f32 masked add", []( TestRegm32 m, TestRegf32 a, TestRegf32 b ) { return FS::MaskedAdd( m, a, b ); }, + []( bool m, float a, float b ) { return m ? a + b : PlusZero( a ); } ).Known( avx512MaskedAddZeroSign ); + RegisterTest( tests, "f32 masked sub", []( TestRegm32 m, TestRegf32 a, TestRegf32 b ) { return FS::MaskedSub( m, a, b ); }, []( bool m, float a, float b ) { return m ? a - b : a; } ); + RegisterTest( tests, "f32 masked mul", []( TestRegm32 m, TestRegf32 a, TestRegf32 b ) { return FS::MaskedMul( m, a, b ); }, []( bool m, float a, float b ) { return m ? a * b : a; } ); + RegisterTest( tests, "f32 inv masked add", []( TestRegm32 m, TestRegf32 a, TestRegf32 b ) { return FS::InvMaskedAdd( m, a, b ); }, + []( bool m, float a, float b ) { return m ? PlusZero( a ) : a + b; } ).Known( avx512MaskedAddZeroSign ); + RegisterTest( tests, "f32 inv masked sub", []( TestRegm32 m, TestRegf32 a, TestRegf32 b ) { return FS::InvMaskedSub( m, a, b ); }, []( bool m, float a, float b ) { return m ? a : a - b; } ); + RegisterTest( tests, "f32 inv masked mul", []( TestRegm32 m, TestRegf32 a, TestRegf32 b ) { return FS::InvMaskedMul( m, a, b ); }, []( bool m, float a, float b ) { return m ? a : a * b; } ); + + RegisterTest( tests, "f32 round", []( TestRegf32 a ) { return FS::Round( a ); }, []( float a ) { return std::rint( a ); } ) + .Known( sse2RoundZeroSign ); + RegisterTest( tests, "f32 ceil", []( TestRegf32 a ) { return FS::Ceil( a ); }, []( float a ) { return std::ceil( a ); } ) + .Known( sse2RoundZeroSign ); + RegisterTest( tests, "f32 floor", []( TestRegf32 a ) { return FS::Floor( a ); }, []( float a ) { return std::floor( a ); } ) + .Known( sse2RoundZeroSign ); + RegisterTest( tests, "f32 trunc", []( TestRegf32 a ) { return FS::Trunc( a ); }, []( float a ) { return std::trunc( a ); } ) + .Known( sse2RoundZeroSign ); + RegisterTest( tests, "f32 signbit", []( TestRegf32 a, TestRegf32 b ) { return FS::SignBit( a ) ^ b; }, []( float a, float b ) { return ( ToBits( a ) & 0x80000000u ) ^ ToBits( b ); } ).BitExact(); RegisterTest( tests, "f32 modulus", []( TestRegf32 a, TestRegf32 b ) { return FS::Modulus( a, b ); } ) .Inputs( In::Range( -100, 100 ), In::Magnitude( 0.25, 100 ) ).RelaxedAccuracy( 16384 ) - .KnownFailure( "Scalar uses std::fmod, SIMD and relaxed use a - trunc( a / b ) * b which isn't exact" ); + .Known( KnownFailure( "Scalar uses std::fmod, SIMD and relaxed use a - trunc( a / b ) * b which isn't exact" ) ); // b is a power of 2 and a is never tiny, so a - trunc( a / b ) * b is exact and must match std::fmod RegisterTest( tests, "f32 modulus exact", []( TestRegf32 a, TestRegf32 b ) { return FS::Modulus( a, b ); }, []( float a, float b ) { return std::fmod( a, b ); } ) .Inputs( In::Magnitude( 1.e-30, 100 ), In::Magnitude( 0.25, 64 ).Bits( 1 ) ) - .KnownZeroSign( "a - trunc( a / b ) * b gives +0 for exact negative multiples, std::fmod gives -0", Tiers::NonRelaxed ); + .Known( KnownFailure( "a - trunc( a / b ) * b gives +0 for exact negative multiples, std::fmod gives -0", Tiers::NonRelaxed ).ZeroSignOnly() ); + } - RegisterTest( tests, "f32 sqrt", []( TestRegf32 a ) { return FS::Sqrt( a ); }, []( float a ) { return std::sqrt( a ); } ).Inputs( In::AnyNonFinite() ); + void RegisterFloatMathTests( TestCollection& tests ) + { + // Range reduction boundaries for trig + const In trigRange = In::Range( -4096, 4096 ).Specials( { pi / 4, pi / 2, pi, 3 * pi / 2, 2 * pi, 4 * pi, 100 * pi, 1000 * pi, + -pi / 4, -pi / 2, -pi, -3 * pi / 2, -2 * pi, -4 * pi, -100 * pi, -1000 * pi } ); + const In smallTrigRange = In::Range( -2 * pi, 2 * pi ).Specials( { pi / 4, pi / 2, pi, 3 * pi / 2, 2 * pi, + -pi / 4, -pi / 2, -pi, -3 * pi / 2, -2 * pi } ); + + RegisterTest( tests, "f32 sqrt", []( TestRegf32 a ) { return FS::Sqrt( a ); }, []( float a ) { return std::sqrt( a ); } ); // Relaxed x86 uses the rcpps/rsqrtps estimates, their 1.5 * 2^-12 relative error is up to ~6144 ULP RegisterTest( tests, "f32 inv sqrt", []( TestRegf32 a ) { return FS::InvSqrt( a ); }, []( float a ) { return 1.0f / std::sqrt( a ); } ).Inputs( In::Range( 1.e-16, 1.e+16 ) ).RelaxedAccuracy( 8192 ) - .KnownFailure( neonDivide, Tiers::NonRelaxed, FeatureSet::NEON, FeatureSet::NEON ); + .Known( neonDivide ); RegisterTest( tests, "f32 reciprocal", []( TestRegf32 a ) { return FS::Reciprocal( a ); }, []( float a ) { return 1.0f / a; } ).Inputs( In::Magnitude( 1.e-16, 1.e+16 ) ).RelaxedAccuracy( 8192 ) - .KnownFailure( neonDivide, Tiers::NonRelaxed, FeatureSet::NEON, FeatureSet::NEON ); + .Known( neonDivide ); // Absolute error allowance is ~1 ULP of the largest input, float inputs that large are only that precise RegisterTest( tests, "f32 cos", []( TestRegf32 a ) { return FS::Cos( a ); } ).Inputs( trigRange ).RelaxedAccuracy( 4, 5.e-4f ); RegisterTest( tests, "f32 sin", []( TestRegf32 a ) { return FS::Sin( a ); } ).Inputs( trigRange ).RelaxedAccuracy( 4, 5.e-4f ) - .KnownZeroSign( "Sin_Reduce uses Trunc, see f32 trunc: Sin( -0 ) is -0 below SSE4.1 and +0 elsewhere", Tiers::NonRelaxed, FeatureSet::SSE, FeatureSet::SSSE3 ); + .Known( sinReduceZeroSign ); // Within +-2pi range reduction is exact apart from the float rounding of pi and 2pi (<= ~2.6e-7), so the polynomials are checked closely RegisterTest( tests, "f32 cos small range", []( TestRegf32 a ) { return FS::Cos( a ); } ).Inputs( smallTrigRange ).RelaxedAccuracy( 4, 5.e-7f ); RegisterTest( tests, "f32 sin small range", []( TestRegf32 a ) { return FS::Sin( a ); } ).Inputs( smallTrigRange ).RelaxedAccuracy( 4, 5.e-7f ) - .KnownZeroSign( "Sin_Reduce uses Trunc, see f32 trunc: Sin( -0 ) is -0 below SSE4.1 and +0 elsewhere", Tiers::NonRelaxed, FeatureSet::SSE, FeatureSet::SSSE3 ); + .Known( sinReduceZeroSign ); // The absolute allowances above hide relative accuracy near 0 RegisterTest( tests, "f32 sin near 0", []( TestRegf32 a ) { return FS::Sin( a ); } ).Inputs( In::Magnitude( denormMin, 0.01 ) ).RelaxedAccuracy( 4 ); // Tan is ill conditioned near its poles, so relaxed accuracy is checked between them RegisterTest( tests, "f32 tan", []( TestRegf32 a ) { return FS::Tan( a ); } ).Inputs( In::Range( -1.5, 1.5 ).Specials( { pi / 4, -pi / 4 } ) ).RelaxedAccuracy( 64 ) - .KnownFailure( neonDivide, Tiers::NonRelaxed, FeatureSet::NEON, FeatureSet::NEON ); + .Known( neonDivide ); // Between the poles at +-pi/2 and +-3pi/2, Tan_Reduce subtracts pi and folds the sign, near +-pi the float rounding of pi is absolute error RegisterTest( tests, "f32 tan reduced", []( TestRegf32 a ) { return FS::Tan( a ); } ).Inputs( In::Magnitude( 1.6, 4.6 ).Specials( { pi, 3 * pi / 4, 5 * pi / 4 } ) ).RelaxedAccuracy( 128, 3.e-7f ) - .KnownFailure( neonDivide, Tiers::NonRelaxed, FeatureSet::NEON, FeatureSet::NEON ); + .Known( neonDivide ); RegisterTest( tests, "f32 acos", []( TestRegf32 a ) { return FS::ACos( a ); } ).Inputs( In::Range( -1, 1 ) ).RelaxedAccuracy( 8 ); // ASin is K_HALF_PI - ACos( x ), near 0 that cancels and ACos's error (~2 ULP of pi/2) becomes absolute error, +0 for |x| <= 2^-25 RegisterTest( tests, "f32 asin", []( TestRegf32 a ) { return FS::ASin( a ); } ).Inputs( In::Range( -1, 1 ) ).RelaxedAccuracy( 32, 3.e-7f ); // Relaxed x86 ATan uses the rcpps estimate for |x| > 1, its 1.5 * 2^-12 relative error is up to ~3072 ULP of atan near 1 // Relaxed scalar is std::atan, so FS::ATan itself is checked much more closely - RegisterTest( tests, "f32 atan", []( TestRegf32 a ) { return FS::ATan( a ); } ).Inputs( In::AnyInf() ).RelaxedAccuracy( 4096 ).ScalarRelaxedAccuracy( 64 ) - .KnownFailure( neonDivide, Tiers::NonRelaxed, FeatureSet::NEON, FeatureSet::NEON ); - RegisterTest( tests, "f32 atan2", []( TestRegf32 a, TestRegf32 b ) { return FS::ATan2( a, b ); } ).Inputs( In::AnyInf(), In::AnyInf() ).RelaxedAccuracy( 4096 ).ScalarRelaxedAccuracy( 64 ) - .KnownFailure( "Relaxed scalar uses std::atan2: for atan2( +-0, -0 ) it returns +-pi where FS::ATan2 returns 0, and for atan2( +-inf, +-inf ) +-pi/4 or +-3pi/4 where FS::ATan2 returns NaN (inf / inf)", Tiers::ScalarRelaxed ) - .KnownFailure( neonDivide, Tiers::NonRelaxed, FeatureSet::NEON, FeatureSet::NEON ) - .KnownFailureIf( neonDivideDenormal, denormalArg1, Tiers::Relaxed, FeatureSet::NEON, FeatureSet::NEON ); + RegisterTest( tests, "f32 atan", []( TestRegf32 a ) { return FS::ATan( a ); } ).Inputs( In::NoNaN() ).RelaxedAccuracy( 4096 ).ScalarRelaxedAccuracy( 64 ) + .Known( neonDivide ); + RegisterTest( tests, "f32 atan2", []( TestRegf32 a, TestRegf32 b ) { return FS::ATan2( a, b ); } ).Inputs( In::NoNaN(), In::NoNaN() ).RelaxedAccuracy( 4096 ).ScalarRelaxedAccuracy( 64 ) + .Known( KnownFailure( "Relaxed scalar uses std::atan2: for atan2( +-0, -0 ) it returns +-pi where FS::ATan2 returns 0, and for atan2( +-inf, +-inf ) +-pi/4 or +-3pi/4 where FS::ATan2 returns NaN (inf / inf)", + Tiers::Relaxed, FeatureSet::SCALAR, FeatureSet::SCALAR ), + neonDivide, neonDivideDenormal ); // Keep relaxed scalar checking FS::ATan2 against std::atan2 everywhere except both inputs 0 - RegisterTest( tests, "f32 atan2 nonzero y", []( TestRegf32 a, TestRegf32 b ) { return FS::ATan2( a, b ); } ).Inputs( In::Magnitude( denormMin, floatMax ), In::AnyInf() ).RelaxedAccuracy( 4096 ).ScalarRelaxedAccuracy( 64 ) - .KnownFailure( neonDivide, Tiers::NonRelaxed, FeatureSet::NEON, FeatureSet::NEON ) - .KnownFailureIf( neonDivideDenormal, denormalArg1, Tiers::Relaxed, FeatureSet::NEON, FeatureSet::NEON ); - RegisterTest( tests, "f32 atan2 nonzero x", []( TestRegf32 a, TestRegf32 b ) { return FS::ATan2( a, b ); } ).Inputs( In::AnyInf(), In::Magnitude( denormMin, floatMax ) ).RelaxedAccuracy( 4096 ).ScalarRelaxedAccuracy( 64 ) - .KnownFailure( neonDivide, Tiers::NonRelaxed, FeatureSet::NEON, FeatureSet::NEON ) - .KnownFailureIf( neonDivideDenormal, denormalArg1, Tiers::Relaxed, FeatureSet::NEON, FeatureSet::NEON ); + RegisterTest( tests, "f32 atan2 nonzero y", []( TestRegf32 a, TestRegf32 b ) { return FS::ATan2( a, b ); } ).Inputs( In::Magnitude( denormMin, floatMax ), In::NoNaN() ).RelaxedAccuracy( 4096 ).ScalarRelaxedAccuracy( 64 ) + .Known( neonDivide, neonDivideDenormal ); + RegisterTest( tests, "f32 atan2 nonzero x", []( TestRegf32 a, TestRegf32 b ) { return FS::ATan2( a, b ); } ).Inputs( In::NoNaN(), In::Magnitude( denormMin, floatMax ) ).RelaxedAccuracy( 4096 ).ScalarRelaxedAccuracy( 64 ) + .Known( neonDivide, neonDivideDenormal ); // FS::Exp clamps its input to +-88.3762626647949, the range reaches just below that and keeps results above FLT_MIN RegisterTest( tests, "f32 exp", []( TestRegf32 a ) { return FS::Exp( a ); } ).Inputs( In::Range( -87.3, 88.37 ) ).RelaxedAccuracy( 8 ); RegisterTest( tests, "f32 exp2", []( TestRegf32 a ) { return FS::Exp2( a ); } ).Inputs( In::Range( -126, 127 ) ).RelaxedAccuracy( 8 ); @@ -753,47 +759,37 @@ class FastSIMD::DispatchClass, SIMD> : publ .Inputs( In::Range( 1.e-4, 1.e+4 ), In::Range( -8, 8 ) ).RelaxedAccuracy( 256 ); RegisterTest( tests, "f32 pow zero base", []( TestRegf32 b ) { return FS::Pow( TestRegf32( 0.0f ), b ); }, []( float ) { return 0.0f; } ).Inputs( In::Range( 0.1, 8 ) ); - // Relaxed x86 uses rcpps/rsqrtps, AVX512 512bit registers use rcp14/rsqrt14 which handle denormals - const char* x86EstimateDenormals = "Relaxed rcpps/rsqrtps estimates treat denormal inputs as 0 and return inf"; - // The Intel SDM allows rcpps to flush results just above FLT_MIN to 0, for |x| just below 2^126 - const char* x86ReciprocalEstimate = "Relaxed rcpps treats denormal inputs as 0 and returns inf, and may flush results near FLT_MIN to 0"; - constexpr auto x86ReciprocalInput = []( const LaneArgs& args ) { return std::fpclassify( args.floats[0] ) == FP_SUBNORMAL || std::abs( args.floats[0] ) >= 0x1.ffcp125f; }; - constexpr FastSIMD::FeatureSet x86EstimateMax = RegisterBytes >= 64 ? FeatureSet::AVX2 : FeatureSet::AVX512; - // Relaxed ARM refines vrsqrte/vrecpe with one Newton-Raphson step, which breaks where the estimate is 0 or inf - const char* armEstimateSpecials = "Relaxed ARM estimate + Newton-Raphson step: InvSqrt( +-0 ) and InvSqrt( inf ) are NaN (0 * inf), Reciprocal of a denormal is inf of the wrong sign"; - constexpr auto armEstimateInput = []( const LaneArgs& args ) { return args.floats[0] == 0 || std::isinf( args.floats[0] ) || std::fpclassify( args.floats[0] ) == FP_SUBNORMAL; }; - RegisterTest( tests, "f32 inv sqrt full range", []( TestRegf32 a ) { return FS::InvSqrt( a ); }, []( float a ) { return 1.0f / std::sqrt( a ); } ).Inputs( In::AnyNonFinite() ).RelaxedAccuracy( 8192 ) - .KnownFailure( neonDivide, Tiers::NonRelaxed, FeatureSet::NEON, FeatureSet::NEON ) - .KnownFailureIf( x86EstimateDenormals, denormalArg0, Tiers::Relaxed, FeatureSet::SSE, x86EstimateMax ) - .KnownFailureIf( armEstimateSpecials, armEstimateInput, Tiers::Relaxed, FeatureSet::NEON, FeatureSet::AARCH64 ); - RegisterTest( tests, "f32 reciprocal full range", []( TestRegf32 a ) { return FS::Reciprocal( a ); }, []( float a ) { return 1.0f / a; } ).Inputs( In::AnyNonFinite() ).RelaxedAccuracy( 8192 ) - .KnownFailure( neonDivide, Tiers::NonRelaxed, FeatureSet::NEON, FeatureSet::NEON ) - .KnownFailureIf( x86ReciprocalEstimate, x86ReciprocalInput, Tiers::Relaxed, FeatureSet::SSE, x86EstimateMax ) - .KnownFailureIf( armEstimateSpecials, armEstimateInput, Tiers::Relaxed, FeatureSet::NEON, FeatureSet::AARCH64 ); + RegisterTest( tests, "f32 inv sqrt full range", []( TestRegf32 a ) { return FS::InvSqrt( a ); }, []( float a ) { return 1.0f / std::sqrt( a ); } ).RelaxedAccuracy( 8192 ) + .Known( neonDivide, x86EstimateDenormals, armEstimateSpecials ); + RegisterTest( tests, "f32 reciprocal full range", []( TestRegf32 a ) { return FS::Reciprocal( a ); }, []( float a ) { return 1.0f / a; } ).RelaxedAccuracy( 8192 ) + .Known( neonDivide, x86ReciprocalEstimate, armEstimateSpecials ); // Relaxed accuracy is only meaningful over the ranges above, non relaxed results must still match exactly over the whole domain if constexpr( !FastSIMD::IsRelaxed() ) { - RegisterTest( tests, "f32 min full range", []( TestRegf32 a, TestRegf32 b ) { return FS::Min( a, b ); }, []( float a, float b ) { return std::min( a, b ); } ).Inputs( In::AnyNonFinite(), In::AnyNonFinite() ); - RegisterTest( tests, "f32 max full range", []( TestRegf32 a, TestRegf32 b ) { return FS::Max( a, b ); }, []( float a, float b ) { return std::max( a, b ); } ).Inputs( In::AnyNonFinite(), In::AnyNonFinite() ); - RegisterTest( tests, "f32 cos full range", []( TestRegf32 a ) { return FS::Cos( a ); } ).Inputs( In::AnyNonFinite() ); - RegisterTest( tests, "f32 sin full range", []( TestRegf32 a ) { return FS::Sin( a ); } ).Inputs( In::AnyNonFinite() ) - .KnownZeroSign( "Sin_Reduce uses Trunc, see f32 trunc: Sin( -0 ) is -0 below SSE4.1 and +0 elsewhere", Tiers::NonRelaxed, FeatureSet::SSE, FeatureSet::SSSE3 ); - RegisterTest( tests, "f32 tan full range", []( TestRegf32 a ) { return FS::Tan( a ); } ).Inputs( In::AnyNonFinite() ) - .KnownFailure( neonDivide, Tiers::NonRelaxed, FeatureSet::NEON, FeatureSet::NEON ); + RegisterTest( tests, "f32 min full range", []( TestRegf32 a, TestRegf32 b ) { return FS::Min( a, b ); }, []( float a, float b ) { return std::min( a, b ); } ); + RegisterTest( tests, "f32 max full range", []( TestRegf32 a, TestRegf32 b ) { return FS::Max( a, b ); }, []( float a, float b ) { return std::max( a, b ); } ); + RegisterTest( tests, "f32 cos full range", []( TestRegf32 a ) { return FS::Cos( a ); } ); + RegisterTest( tests, "f32 sin full range", []( TestRegf32 a ) { return FS::Sin( a ); } ) + .Known( sinReduceZeroSign ); + RegisterTest( tests, "f32 tan full range", []( TestRegf32 a ) { return FS::Tan( a ); } ) + .Known( neonDivide ); // Exp clamps its input, no NaN since converting it to int is undefined in the scalar reference RegisterTest( tests, "f32 exp full range", []( TestRegf32 a ) { return FS::Exp( a ); } ) - .Inputs( In::AnyInf().Specials( { 88.3762626647949, -88.3762626647949, 88.72283935546875, -87.33654022216797, -103.97207641601562 } ) ); + .Inputs( In::NoNaN().Specials( { 88.3762626647949, -88.3762626647949, 88.72283935546875, -87.33654022216797, -103.97207641601562 } ) ); RegisterTest( tests, "f32 log full range", []( TestRegf32 a ) { return FS::Log( a ); } ).Inputs( In::Range( denormMin, floatMax ) ); RegisterTest( tests, "f32 log2 full range", []( TestRegf32 a ) { return FS::Log2( a ); } ).Inputs( In::Range( denormMin, floatMax ) ); - RegisterTest( tests, "f32 pow full range", []( TestRegf32 a, TestRegf32 b ) { return FS::Pow( a, b ); } ).Inputs( In::Range( denormMin, floatMax ), In::Any() ); + RegisterTest( tests, "f32 pow full range", []( TestRegf32 a, TestRegf32 b ) { return FS::Pow( a, b ); } ).Inputs( In::Range( denormMin, floatMax ), In::Finite() ); } + } + void RegisterConvertTests( TestCollection& tests ) + { RegisterTest( tests, "i32 convert to f32", []( TestRegi32 a ) { return FS::Convert( a ); }, []( int32_t a ) { return static_cast( a ); } ); RegisterTest( tests, "f32 convert to i32", []( TestRegf32 a ) { return FS::Convert( a ); }, []( float a ) { return static_cast( std::rint( a ) ); } ) .Inputs( In::Range( -2147483648.0, 2147483520.0 ) ); - RegisterTest( tests, "f32 cast to i32", []( TestRegf32 a ) { return FS::Cast( a ); }, []( float a ) { return static_cast( Bits( a ) ); } ).Inputs( In::AnyNonFinite() ); + RegisterTest( tests, "f32 cast to i32", []( TestRegf32 a ) { return FS::Cast( a ); }, []( float a ) { return static_cast( ToBits( a ) ); } ); RegisterTest( tests, "i32 cast to f32", []( TestRegi32 a ) { return FS::Cast( a ); }, []( int32_t a ) { return U( a ); } ).BitExact(); // Cast is a raw reinterpret, a mask's bits as another type differ between SIMD levels (and k-registers can't be cast) @@ -805,7 +801,6 @@ class FastSIMD::DispatchClass, SIMD> : publ RegisterTest( tests, "m32i cast to i32", []( TestRegm32i a ) { return FS_BIND_INTRINSIC( FS::Cast )( FS_BIND_INTRINSIC( FS::Cast )( a ) ); }, []( bool a ) { return a; } ); RegisterTest( tests, "m32i cast to f32", []( TestRegm32i a ) { return FS_BIND_INTRINSIC( FS::Cast )( FS_BIND_INTRINSIC( FS::Cast )( a ) ); }, []( bool a ) { return a; } ); } - return tests; } }; diff --git a/tests/test_inputs.cpp b/tests/test_inputs.cpp new file mode 100644 index 0000000..6a5a826 --- /dev/null +++ b/tests/test_inputs.cpp @@ -0,0 +1,582 @@ +#include "test_inputs.h" + +#include +#include +#include +#include +#include +#include +#include + +static uint64_t SplitMix64( uint64_t& state ) +{ + uint64_t z = ( state += 0x9E3779B97F4A7C15ull ); + z = ( z ^ ( z >> 30 ) ) * 0xBF58476D1CE4E5B9ull; + z = ( z ^ ( z >> 27 ) ) * 0x94D049BB133111EBull; + return z ^ ( z >> 31 ); +} + +// FNV-1a, stable across compilers unlike std::hash +static uint64_t HashName( std::string_view name ) +{ + uint64_t hash = 0xCBF29CE484222325ull; + for( char c : name ) + { + hash = ( hash ^ static_cast( c ) ) * 0x100000001B3ull; + } + return hash; +} + +void ValidateInputs( std::string_view testName, const TestData& test ) +{ + auto fail = [&]( size_t argIdx, const char* problem ) + { + std::string message = "Invalid input spec for \"" + std::string( testName ) + "\" argument " + std::to_string( argIdx ) + ": " + problem; + std::cerr << message << std::endl; + throw std::runtime_error( message ); + }; + + constexpr double floatMax = std::numeric_limits::max(); + + for( size_t argIdx = 0; argIdx < kMaxTestArgs; argIdx++ ) + { + const InputSpec& spec = test.inputs[argIdx]; + + if( argIdx >= test.argCount ) + { + if( !spec.IsDefault() ) + { + fail( argIdx, "test has fewer arguments than input specs" ); + } + continue; + } + + TestArgType type = test.argTypes[argIdx]; + if( type.mask && !spec.IsDefault() ) + { + fail( argIdx, "mask arguments are always generated from lane patterns, they take no input spec" ); + } + int maxBits = type.element == TestElement::Float ? FloatTraits::kSignificandBits : type.element == TestElement::Double ? FloatTraits::kSignificandBits : 0; + if( spec.significandBits < 0 || spec.significandBits > maxBits ) + { + fail( argIdx, maxBits ? "significand bits must be in [1, 24] for floats, [1, 53] for doubles" : "significand bits only apply to floats and doubles" ); + } + for( double special : spec.specials ) + { + if( std::isnan( special ) ) + { + fail( argIdx, "special values can't be NaN, the default spec includes NaNs" ); + } + } + if( ( spec.kind == InputSpec::Kind::NoNaN || spec.kind == InputSpec::Kind::Finite ) && type.element == TestElement::Int32 ) + { + fail( argIdx, "NoNaN and Finite only apply to floats and doubles" ); + } + if( spec.kind != InputSpec::Kind::Range && spec.kind != InputSpec::Kind::Magnitude ) + { + continue; + } + if( !std::isfinite( spec.lo ) || !std::isfinite( spec.hi ) || spec.lo > spec.hi ) + { + fail( argIdx, "range bounds must be finite with lo <= hi" ); + } + if( spec.kind == InputSpec::Kind::Magnitude && spec.lo <= 0 ) + { + fail( argIdx, "magnitude lo must be > 0" ); + } + if( type.element == TestElement::Float && ( std::abs( spec.lo ) > floatMax || std::abs( spec.hi ) > floatMax ) ) + { + fail( argIdx, "range bounds must be within float range" ); + } + if( type.element == TestElement::Int32 && + ( std::ceil( spec.lo ) > std::floor( spec.hi ) || std::ceil( spec.lo ) < -2147483648.0 || std::floor( spec.hi ) > 2147483647.0 ) ) + { + fail( argIdx, "int range must contain an integer and be within int32 range" ); + } + } +} + +template +static T RangeLo( const InputSpec& spec ) { return static_cast( spec.lo ); } +template +static T RangeHi( const InputSpec& spec ) { return static_cast( spec.hi ); } + +// Int bounds are the integers inside [lo, hi], ValidateInputs checks there is at least one +static int64_t IntLo( const InputSpec& spec ) { return static_cast( std::ceil( spec.lo ) ); } +static int64_t IntHi( const InputSpec& spec ) { return static_cast( std::floor( spec.hi ) ); } + +// Drops the low significand bits, finite values only +template +static T Truncate( T value, int significandBits ) +{ + constexpr int kBits = FloatTraits::kSignificandBits; + if( significandBits == 0 || significandBits >= kBits || !std::isfinite( value ) ) + { + return value; + } + using Bits = typename FloatTraits::Bits; + Bits lowBits = ( Bits( 1 ) << ( kBits - significandBits ) ) - 1; + return FromBits( ToBits( value ) & ~lowBits ); +} + +template +static bool InFloatDomain( const InputSpec& spec, T value ) +{ + switch( spec.kind ) + { + case InputSpec::Kind::Any: + return true; + case InputSpec::Kind::NoNaN: + return !std::isnan( value ); + case InputSpec::Kind::Finite: + return std::isfinite( value ); + case InputSpec::Kind::Range: + return value >= RangeLo( spec ) && value <= RangeHi( spec ); + case InputSpec::Kind::Magnitude: + return std::abs( value ) >= RangeLo( spec ) && std::abs( value ) <= RangeHi( spec ); + } + return false; +} + +// Values of every magnitude that commonly break SIMD code +static std::vector AnySpecials( const InputSpec& spec, float ) +{ + constexpr float inf = std::numeric_limits::infinity(); + constexpr float floatMin = std::numeric_limits::min(); + constexpr float denormMin = std::numeric_limits::denorm_min(); + const float denormMax = FromBits( 0x007FFFFF ); + + std::vector candidates = { + 0.0f, -0.0f, 1.0f, -1.0f, 0.5f, -0.5f, 1.5f, -1.5f, 2.5f, -2.5f, + 0.49999997f, -0.49999997f, 0.99999994f, -0.99999994f, 1.0000001f, -1.0000001f, + floatMin, -floatMin, denormMin, -denormMin, denormMax, -denormMax, + std::numeric_limits::max(), -std::numeric_limits::max(), + 8388607.5f, -8388607.5f, 8388608.0f, -8388608.0f, 16777216.0f, -16777216.0f, + 2147483520.0f, -2147483520.0f, 2147483648.0f, -2147483648.0f, + 3.14159274f, -3.14159274f, 1.e-10f, 1.e+10f, + }; + if( spec.kind != InputSpec::Kind::Finite ) + { + candidates.insert( candidates.end(), { inf, -inf } ); + } + if( spec.kind == InputSpec::Kind::Any ) + { + // Quiet, negative, signalling and payload NaNs + for( uint32_t bits : { 0x7FC00000u, 0xFFC00000u, 0x7F800001u, 0xFF800001u, 0x7FFFFFFFu, 0xFFC00001u } ) + { + candidates.push_back( FromBits( bits ) ); + } + } + return candidates; +} + +static std::vector AnySpecials( const InputSpec& spec, double ) +{ + constexpr double inf = std::numeric_limits::infinity(); + constexpr double doubleMin = std::numeric_limits::min(); + constexpr double denormMin = std::numeric_limits::denorm_min(); + const double denormMax = FromBits( 0x000FFFFFFFFFFFFFull ); + constexpr double floatMax = std::numeric_limits::max(); + constexpr double floatMin = std::numeric_limits::min(); + constexpr double floatDenormMin = std::numeric_limits::denorm_min(); + + std::vector candidates = { + 0.0, -0.0, 1.0, -1.0, 0.5, -0.5, 1.5, -1.5, 2.5, -2.5, + std::nextafter( 0.5, 0.0 ), -std::nextafter( 0.5, 0.0 ), std::nextafter( 1.0, 0.0 ), -std::nextafter( 1.0, 0.0 ), + std::nextafter( 1.0, 2.0 ), -std::nextafter( 1.0, 2.0 ), + doubleMin, -doubleMin, denormMin, -denormMin, denormMax, -denormMax, + std::numeric_limits::max(), -std::numeric_limits::max(), + // Integer precision limits, and the int32 convert range + 4503599627370495.5, -4503599627370495.5, 4503599627370496.0, -4503599627370496.0, 9007199254740992.0, -9007199254740992.0, + 2147483647.0, -2147483648.0, 2147483647.5, -2147483648.5, 2147483648.0, -2147483649.0, + // Narrowing to float: overflow, float denormals, and rounding ties + floatMax, -floatMax, 3.4028235677973366e+38, -3.4028235677973366e+38, floatMin, -floatMin, floatDenormMin, -floatDenormMin, + floatDenormMin / 2, 1.0 + 0x1p-24, -( 1.0 + 0x1p-24 ), 1.0 + 0x1.8p-23, 1.0 + 0x1p-23 + 0x1p-25, + 3.141592653589793, -3.141592653589793, 1.e-10, 1.e+10, 1.e-300, 1.e+300, + }; + if( spec.kind != InputSpec::Kind::Finite ) + { + candidates.insert( candidates.end(), { inf, -inf } ); + } + if( spec.kind == InputSpec::Kind::Any ) + { + // Quiet, negative, signalling and payload NaNs, including payloads narrowing to float drops + for( uint64_t bits : { 0x7FF8000000000000ull, 0xFFF8000000000000ull, 0x7FF0000000000001ull, 0xFFF0000000000001ull, + 0x7FFFFFFFFFFFFFFFull, 0xFFF8000000000001ull, 0x7FF4000000000000ull } ) + { + candidates.push_back( FromBits( bits ) ); + } + } + return candidates; +} + +template +static std::vector FloatSpecials( const InputSpec& spec ) +{ + constexpr T inf = std::numeric_limits::infinity(); + constexpr T typeMin = std::numeric_limits::min(); + constexpr T denormMin = std::numeric_limits::denorm_min(); + const T denormMax = std::nextafter( typeMin, T( 0 ) ); + const T belowHalf = std::nextafter( T( 0.5 ), T( 0 ) ); + const T belowOne = std::nextafter( T( 1 ), T( 0 ) ); + const T aboveOne = std::nextafter( T( 1 ), T( 2 ) ); + std::vector candidates; + + switch( spec.kind ) + { + case InputSpec::Kind::Any: + case InputSpec::Kind::NoNaN: + case InputSpec::Kind::Finite: + candidates = AnySpecials( spec, T() ); + break; + + case InputSpec::Kind::Range: + candidates = { RangeLo( spec ), RangeHi( spec ), std::nextafter( RangeLo( spec ), inf ), std::nextafter( RangeHi( spec ), -inf ) }; + for( T v : { T( 0 ), T( 0.5 ), T( 1 ), T( 1.5 ), T( 2.5 ), belowHalf, belowOne, aboveOne, typeMin, denormMin, denormMax, T( 1.e-10 ), T( 1.e-30 ) } ) + { + candidates.insert( candidates.end(), { v, -v } ); + } + break; + + case InputSpec::Kind::Magnitude: + for( T v : { RangeLo( spec ), RangeHi( spec ), std::nextafter( RangeLo( spec ), inf ), std::nextafter( RangeHi( spec ), -inf ), + T( 0.5 ), T( 1 ), T( 2 ), belowOne, aboveOne, typeMin, denormMin, denormMax } ) + { + candidates.insert( candidates.end(), { v, -v } ); + } + break; + } + + for( double special : spec.specials ) + { + T v = static_cast( special ); + candidates.insert( candidates.end(), { v, std::nextafter( v, inf ), std::nextafter( v, -inf ) } ); + } + + std::vector specials; + for( T candidate : candidates ) + { + T v = Truncate( candidate, spec.significandBits ); + + // Compare bits to keep both 0 and -0, and each NaN + bool duplicate = std::any_of( specials.begin(), specials.end(), [v]( T s ) { return ToBits( s ) == ToBits( v ); } ); + + if( InFloatDomain( spec, v ) && !duplicate ) + { + specials.push_back( v ); + } + } + if( specials.empty() ) + { + throw std::runtime_error( "Input spec has no special values in its domain" ); + } + return specials; +} + +static std::vector IntSpecials( const InputSpec& spec ) +{ + constexpr int32_t intMax = std::numeric_limits::max(); + constexpr int32_t intMin = std::numeric_limits::min(); + std::vector candidates; + + switch( spec.kind ) + { + case InputSpec::Kind::Any: + case InputSpec::Kind::NoNaN: + case InputSpec::Kind::Finite: + candidates = { + 0, 1, -1, 2, -2, 31, 32, 33, -31, -32, + intMax, intMin, intMax - 1, intMin + 1, + 0x7F800000, static_cast( 0xFF800000 ), 0x7FC00000, 0x7F800001, // inf, -inf, NaN, signalling NaN float bits + 0x3F800000, 0x00800000, 0x0000FFFF, 0x00010000, 0x55555555, static_cast( 0xAAAAAAAA ), + }; + break; + case InputSpec::Kind::Range: + candidates = { IntLo( spec ), IntHi( spec ), IntLo( spec ) + 1, IntHi( spec ) - 1, 0, 1, -1 }; + break; + case InputSpec::Kind::Magnitude: + for( int64_t v : { IntLo( spec ), IntHi( spec ), IntLo( spec ) + 1, IntHi( spec ) - 1, int64_t( 1 ) } ) + { + candidates.insert( candidates.end(), { v, -v } ); + } + break; + } + + for( double special : spec.specials ) + { + candidates.push_back( static_cast( std::round( std::min( std::max( special, -2147483648.0 ), 2147483647.0 ) ) ) ); + } + + std::vector specials; + for( int64_t v : candidates ) + { + bool inDomain = true; + switch( spec.kind ) + { + case InputSpec::Kind::Any: + case InputSpec::Kind::NoNaN: + case InputSpec::Kind::Finite: + inDomain = v >= intMin && v <= intMax; + break; + case InputSpec::Kind::Range: + inDomain = v >= IntLo( spec ) && v <= IntHi( spec ); + break; + case InputSpec::Kind::Magnitude: + inDomain = std::abs( v ) >= IntLo( spec ) && std::abs( v ) <= IntHi( spec ); + break; + } + + if( inDomain && std::find( specials.begin(), specials.end(), v ) == specials.end() ) + { + specials.push_back( static_cast( v ) ); + } + } + return specials; +} + +// Uniform [0, 1) +static double RandomUnit( std::mt19937_64& rng ) +{ + return static_cast( rng() >> 11 ) * ( 1.0 / 9007199254740992.0 ); +} + +// Log scaled magnitude within [lo, hi], lo > 0: a uniform power of 2 binade, then uniform within its part of [lo, hi] +template +static T RandomMagnitude( double lo, double hi, std::mt19937_64& rng ) +{ + int loExp, hiExp; + std::frexp( lo, &loExp ); + std::frexp( hi, &hiExp ); + + int exponent = loExp + static_cast( rng() % static_cast( hiExp - loExp + 1 ) ); + double binadeLo = std::max( lo, std::ldexp( 0.5, exponent ) ); + double binadeHi = std::min( hi, std::ldexp( 1.0, exponent ) ); + T value = static_cast( binadeLo + ( binadeHi - binadeLo ) * RandomUnit( rng ) ); + + return std::min( std::max( value, static_cast( lo ) ), static_cast( hi ) ); +} + +template +static T RandomFloat( const InputSpec& spec, std::mt19937_64& rng ) +{ + constexpr double denormMin = std::numeric_limits::denorm_min(); + + for( ;; ) + { + T value = 0; + + switch( spec.kind ) + { + case InputSpec::Kind::Any: + case InputSpec::Kind::NoNaN: + case InputSpec::Kind::Finite: + value = FromBits( static_cast::Bits>( rng() ) ); + break; + + case InputSpec::Kind::Range: + if( rng() & 1 ) + { + value = static_cast( spec.lo + ( spec.hi - spec.lo ) * RandomUnit( rng ) ); + } + else + { + // Log scaled half covers small magnitudes, including denormals, that uniform values almost never hit + bool positive = spec.lo >= 0 || ( spec.hi > 0 && ( rng() & 1 ) ); + double maxMag = positive ? spec.hi : -spec.lo; + double minMag = positive ? std::max( spec.lo, 0.0 ) : std::max( -spec.hi, 0.0 ); + value = RandomMagnitude( std::max( minMag, denormMin ), std::max( maxMag, denormMin ), rng ); + value = positive ? value : -value; + } + break; + + case InputSpec::Kind::Magnitude: + value = RandomMagnitude( spec.lo, spec.hi, rng ); + value = ( rng() & 1 ) ? -value : value; + break; + } + + value = Truncate( value, spec.significandBits ); + + if( InFloatDomain( spec, value ) ) + { + return value; + } + } +} + +static int32_t RandomInt( const InputSpec& spec, std::mt19937_64& rng ) +{ + switch( spec.kind ) + { + case InputSpec::Kind::Any: + case InputSpec::Kind::NoNaN: + case InputSpec::Kind::Finite: + return static_cast( rng() ); + + case InputSpec::Kind::Range: + return static_cast( IntLo( spec ) + static_cast( rng() % static_cast( IntHi( spec ) - IntLo( spec ) + 1 ) ) ); + + case InputSpec::Kind::Magnitude: + { + int64_t value = IntLo( spec ) + static_cast( rng() % static_cast( IntHi( spec ) - IntLo( spec ) + 1 ) ); + return static_cast( ( rng() & 1 ) ? -value : value ); + } + } + return 0; +} + +template +static std::vector Specials( const InputSpec& spec ) +{ + if constexpr( std::is_same_v ) + { + return IntSpecials( spec ); + } + else + { + return FloatSpecials( spec ); + } +} + +template +static T RandomValue( const InputSpec& spec, std::mt19937_64& rng ) +{ + if constexpr( std::is_same_v ) + { + return RandomInt( spec, rng ); + } + else + { + return RandomFloat( spec, rng ); + } +} + +InputGenerator::InputGenerator( uint64_t seed, std::string_view testName, const TestData& test ) : + mArgCount( test.argCount ), mLanes( test.lanes ), mSeed( seed ^ HashName( testName ) ), mCombinations( test.argCount ? 1 : 0 ) +{ + for( size_t argIdx = 0; argIdx < mArgCount; argIdx++ ) + { + Arg& arg = mArgs[argIdx]; + arg.type = test.argTypes[argIdx]; + arg.spec = test.inputs[argIdx]; + + switch( arg.type.element ) + { + case TestElement::Int32: + arg.values = MakeValues( arg ); + break; + case TestElement::Float: + arg.values = MakeValues( arg ); + break; + case TestElement::Double: + arg.values = MakeValues( arg ); + break; + } + + arg.specialCount = arg.type.mask ? MaskPatternCount() : std::visit( []( const auto& values ) { return values.specials.size(); }, arg.values ); + mCombinations *= arg.specialCount; + } +} + +size_t InputGenerator::SpecialCount() const +{ + return mCombinations * mLanes; +} + +const TestInputs& InputGenerator::Generate( size_t chunkStart ) +{ + for( size_t argIdx = 0; argIdx < mArgCount; argIdx++ ) + { + uint64_t state = mSeed + chunkStart * kMaxTestArgs + argIdx; + std::mt19937_64 rng( SplitMix64( state ) ); + + std::visit( [&]( auto& values ) + { + Fill( argIdx, values, chunkStart, rng ); + mInputs.values[argIdx] = values.chunk.data(); + }, mArgs[argIdx].values ); + } + return mInputs; +} + +template +InputGenerator::Values InputGenerator::MakeValues( const Arg& arg ) const +{ + Values values; + if( !arg.type.mask ) + { + values.specials = Specials( arg.spec ); + } + values.chunk.resize( kChunkSize ); + return values; +} + +template +void InputGenerator::Fill( size_t argIdx, Values& values, size_t chunkStart, std::mt19937_64& rng ) const +{ + const Arg& arg = mArgs[argIdx]; + + for( size_t i = 0; i < kChunkSize; i++ ) + { + size_t element = chunkStart + i; + bool special = element < SpecialCount(); + + if( arg.type.mask ) + { + int32_t lane = special ? MaskPatternLane( SpecialDigit( element, argIdx ), element % mLanes ) : static_cast( rng() & 1 ); + values.chunk[i] = static_cast( lane ); + } + else + { + values.chunk[i] = special ? values.specials[SpecialDigit( element, argIdx )] : RandomValue( arg.spec, rng ); + } + } +} + +// Value arguments use combination (call + lane), putting every combination in every lane and in lane 0 for scalar arguments +// Mask arguments use combination (call) so a whole mask pattern lands in one call +size_t InputGenerator::SpecialDigit( size_t element, size_t argIdx ) const +{ + size_t call = element / mLanes; + size_t combination = mArgs[argIdx].type.mask ? call : call + element % mLanes; + combination %= mCombinations; + + for( size_t i = 0; i < argIdx; i++ ) + { + combination /= mArgs[i].specialCount; + } + return combination % mArgs[argIdx].specialCount; +} + +// All clear, all set, each single lane set, each single lane clear, alternating, and each half set +size_t InputGenerator::MaskPatternCount() const +{ + return 2 + mLanes * 2 + 4; +} + +int32_t InputGenerator::MaskPatternLane( size_t pattern, size_t lane ) const +{ + if( pattern < 2 ) + { + return static_cast( pattern ); + } + pattern -= 2; + if( pattern < mLanes ) + { + return lane == pattern; + } + pattern -= mLanes; + if( pattern < mLanes ) + { + return lane != pattern; + } + pattern -= mLanes; + + switch( pattern ) + { + case 0: return ( lane & 1 ) == 0; + case 1: return ( lane & 1 ) == 1; + case 2: return lane < mLanes / 2; + default: return lane >= mLanes / 2; + } +} diff --git a/tests/test_inputs.h b/tests/test_inputs.h new file mode 100644 index 0000000..8f0280d --- /dev/null +++ b/tests/test_inputs.h @@ -0,0 +1,62 @@ +#pragma once +#include "test.h" + +#include +#include +#include +#include + +// Inputs are generated and tested in chunks to keep memory use flat +constexpr size_t kChunkSize = 1 << 14; + +static_assert( kChunkSize % kMaxTestLanes == 0, "Calls must not straddle input chunks" ); + +// Checks a test's input specs can be generated, mistakes here would silently test the wrong domain +void ValidateInputs( std::string_view testName, const TestData& test ); + +// Generates the input values for every argument of a test +// Only uses std::mt19937_64 and arithmetic without FMA contraction (see tests/CMakeLists.txt) so a seed produces the same values on any compiler +class InputGenerator +{ +public: + InputGenerator( uint64_t seed, std::string_view testName, const TestData& test ); + + // Inputs below this index are combinations of special values + size_t SpecialCount() const; + + // The kChunkSize input values per argument from chunkStart, valid until the next call + const TestInputs& Generate( size_t chunkStart ); + +private: + template + struct Values + { + std::vector specials; // Empty for masks, their special values are lane patterns + std::vector chunk; + }; + + struct Arg + { + TestArgType type = {}; + InputSpec spec; + size_t specialCount = 0; + std::variant, Values, Values> values; + }; + + template + Values MakeValues( const Arg& arg ) const; + + template + void Fill( size_t argIdx, Values& values, size_t chunkStart, std::mt19937_64& rng ) const; + + size_t SpecialDigit( size_t element, size_t argIdx ) const; + size_t MaskPatternCount() const; + int32_t MaskPatternLane( size_t pattern, size_t lane ) const; + + size_t mArgCount; + size_t mLanes; + uint64_t mSeed; + size_t mCombinations; + Arg mArgs[kMaxTestArgs]; + TestInputs mInputs = {}; +}; From 8beb3d1fecbb2c14646cf07d0e45e99d7cada80c Mon Sep 17 00:00:00 2001 From: Jordan Peck Date: Fri, 16 Oct 2026 22:22:13 +0100 Subject: [PATCH 4/4] Simplify the test harness types - One TestValue trait says how each kind of argument and result (scalar, register, mask register) is loaded, passed to the oracle and stored, replacing eight per shape trait families - Failure inputs and known failure lane arguments are read from the typed input buffers at runtime, TestArgType has each argument's lane count, so neither is instantiated for every test and tier any more - Collected tests keep the SCALAR non relaxed registration whole and only the test function of the other tiers. Tiers are checked against the reference while collecting, known failures are compared by value rather than count, and every test's inputs are validated, not only those matching --filter Inputs and results are unchanged for every seed. Checked against the previous harness with NaN payloads canonicalised on MSVC x64 (65536 and full count), Linux GCC/Clang and AArch64 GCC/Clang. Clang matches bit for bit, MSVC and GCC only differ in which NaN payload propagates. --- tests/test.cpp | 130 ++++++++------- tests/test.h | 17 +- tests/test.inl | 371 ++++++++++++------------------------------ tests/test_inputs.cpp | 94 +++++++++-- tests/test_inputs.h | 7 + 5 files changed, 270 insertions(+), 349 deletions(-) diff --git a/tests/test.cpp b/tests/test.cpp index a5745c6..de19e96 100644 --- a/tests/test.cpp +++ b/tests/test.cpp @@ -66,11 +66,6 @@ static std::string FeatureSetName( FastSIMD::FeatureSet featureSet ) return FastSIMD::GetFeatureSetString( featureSet ); } -static TestTier TierOf( const TestData& test ) -{ - return { test.featureSet, test.relaxed ? Tiers::Relaxed : Tiers::NonRelaxed }; -} - static std::string TierName( const TestTier& tier ) { if( tier.kind == Tiers::Oracle ) @@ -90,9 +85,58 @@ static std::string JoinNames( const std::vector& names ) return joined; } -// Each test name's registrations in tier order, the first is the SCALAR non relaxed reference -using TestSet = std::vector>>; +// A tier's own version of a test, everything else is the reference's +struct TierTest +{ + TestTier tier; + std::function testFunc; +}; +// A test's SCALAR non relaxed reference registration, and the test function of every other tier in tier order +struct Test +{ + std::string_view name; + TestData reference; + std::vector tiers; +}; + +using TestSet = std::vector; + +static void CheckReference( std::string_view testName, const TestData& reference ) +{ + for( const KnownFailure& knownFailure : reference.knownFailures ) + { + if( knownFailure.reason.empty() ) + { + throw std::runtime_error( "Known failure with no reason in \"" + std::string( testName ) + "\"" ); + } + } + ValidateInputs( testName, reference ); +} + +// Input conditions are a different function in each tier, so only whether there is one can be compared +static bool SameKnownFailure( const KnownFailure& a, const KnownFailure& b ) +{ + return a.reason == b.reason && a.tiers == b.tiers && a.minFeatureSet == b.minFeatureSet && a.maxFeatureSet == b.maxFeatureSet && + a.zeroSignOnly == b.zeroSignOnly && !a.inputCondition == !b.inputCondition; +} + +// Every tier's results are read as the reference's types and checked with the reference's settings, so a tier registering others is an error +static void CheckMatchesReference( const Test& test, const TestTier& tier, const TestData& registered ) +{ + const TestData& reference = test.reference; + + if( registered.returnType != reference.returnType || registered.lanes != reference.lanes || registered.argCount != reference.argCount || + !std::equal( registered.argTypes, registered.argTypes + kMaxTestArgs, reference.argTypes ) || + !std::equal( registered.knownFailures.begin(), registered.knownFailures.end(), reference.knownFailures.begin(), reference.knownFailures.end(), SameKnownFailure ) || + registered.bitExact != reference.bitExact || + registered.relaxedTolerance != reference.relaxedTolerance || registered.scalarRelaxedTolerance != reference.scalarRelaxedTolerance ) + { + throw std::runtime_error( "Tests do not match: " + std::string( test.name ) + " (" + TierName( tier ) + ")" ); + } +} + +// NewDispatchClass gives the highest compiled feature set up to the one asked for, every test library compiles TestFeatureSets so it's always that one template static void AddTier( TestSet& set, FastSIMD::FeatureSet featureSet ) { @@ -103,6 +147,7 @@ static void AddTier( TestSet& set, FastSIMD::FeatureSet featureSet ) throw std::runtime_error( "No test class created for " + FeatureSetName( featureSet ) ); } + TestTier tier = { featureSet, Relaxed ? Tiers::Relaxed : Tiers::NonRelaxed }; bool isReference = featureSet == FastSIMD::FeatureSet::SCALAR && !Relaxed; for( auto& registered : testClass->RegisterTests() ) @@ -110,12 +155,7 @@ static void AddTier( TestSet& set, FastSIMD::FeatureSet featureSet ) std::string_view testName = registered.first; TestData& test = registered.second; - if( test.featureSet != featureSet || test.relaxed != Relaxed ) - { - throw std::runtime_error( "Test class for " + FeatureSetName( featureSet ) + " registered \"" + std::string( testName ) + "\" for " + FeatureSetName( test.featureSet ) ); - } - - auto find = std::find_if( set.begin(), set.end(), [testName]( const auto& pair ) { return pair.first == testName; } ); + auto find = std::find_if( set.begin(), set.end(), [testName]( const Test& other ) { return other.name == testName; } ); if( isReference ) { @@ -123,7 +163,8 @@ static void AddTier( TestSet& set, FastSIMD::FeatureSet featureSet ) { throw std::runtime_error( "Multiple tests with same name: " + std::string( testName ) ); } - set.emplace_back( testName, std::vector{ std::move( test ) } ); + CheckReference( testName, test ); + set.push_back( { testName, std::move( test ), {} } ); continue; } @@ -131,11 +172,12 @@ static void AddTier( TestSet& set, FastSIMD::FeatureSet featureSet ) { throw std::runtime_error( "\"" + std::string( testName ) + "\" has no SCALAR non relaxed reference" ); } - if( std::any_of( find->second.begin(), find->second.end(), [&]( const TestData& other ) { return other.featureSet == featureSet && other.relaxed == Relaxed; } ) ) + if( std::any_of( find->tiers.begin(), find->tiers.end(), [&]( const TierTest& other ) { return other.tier.featureSet == featureSet && other.tier.kind == tier.kind; } ) ) { - throw std::runtime_error( "Multiple tests with same name: " + std::string( testName ) + " (" + TierName( TierOf( test ) ) + ")" ); + throw std::runtime_error( "Multiple tests with same name: " + std::string( testName ) + " (" + TierName( tier ) + ")" ); } - find->second.push_back( std::move( test ) ); + CheckMatchesReference( *find, tier, test ); + find->tiers.push_back( { tier, std::move( test.testFunc ) } ); } } @@ -158,33 +200,6 @@ static TestSet CollectTests( FastSIMD::FeatureSet maxFeatureSet ) return set; } -// Every tier's results are read as the reference's types, and only the reference's settings are used, so a tier registering others is an error -static void CheckTests( std::string_view testName, const std::vector& tests ) -{ - const TestData& reference = tests[0]; - - for( size_t testIdx = 1; testIdx < tests.size(); testIdx++ ) - { - const TestData& test = tests[testIdx]; - - if( test.returnType != reference.returnType || test.lanes != reference.lanes || test.argCount != reference.argCount || - !std::equal( test.argTypes, test.argTypes + kMaxTestArgs, reference.argTypes ) || - test.knownFailures.size() != reference.knownFailures.size() || test.bitExact != reference.bitExact || - test.relaxedTolerance != reference.relaxedTolerance || test.scalarRelaxedTolerance != reference.scalarRelaxedTolerance ) - { - throw std::runtime_error( "Tests do not match: " + std::string( testName ) + " (" + TierName( TierOf( test ) ) + ")" ); - } - } - - for( const KnownFailure& knownFailure : reference.knownFailures ) - { - if( knownFailure.reason.empty() ) - { - throw std::runtime_error( "Known failure with no reason in \"" + std::string( testName ) + "\"" ); - } - } -} - // Value of the least significant bit, relaxed differences are counted in these // Zero and denormals use the smallest normal rather than the denormal spacing, so relaxed results may flush denormals template @@ -340,8 +355,8 @@ struct CallResult class TestRun { public: - TestRun( size_t registerBytes, std::string_view testName, const std::vector& tests ) : - mWidth( std::to_string( registerBytes * 8 ) + "bit " ), mTestName( testName ), mReference( tests[0] ), mKnownSeen( tests[0].knownFailures.size() ) + TestRun( size_t registerBytes, const Test& test ) : + mWidth( std::to_string( registerBytes * 8 ) + "bit " ), mTestName( test.name ), mReference( test.reference ), mKnownSeen( test.reference.knownFailures.size() ) { auto addComparison = [this]( TestTier tier, const std::function& testFunc ) { @@ -355,9 +370,9 @@ class TestRun { addComparison( { FastSIMD::FeatureSet::SCALAR, Tiers::Oracle }, mReference.oracleFunc ); } - for( size_t testIdx = 1; testIdx < tests.size(); testIdx++ ) + for( const TierTest& tierTest : test.tiers ) { - addComparison( TierOf( tests[testIdx] ), tests[testIdx].testFunc ); + addComparison( tierTest.tier, tierTest.testFunc ); } } @@ -481,7 +496,7 @@ class TestRun { std::cerr << std::setprecision( 16 ) << "--- " << mWidth << mTestName << " " << tier.name << " FAILED ---" << std::endl; std::cerr << call.details; - std::cerr << "Inputs (index " << chunkStart + idx << "): " << mReference.inputsFunc( idx, inputs ) << std::endl; + std::cerr << "Inputs (index " << chunkStart + idx << "): " << DescribeInputs( mReference, inputs, idx ) << std::endl; } tier.failures++; } @@ -511,7 +526,7 @@ class TestRun continue; } - LaneArgs laneArgs = mReference.laneArgsFunc( inIdx, idx, inputs ); + LaneArgs laneArgs = GetLaneArgs( mReference, inputs, inIdx, idx ); auto known = std::find_if( knownFailures.begin(), knownFailures.end(), [&]( const KnownFailure& knownFailure ) { return knownFailure.Matches( tier, lane.result == LaneResult::ZeroSign, laneArgs ); @@ -543,14 +558,11 @@ class TestRun std::vector> mKnownSeen; // Tiers each known failure was seen on }; -static void DoTest( size_t registerBytes, std::string_view testName, const std::vector& tests, const TestOptions& options, TestSummary& summary ) +static void DoTest( size_t registerBytes, const Test& test, const TestOptions& options, TestSummary& summary ) { - std::cout << "Testing: " << testName << std::endl; - - CheckTests( testName, tests ); - ValidateInputs( testName, tests[0] ); + std::cout << "Testing: " << test.name << std::endl; - TestRun run( registerBytes, testName, tests ); + TestRun run( registerBytes, test ); run.Run( options ); run.Report( options, summary ); } @@ -565,11 +577,11 @@ static void RunRegisterSize( const TestOptions& options, TestSummary& summary ) std::cout << "Starting Tests - Register Size: " << RegisterBytes * 8 << " (" << RegisterBytes << "b)" << std::endl; - for( const auto& [testName, tests] : CollectTests( options.maxFeatureSet ) ) + for( const Test& test : CollectTests( options.maxFeatureSet ) ) { - if( testName.find( options.filter ) != std::string_view::npos ) + if( test.name.find( options.filter ) != std::string_view::npos ) { - DoTest( RegisterBytes, testName, tests, options, summary ); + DoTest( RegisterBytes, test, options, summary ); } } } diff --git a/tests/test.h b/tests/test.h index c2c35e8..0019398 100644 --- a/tests/test.h +++ b/tests/test.h @@ -63,9 +63,10 @@ enum class TestElement : uint8_t struct TestArgType { TestElement element; - bool mask; // Lanes are 0 or 1 in the element type, from lane patterns rather than an input spec, compared != 0 to make the mask + bool mask; // Lanes are 0 or 1 in the element type, from lane patterns rather than an input spec, compared != 0 to make the mask + size_t lanes; // Values the argument takes per call, 1 for scalars which are the same value in every lane - bool operator==( const TestArgType& other ) const { return element == other.element && mask == other.mask; } + bool operator==( const TestArgType& other ) const { return element == other.element && mask == other.mask && lanes == other.lanes; } }; // One pointer per argument to its values for the current chunk, indexed by input index and stored as the argument's TestElement @@ -80,7 +81,7 @@ struct TestInputs } }; -// A lane's input values, for known failure conditions. Arguments of other types are 0 +// A lane's input values as the oracle sees them, for known failure conditions. Arguments of other types are 0 struct LaneArgs { int32_t ints[kMaxTestArgs]; // Masks are 0 or 1 @@ -134,8 +135,6 @@ struct InputSpec }; using TestFunction = size_t ( void* outResult, size_t inIdx, const TestInputs& inputs ); -using InputsFunction = std::string ( size_t inIdx, const TestInputs& inputs ); -using LaneArgsFunction = LaneArgs ( size_t inIdx, size_t lane, const TestInputs& inputs ); // Kinds of tier, each tier is one kind and known failures are scoped to a set of them enum class Tiers : uint8_t @@ -224,8 +223,8 @@ struct Tolerance bool operator!=( const Tolerance& other ) const { return !( *this == other ); } }; -// A test as registered by one feature set, plain or relaxed -// Only the SCALAR non relaxed registration's settings (inputs, tolerances, known failures, oracle) are used, every tier must register the same ones +// A test as registered by one tier +// Only the SCALAR non relaxed registration is kept whole, the other tiers must register the same settings and only their test function is kept struct TestData { enum class ReturnType @@ -233,8 +232,6 @@ struct TestData boolean, f32, f64, i32, i64 // i64 holds 64bit mask lanes }; - FastSIMD::FeatureSet featureSet; - bool relaxed; ReturnType returnType; size_t lanes = 0; // Elements per call, the widest register in the test sets it size_t argCount = 0; @@ -246,8 +243,6 @@ struct TestData std::vector knownFailures; std::function testFunc; std::function oracleFunc; // Optional plain C++ version of the test, the SCALAR reference must match it exactly - std::function inputsFunc; - std::function laneArgsFunc; template TestData& Inputs( SPECs... specs ) diff --git a/tests/test.inl b/tests/test.inl index b6a205d..13d0bcf 100644 --- a/tests/test.inl +++ b/tests/test.inl @@ -5,10 +5,7 @@ #include #include #include -#include -#include #include -#include #include template @@ -38,53 +35,23 @@ template decltype( lambda_pack_helper( &T::operator() ) ) pack_helper( T ); -template -struct GetReturn; - -template<> -struct GetReturn -{ - static constexpr auto Type = TestData::ReturnType::i32; -}; - -template<> -struct GetReturn -{ - static constexpr auto Type = TestData::ReturnType::f32; -}; - -template<> -struct GetReturn -{ - static constexpr auto Type = TestData::ReturnType::f64; -}; - -template<> -struct GetReturn -{ - static constexpr auto Type = TestData::ReturnType::boolean; -}; - -// Mask results are stored as integer lanes of the mask's width so every bit of each lane is compared -template -struct GetReturn> -{ - static_assert( N == 32 || N == 64, "Mask results are stored as 32 or 64bit lanes" ); - static constexpr auto Type = N == 32 ? TestData::ReturnType::i32 : TestData::ReturnType::i64; -}; - -template -struct GetReturn> +// The type an argument's inputs are stored as +template +constexpr TestElement ElementOf() { - static constexpr auto Type = GetReturn::Type; -}; + static_assert( std::is_same_v || std::is_same_v || std::is_same_v, "No test inputs for type" ); + return std::is_same_v ? TestElement::Float : std::is_same_v ? TestElement::Double : TestElement::Int32; +} -// Elements per call of a test argument or result, scalars are 1 +// The type a result's lanes are stored and compared as template -static constexpr size_t LaneCount = 1; - -template -static constexpr size_t LaneCount> = N; +constexpr TestData::ReturnType ReturnTypeOf() +{ + static_assert( std::is_same_v || std::is_same_v || std::is_same_v || std::is_same_v || std::is_same_v, + "No test results for type" ); + return std::is_same_v ? TestData::ReturnType::boolean : std::is_same_v ? TestData::ReturnType::f32 : + std::is_same_v ? TestData::ReturnType::f64 : std::is_same_v ? TestData::ReturnType::i64 : TestData::ReturnType::i32; +} template @@ -94,223 +61,129 @@ class FastSIMD::DispatchClass, SIMD> : publ using TestReg = FS::Register; - template - static std::string AsString( T arg ) - { - std::stringstream ss; - ss << std::setprecision( 17 ) << std::boolalpha << arg; - return ss.str(); - } - - template - static std::string AsString( const FS::Register& arg ) + // Stores an oracle lane as T, float/double lanes can be given as their bits in an unsigned int + // Any unsigned type of the right size, uint64_t is unsigned long on LP64 so 64bit literals have a different type + template + static void StoreOracleLane( void* outResult, size_t lane, V value ) { - T scalarValues[N]; - FS::Store( scalarValues, arg ); - - std::ostringstream sstr; - sstr << "[" << AsString( scalarValues[0] ); - - for( size_t idx = 1; idx < N; idx++ ) + if constexpr( std::is_floating_point_v && std::is_integral_v && std::is_unsigned_v && sizeof( V ) == sizeof( T ) ) { - sstr << ", " << AsString( scalarValues[idx] ); + std::memcpy( static_cast( outResult ) + lane, &value, sizeof( T ) ); } - - sstr << "]"; - return sstr.str(); - } - - template - static std::string AsString( const FS::Register, N, S>& arg ) - { - FS::BitStorage bitMask = FS::BitMask( arg ); - - std::ostringstream sstr; - sstr << "[" << AsString( static_cast( bitMask & 1 ) ); - - for( size_t idx = 1; idx < N; idx++ ) + else { - bitMask >>= 1; - - sstr << ", " << AsString( static_cast( bitMask & 1 ) ); + static_assert( std::is_same_v, "Oracle must return the test's element type" ); + static_cast( outResult )[lane] = value; } - - sstr << "]"; - return sstr.str(); } - + // How each kind of test argument and result is read from the inputs, passed to the oracle and stored as results + // Load is a call's argument, Lane is one lane's argument for the oracle, Store and StoreLane write results as Element + // Scalars are one value, the oracle sees the same value in every lane template - static size_t StoreOutput( void* outResult, T result ) - { - *reinterpret_cast( outResult ) = result; - return 1; - } - - template - static size_t StoreOutput( void* outResult, FS::Register result ) + struct TestValue { - static_assert( N * sizeof( T ) <= kMaxTestResultBytes, "Test result is larger than the result buffer" ); - FS::Store( reinterpret_cast( outResult ), result ); - return N; - } + using Element = T; + static constexpr size_t Lanes = 1; - template - static size_t StoreOutput( void* outResult, FS::Register, N, S> result ) - { - static_assert( N * MN / 8 <= kMaxTestResultBytes, "Test result is larger than the result buffer" ); + static constexpr TestArgType ArgType() { return { ElementOf(), false, Lanes }; } - // Store all bits of each lane, BitMask only shows the high bit - if constexpr( MN == 32 ) + static T Load( const TestInputs& inputs, size_t argIdx, size_t inIdx ) { - FS::Store( reinterpret_cast( outResult ), FS::Masked( result, FS::Register( -1 ) ) ); + return inputs.Get( argIdx )[inIdx]; } - else + + static T Lane( const TestInputs& inputs, size_t argIdx, size_t inIdx, size_t ) { - // No 64bit int registers, a double with every bit set keeps all the lane's bits through Masked - static_assert( MN == 64 && MB, "Mask results are stored as 32 or 64bit lanes" ); - FS::Store( reinterpret_cast( outResult ), FS::Masked( result, FS::Register( FromBits( ~uint64_t( 0 ) ) ) ) ); + return Load( inputs, argIdx, inIdx ); } - return N; - } + static void Store( void* outResult, T result ) + { + *static_cast( outResult ) = result; + } - template - static constexpr TestElement ElementOf() - { - static_assert( std::is_same_v || std::is_same_v || std::is_same_v, "No test inputs for type" ); - return std::is_same_v ? TestElement::Float : std::is_same_v ? TestElement::Double : TestElement::Int32; - } - - // A mask argument's 0/1 lanes are stored as, and compared != 0 in: double for 64bit masks, float for float masks, int32 for int masks - template - using MaskElement = std::conditional_t>; - - template - struct ArgType - { - static constexpr TestArgType Value = { ElementOf(), false }; + template + static void StoreLane( void* outResult, size_t lane, V value ) + { + StoreOracleLane( outResult, lane, value ); + } }; template - struct ArgType> : ArgType {}; - - template - struct ArgType, N, S>> + struct TestValue> { - static_assert( MN == 32 || ( MN == 64 && MB ), "Mask inputs are generated as 32bit lanes, or 64bit from doubles" ); + static_assert( N * sizeof( T ) <= kMaxTestResultBytes, "Test result is larger than the result buffer" ); - static constexpr TestArgType Value = { ElementOf>(), true }; - }; + using Element = T; + static constexpr size_t Lanes = N; - template - struct GenArg - { - static T Load( size_t inIdx, size_t argIdx, const TestInputs& inputs ) - { - return inputs.Get( argIdx )[inIdx]; - } - }; + static constexpr TestArgType ArgType() { return { ElementOf(), false, Lanes }; } - template - struct GenArg> - { - static FS::Register Load( size_t inIdx, size_t argIdx, const TestInputs& inputs ) + static FS::Register Load( const TestInputs& inputs, size_t argIdx, size_t inIdx ) { return FS::Load( inputs.Get( argIdx ) + inIdx ); } - }; - // Each mask lane is generated independently, then compared != 0 in its stored type to make the mask - template - struct GenArg, N, S>> - { - static FS::Register, N, S> Load( size_t inIdx, size_t argIdx, const TestInputs& inputs ) + static T Lane( const TestInputs& inputs, size_t argIdx, size_t inIdx, size_t lane ) { - using Element = MaskElement; - return FS::Load( inputs.Get( argIdx ) + inIdx ) != FS::Register( Element( 0 ) ); + return inputs.Get( argIdx )[inIdx + lane]; } - }; - // Argument values for one lane, as plain C++ types for the oracle - template - struct OracleArg - { - // Scalar arguments are the same for every lane - static T Get( const TestInputs& inputs, size_t argIdx, size_t inIdx, size_t ) + static void Store( void* outResult, FS::Register result ) { - return inputs.Get( argIdx )[inIdx]; + FS::Store( static_cast( outResult ), result ); } - }; - template - struct OracleArg> - { - static T Get( const TestInputs& inputs, size_t argIdx, size_t inIdx, size_t lane ) + template + static void StoreLane( void* outResult, size_t lane, V value ) { - return inputs.Get( argIdx )[inIdx + lane]; + StoreOracleLane( outResult, lane, value ); } }; + // Mask lanes are generated independently as 0 or 1, then compared != 0 to make the mask. The oracle sees them as bool + // Results are stored as integer lanes of the mask's width so every bit of each lane is compared template - struct OracleArg, N, S>> + struct TestValue, N, S>> { - static bool Get( const TestInputs& inputs, size_t argIdx, size_t inIdx, size_t lane ) - { - return inputs.Get>( argIdx )[inIdx + lane] != 0; - } - }; + static_assert( MN == 32 || ( MN == 64 && MB ), "Masks are tested as 32bit lanes, or 64bit from doubles" ); - // Stores an oracle lane in the same format as StoreOutput, float/double lanes can be given as their bits in an unsigned int - // Any unsigned type of the right size, uint64_t is unsigned long on LP64 so 64bit literals have a different type - template - static void StoreOracleLane( void* outResult, size_t lane, V value ) - { - if constexpr( std::is_floating_point_v && std::is_integral_v && std::is_unsigned_v && sizeof( V ) == sizeof( T ) ) + // Inputs are stored as double for 64bit masks, float for float masks, int32 for int masks + using Stored = std::conditional_t>; + using Element = std::conditional_t; + static constexpr size_t Lanes = N; + + static constexpr TestArgType ArgType() { return { ElementOf(), true, Lanes }; } + + static FS::Register, N, S> Load( const TestInputs& inputs, size_t argIdx, size_t inIdx ) { - std::memcpy( reinterpret_cast( outResult ) + lane, &value, sizeof( T ) ); + return FS::Load( inputs.Get( argIdx ) + inIdx ) != FS::Register( Stored( 0 ) ); } - else + + static bool Lane( const TestInputs& inputs, size_t argIdx, size_t inIdx, size_t lane ) { - static_assert( std::is_same_v, "Oracle must return the test's element type" ); - reinterpret_cast( outResult )[lane] = value; + return inputs.Get( argIdx )[inIdx + lane] != 0; } - } - - template - struct OracleOutput - { - static constexpr size_t Lanes = 1; - - template - static void Store( void* outResult, size_t lane, V value ) { StoreOracleLane( outResult, lane, value ); } - }; - - template - struct OracleOutput> - { - static constexpr size_t Lanes = N; - template - static void Store( void* outResult, size_t lane, V value ) { StoreOracleLane( outResult, lane, value ); } - }; - - template - struct OracleOutput, N, S>> - { - static constexpr size_t Lanes = N; - - static void Store( void* outResult, size_t lane, bool value ) + // BitMask only shows the high bit of each lane, Masked keeps all of them + static void Store( void* outResult, FS::Register, N, S> result ) { if constexpr( MN == 32 ) { - reinterpret_cast( outResult )[lane] = value ? -1 : 0; + FS::Store( static_cast( outResult ), FS::Masked( result, FS::Register( -1 ) ) ); } else { - reinterpret_cast( outResult )[lane] = value ? -1 : 0; + // No 64bit int registers, a double with every bit set keeps all the lane's bits through Masked + FS::Store( static_cast( outResult ), FS::Masked( result, FS::Register( FromBits( ~uint64_t( 0 ) ) ) ) ); } } + + static void StoreLane( void* outResult, size_t lane, bool value ) + { + static_cast( outResult )[lane] = value ? -1 : 0; + } }; @@ -320,9 +193,9 @@ class FastSIMD::DispatchClass, SIMD> : publ template struct TestFunctionFactory> { - using ReturnType = std::invoke_result_t; + using Result = TestValue>; static constexpr size_t ArgCount = sizeof...( ARGs ); - static constexpr size_t Lanes = std::max( { LaneCount, LaneCount>... } ); + static constexpr size_t Lanes = std::max( { Result::Lanes, TestValue>::Lanes... } ); static_assert( Lanes <= kMaxTestLanes, "Too many lanes per call" ); @@ -331,17 +204,18 @@ class FastSIMD::DispatchClass, SIMD> : publ static void ArgTypes( TestArgType* argTypes ) { [[maybe_unused]] size_t argIdx = 0; - ( ( argTypes[argIdx++] = ArgType>::Value ), ... ); + ( ( argTypes[argIdx++] = TestValue>::ArgType() ), ... ); } static std::function Create( FUNC func ) { - return [func]( void* outResult, size_t inIdx, const TestInputs& inputs ) + return [func]( void* outResult, size_t inIdx, const TestInputs& inputs ) -> size_t { - size_t argIdx = 0; - auto tuple = std::tuple{ GenArg>::Load( inIdx, argIdx++, inputs )... }; + [[maybe_unused]] size_t argIdx = 0; + auto tuple = std::tuple{ TestValue>::Load( inputs, argIdx++, inIdx )... }; - return StoreOutput( outResult, std::apply( func, tuple ) ); + Result::Store( outResult, std::apply( func, tuple ) ); + return Result::Lanes; }; } @@ -351,48 +225,14 @@ class FastSIMD::DispatchClass, SIMD> : publ { return [oracle]( void* outResult, size_t inIdx, const TestInputs& inputs ) -> size_t { - using Output = OracleOutput; - - for( size_t lane = 0; lane < Output::Lanes; lane++ ) + for( size_t lane = 0; lane < Result::Lanes; lane++ ) { [[maybe_unused]] size_t argIdx = 0; - auto tuple = std::tuple{ OracleArg>::Get( inputs, argIdx++, inIdx, lane )... }; + auto tuple = std::tuple{ TestValue>::Lane( inputs, argIdx++, inIdx, lane )... }; - Output::Store( outResult, lane, std::apply( oracle, tuple ) ); + Result::StoreLane( outResult, lane, std::apply( oracle, tuple ) ); } - return Output::Lanes; - }; - } - - // A lane's argument values as the oracle sees them, for known failure conditions - static LaneArgs GetLaneArgs( [[maybe_unused]] size_t inIdx, [[maybe_unused]] size_t lane, [[maybe_unused]] const TestInputs& inputs ) - { - LaneArgs args = {}; - [[maybe_unused]] size_t argIdx = 0; - ( ( args.Set( argIdx, OracleArg>::Get( inputs, argIdx, inIdx, lane ) ), argIdx++ ), ... ); - return args; - } - - static std::function GetInputs() - { - return []( size_t inIdx, const TestInputs& inputs ) -> std::string - { - if( !sizeof...( ARGs ) ) - { - return "No Inputs"; - } - - size_t argIdx = 0; - std::initializer_list tuple = { AsString( GenArg>::Load( inIdx, argIdx++, inputs ) )... }; - - std::ostringstream inputsString; - inputsString << "\n"; - - std::copy( tuple.begin(), tuple.end() - 1, std::ostream_iterator( inputsString, "\n" ) ); - - inputsString << *(tuple.end() - 1); - - return inputsString.str(); + return Result::Lanes; }; } }; @@ -401,19 +241,16 @@ class FastSIMD::DispatchClass, SIMD> : publ template TestData& RegisterTest( TestCollection& testMap, const char* name, FUNC func ) { + using Factory = TestFunctionFactory; + TestData data; - data.featureSet = SIMD; - data.relaxed = FastSIMD::IsRelaxed(); - data.returnType = GetReturn::ReturnType>::Type; - data.lanes = TestFunctionFactory::Lanes; - data.argCount = TestFunctionFactory::ArgCount; - TestFunctionFactory::ArgTypes( data.argTypes ); - - data.testFunc = TestFunctionFactory::Create( func ); - data.inputsFunc = TestFunctionFactory::GetInputs(); - data.laneArgsFunc = TestFunctionFactory::GetLaneArgs; - - return testMap.emplace_back( name, data ).second; + data.returnType = ReturnTypeOf(); + data.lanes = Factory::Lanes; + data.argCount = Factory::ArgCount; + Factory::ArgTypes( data.argTypes ); + data.testFunc = Factory::Create( func ); + + return testMap.emplace_back( name, std::move( data ) ).second; } // oracle is a plain C++ version of the test, called per lane with plain values (masks as bool) diff --git a/tests/test_inputs.cpp b/tests/test_inputs.cpp index 6a5a826..984af37 100644 --- a/tests/test_inputs.cpp +++ b/tests/test_inputs.cpp @@ -3,11 +3,28 @@ #include #include #include +#include #include #include +#include #include #include +// Calls func with a value of the type an argument's inputs are stored as +template +static void WithStoredType( TestElement element, FUNC&& func ) +{ + switch( element ) + { + case TestElement::Int32: + return func( int32_t() ); + case TestElement::Float: + return func( float() ); + case TestElement::Double: + return func( double() ); + } +} + static uint64_t SplitMix64( uint64_t& state ) { uint64_t z = ( state += 0x9E3779B97F4A7C15ull ); @@ -96,6 +113,70 @@ void ValidateInputs( std::string_view testName, const TestData& test ) } } +std::string DescribeInputs( const TestData& test, const TestInputs& inputs, size_t inIdx ) +{ + if( !test.argCount ) + { + return "No Inputs"; + } + + std::ostringstream description; + description << std::setprecision( 17 ) << std::boolalpha; + + for( size_t argIdx = 0; argIdx < test.argCount; argIdx++ ) + { + TestArgType type = test.argTypes[argIdx]; + bool isRegister = type.lanes > 1; + + // Registers list their lanes, mask lanes are true or false + WithStoredType( type.element, [&]( auto stored ) + { + const auto* values = inputs.Get( argIdx ) + inIdx; + + description << "\n" << ( isRegister ? "[" : "" ); + for( size_t lane = 0; lane < type.lanes; lane++ ) + { + description << ( lane ? ", " : "" ); + if( type.mask ) + { + description << ( values[lane] != 0 ); + } + else + { + description << values[lane]; + } + } + description << ( isRegister ? "]" : "" ); + } ); + } + return description.str(); +} + +LaneArgs GetLaneArgs( const TestData& test, const TestInputs& inputs, size_t inIdx, size_t lane ) +{ + LaneArgs args = {}; + + for( size_t argIdx = 0; argIdx < test.argCount; argIdx++ ) + { + TestArgType type = test.argTypes[argIdx]; + size_t idx = inIdx + ( type.lanes > 1 ? lane : 0 ); + + WithStoredType( type.element, [&]( auto stored ) + { + auto value = inputs.Get( argIdx )[idx]; + if( type.mask ) + { + args.Set( argIdx, value != 0 ); + } + else + { + args.Set( argIdx, value ); + } + } ); + } + return args; +} + template static T RangeLo( const InputSpec& spec ) { return static_cast( spec.lo ); } template @@ -460,18 +541,7 @@ InputGenerator::InputGenerator( uint64_t seed, std::string_view testName, const arg.type = test.argTypes[argIdx]; arg.spec = test.inputs[argIdx]; - switch( arg.type.element ) - { - case TestElement::Int32: - arg.values = MakeValues( arg ); - break; - case TestElement::Float: - arg.values = MakeValues( arg ); - break; - case TestElement::Double: - arg.values = MakeValues( arg ); - break; - } + WithStoredType( arg.type.element, [&]( auto stored ) { arg.values = MakeValues( arg ); } ); arg.specialCount = arg.type.mask ? MaskPatternCount() : std::visit( []( const auto& values ) { return values.specials.size(); }, arg.values ); mCombinations *= arg.specialCount; diff --git a/tests/test_inputs.h b/tests/test_inputs.h index 8f0280d..be3eb82 100644 --- a/tests/test_inputs.h +++ b/tests/test_inputs.h @@ -2,6 +2,7 @@ #include "test.h" #include +#include #include #include #include @@ -14,6 +15,12 @@ static_assert( kChunkSize % kMaxTestLanes == 0, "Calls must not straddle input c // Checks a test's input specs can be generated, mistakes here would silently test the wrong domain void ValidateInputs( std::string_view testName, const TestData& test ); +// A call's input values as text, one line per argument, for failure messages +std::string DescribeInputs( const TestData& test, const TestInputs& inputs, size_t inIdx ); + +// A lane's argument values as the oracle sees them (TestValue::Lane in test.inl), for known failure conditions +LaneArgs GetLaneArgs( const TestData& test, const TestInputs& inputs, size_t inIdx, size_t lane ); + // Generates the input values for every argument of a test // Only uses std::mt19937_64 and arithmetic without FMA contraction (see tests/CMakeLists.txt) so a seed produces the same values on any compiler class InputGenerator