diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bf53b89..d1999b2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -64,3 +64,6 @@ jobs: - name: 'CMake Build Release' run: cmake --build ${{ github.workspace }}/release --config Release --parallel 4 + + - name: 'Test Release' + run: ctest --test-dir ${{ github.workspace }}/release -C Release --output-on-failure diff --git a/CMakeLists.txt b/CMakeLists.txt index 0d44afc..c6be961 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -44,6 +44,7 @@ if(FASTSIMD_DISPATCH_CLASS) endif() if(FASTSIMD_TESTS) + enable_testing() add_subdirectory(tests) endif() diff --git a/include/FastSIMD/ToolSet/x86/128/f32x4.h b/include/FastSIMD/ToolSet/x86/128/f32x4.h index badd308..7667a8c 100644 --- a/include/FastSIMD/ToolSet/x86/128/f32x4.h +++ b/include/FastSIMD/ToolSet/x86/128/f32x4.h @@ -224,7 +224,8 @@ namespace FS } else { - return _mm_xor_ps( ifFalse.native, _mm_and_ps( mask.native, _mm_xor_ps( ifTrue.native, ifFalse.native ) ) ); + // MSVC 19.51 miscompiles the equivalent xor( ifFalse, and( mask, xor( ifTrue, ifFalse ) ) ) inside FS::ATan2 + return _mm_xor_ps( ifTrue.native, _mm_andnot_ps( mask.native, _mm_xor_ps( ifTrue.native, ifFalse.native ) ) ); } } diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 15bc605..b6d817a 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1,13 +1,60 @@ -fastsimd_create_dispatch_library(simd_test SOURCES "test.inl" FEATURE_SETS SCALAR SSE2 SSE41 AVX2 AVX512 NEON AARCH64 WASM) -fastsimd_create_dispatch_library(simd_test_relaxed RELAXED SOURCES "test.inl" FEATURE_SETS SCALAR SSE2 SSE41 AVX2 AVX512 NEON AARCH64 WASM) +set(FASTSIMD_TEST_FEATURE_SETS SCALAR SSE2 SSE41 AVX2 AVX512 NEON AARCH64 WASM) -add_executable(test "test.cpp") -target_link_libraries(test PRIVATE FastSIMD simd_test simd_test_relaxed) +# Each register size is its own pair of dispatch libraries, test.inl instantiates FASTSIMD_TEST_REGISTER_BYTES +# All sizes in one translation unit needs ~4.5GB per unit in a GCC Debug build, too much for 4 parallel jobs on a CI runner +# Keep in sync with kTestRegisterBytes in test.h +set(simd_test_libraries "") +foreach(register_bytes 16 32 64 128) + fastsimd_create_dispatch_library(simd_test_${register_bytes} SOURCES "test.inl" FEATURE_SETS ${FASTSIMD_TEST_FEATURE_SETS}) + fastsimd_create_dispatch_library(simd_test_relaxed_${register_bytes} RELAXED SOURCES "test.inl" FEATURE_SETS ${FASTSIMD_TEST_FEATURE_SETS}) + + foreach(simd_test_library simd_test_${register_bytes} simd_test_relaxed_${register_bytes}) + target_compile_definitions(${simd_test_library} PRIVATE FASTSIMD_TEST_REGISTER_BYTES=${register_bytes}) + + # Every test is too many sections for an unoptimised COFF object file + if(MSVC) + target_compile_options(${simd_test_library} PRIVATE /bigobj) + elseif(WIN32 AND CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + # GNU as (MinGW) has the same limit, clang's integrated assembler switches to big objects itself + target_compile_options(${simd_test_library} PRIVATE -Wa,-mbig-obj) + endif() + + list(APPEND simd_test_libraries ${simd_test_library}) + endforeach() +endforeach() + +# "test" is a reserved target name when CTest is enabled +add_executable(FastSIMD_tests "test.cpp" "test_inputs.cpp") +target_link_libraries(FastSIMD_tests PRIVATE FastSIMD ${simd_test_libraries}) + +# Random inputs must be the same on every platform for a seed, stop arm64 compilers fusing the range maths into FMA +if(MSVC AND CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + target_compile_options(FastSIMD_tests PRIVATE /clang:-ffp-contract=off) +elseif(NOT CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") + target_compile_options(FastSIMD_tests PRIVATE -ffp-contract=off) +endif() + +# Fewer random inputs keeps ctest (and CI) quick, run FastSIMD_tests directly for a full run +set(FASTSIMD_CTEST_INPUT_COUNT 65536 CACHE STRING "FastSIMD random input values per test when run through ctest") +add_test(NAME FastSIMD_tests COMMAND FastSIMD_tests --count ${FASTSIMD_CTEST_INPUT_COUNT}) if(CMAKE_SYSTEM_NAME STREQUAL "Emscripten") - set(CMAKE_EXECUTABLE_SUFFIX ".html") - target_link_options(test PRIVATE -sALLOW_MEMORY_GROWTH=1 -sSINGLE_FILE) + # Runs under node, which the Emscripten toolchain sets as the ctest emulator + # EXIT_RUNTIME makes main's return value the exit code + # Debug info makes the single file output over 1GB, larger than node can load, so it's left out when linking + target_link_options(FastSIMD_tests PRIVATE -sALLOW_MEMORY_GROWTH=1 -sSINGLE_FILE -sEXIT_RUNTIME=1 $<$:-g0>) + + # The toolchain uses whichever node is first on PATH, emsdk's own node supports the relaxed SIMD the relaxed tests use + if(DEFINED ENV{EMSDK_NODE}) + set_target_properties(FastSIMD_tests PROPERTIES CROSSCOMPILING_EMULATOR "$ENV{EMSDK_NODE}") + endif() + + # Single page to run the tests in a browser + add_executable(FastSIMD_tests_web "test.cpp" "test_inputs.cpp") + target_link_libraries(FastSIMD_tests_web PRIVATE FastSIMD ${simd_test_libraries}) + set_target_properties(FastSIMD_tests_web PROPERTIES SUFFIX ".html") + target_link_options(FastSIMD_tests_web PRIVATE -sALLOW_MEMORY_GROWTH=1 -sSINGLE_FILE $<$:-g0>) endif() # Enable Warnings diff --git a/tests/test.cpp b/tests/test.cpp index fa07caf..de19e96 100644 --- a/tests/test.cpp +++ b/tests/test.cpp @@ -1,308 +1,767 @@ #include "test.h" +#include "test_inputs.h" +#include +#include +#include +#include +#include +#include #include -#include #include -#include -#include +#include +#include #include #include +#include +#include #include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +// Every register size has a test library pair, all created for the same feature sets +using TestFeatureSets = FastSIMD::simd_test_16::CompiledFeatureSets; + +static_assert( std::is_same_v && + std::is_same_v && + std::is_same_v && + std::is_same_v && + std::is_same_v && + std::is_same_v && + std::is_same_v, + "Every simd_test library must compile the same feature sets" ); + +struct TestOptions +{ + uint64_t seed = 24301; + uint64_t count = 1 << 22; // Random input values per test argument, after the special value combinations + std::string filter; // Only run tests with names containing this + size_t registerBytes = 0; // Only run this register size, 0 for all + FastSIMD::FeatureSet maxFeatureSet = FastSIMD::FeatureSet::Max; + bool stats = false; // Print the largest relaxed differences +}; -#include +struct TestSummary +{ + size_t passed = 0; + std::vector knownFailures; + std::vector knownNotSeen; + std::vector failed; +}; -static constexpr size_t TestCount = 4096 * 4096; +static constexpr int kMaxFailuresPerTier = 3; -static int * rndInts; -static float* rndFloats; +static std::string FeatureSetName( FastSIMD::FeatureSet featureSet ) +{ + return FastSIMD::GetFeatureSetString( featureSet ); +} -static float GenFiniteFloat( std::mt19937& gen ) +static std::string TierName( const TestTier& tier ) { - union + if( tier.kind == Tiers::Oracle ) { - float f; - int32_t i; - } u; + return "SCALAR vs ORACLE"; + } + return FeatureSetName( tier.featureSet ) + ( tier.kind == Tiers::Relaxed ? " RELAXED" : "" ); +} - do +static std::string JoinNames( const std::vector& names ) +{ + std::string joined; + for( size_t i = 0; i < names.size(); i++ ) { - u.i = static_cast( gen() ); + joined += ( i ? ", " : "" ) + names[i]; + } + return joined; +} - } while( !std::isfinite( u.f ) ); +// A tier's own version of a test, everything else is the reference's +struct TierTest +{ + TestTier tier; + std::function testFunc; +}; + +// A test's SCALAR non relaxed reference registration, and the test function of every other tier in tier order +struct Test +{ + std::string_view name; + TestData reference; + std::vector tiers; +}; - return u.f; +using TestSet = std::vector; + +static void CheckReference( std::string_view testName, const TestData& reference ) +{ + for( const KnownFailure& knownFailure : reference.knownFailures ) + { + if( knownFailure.reason.empty() ) + { + throw std::runtime_error( "Known failure with no reason in \"" + std::string( testName ) + "\"" ); + } + } + ValidateInputs( testName, reference ); } -static void GenerateRandomValues() +// Input conditions are a different function in each tier, so only whether there is one can be compared +static bool SameKnownFailure( const KnownFailure& a, const KnownFailure& b ) { - std::cout << "Generating random values..." << std::endl; + return a.reason == b.reason && a.tiers == b.tiers && a.minFeatureSet == b.minFeatureSet && a.maxFeatureSet == b.maxFeatureSet && + a.zeroSignOnly == b.zeroSignOnly && !a.inputCondition == !b.inputCondition; +} - rndInts = new int[TestCount + 1024]; - rndFloats = new float[TestCount + 1024]; +// Every tier's results are read as the reference's types and checked with the reference's settings, so a tier registering others is an error +static void CheckMatchesReference( const Test& test, const TestTier& tier, const TestData& registered ) +{ + const TestData& reference = test.reference; - std::random_device rd; //Will be used to obtain a seed for the random number engine - std::mt19937 gen( rd() ); //Standard mersenne_twister_engine seeded with rd() + if( registered.returnType != reference.returnType || registered.lanes != reference.lanes || registered.argCount != reference.argCount || + !std::equal( registered.argTypes, registered.argTypes + kMaxTestArgs, reference.argTypes ) || + !std::equal( registered.knownFailures.begin(), registered.knownFailures.end(), reference.knownFailures.begin(), reference.knownFailures.end(), SameKnownFailure ) || + registered.bitExact != reference.bitExact || + registered.relaxedTolerance != reference.relaxedTolerance || registered.scalarRelaxedTolerance != reference.scalarRelaxedTolerance ) + { + throw std::runtime_error( "Tests do not match: " + std::string( test.name ) + " (" + TierName( tier ) + ")" ); + } +} - for ( std::size_t i = 0; i < TestCount; i++ ) +// NewDispatchClass gives the highest compiled feature set up to the one asked for, every test library compiles TestFeatureSets so it's always that one +template +static void AddTier( TestSet& set, FastSIMD::FeatureSet featureSet ) +{ + std::unique_ptr> testClass( FastSIMD::NewDispatchClass>( featureSet ) ); + + if( !testClass ) { - rndInts[i] = (int)gen(); - rndFloats[i] = GenFiniteFloat( gen ); + throw std::runtime_error( "No test class created for " + FeatureSetName( featureSet ) ); } - //std::sort( rndFloats, rndFloats + TestCount + 1024, std::less() ); + TestTier tier = { featureSet, Relaxed ? Tiers::Relaxed : Tiers::NonRelaxed }; + bool isReference = featureSet == FastSIMD::FeatureSet::SCALAR && !Relaxed; + + for( auto& registered : testClass->RegisterTests() ) + { + std::string_view testName = registered.first; + TestData& test = registered.second; + + auto find = std::find_if( set.begin(), set.end(), [testName]( const Test& other ) { return other.name == testName; } ); + + if( isReference ) + { + if( find != set.end() ) + { + throw std::runtime_error( "Multiple tests with same name: " + std::string( testName ) ); + } + CheckReference( testName, test ); + set.push_back( { testName, std::move( test ), {} } ); + continue; + } + + if( find == set.end() ) + { + throw std::runtime_error( "\"" + std::string( testName ) + "\" has no SCALAR non relaxed reference" ); + } + if( std::any_of( find->tiers.begin(), find->tiers.end(), [&]( const TierTest& other ) { return other.tier.featureSet == featureSet && other.tier.kind == tier.kind; } ) ) + { + throw std::runtime_error( "Multiple tests with same name: " + std::string( testName ) + " (" + TierName( tier ) + ")" ); + } + CheckMatchesReference( *find, tier, test ); + find->tiers.push_back( { tier, std::move( test.testFunc ) } ); + } } +// Only creating the test classes depends on the register size template -struct TestRunner +static TestSet CollectTests( FastSIMD::FeatureSet maxFeatureSet ) { - using TestSet = std::vector>>; + TestSet set; - template - struct TestOrganiser + for( FastSIMD::FeatureSet featureSet : TestFeatureSets::AsArray ) { - static TestCollection GetCollections() + if( featureSet <= FastSIMD::DetectCpuMaxFeatureSet() && featureSet <= maxFeatureSet ) { - return {}; + std::cout << "Generating Tests: " << FeatureSetName( featureSet ) << std::endl; + + AddTier( set, featureSet ); + AddTier( set, featureSet ); } - }; + } + return set; +} - template - struct TestOrganiser> - { - static TestCollection GetCollections() - { - TestCollection collections = TestOrganiser>::GetCollections(); +// Value of the least significant bit, relaxed differences are counted in these +// Zero and denormals use the smallest normal rather than the denormal spacing, so relaxed results may flush denormals +template +static T DeltaUnit( T x ) +{ + x = std::fabs( x ); - if( HEAD <= FastSIMD::DetectCpuMaxFeatureSet() ) - { - std::cout << "Generating Tests: " << FastSIMD::GetFeatureSetString( HEAD ) << std::endl; - { - std::unique_ptr> testSimd( FastSIMD::NewDispatchClass>( HEAD ) ); + if( !std::isfinite( x ) ) + { + return T( 1 ); + } + if( x == T( 0 ) || !std::isnormal( x ) ) + { + return std::numeric_limits::min(); // smallest positive normal number + } - TestCollection simdCollection = testSimd->RegisterTests(); + T above = std::nextafter( x, std::numeric_limits::infinity() ); - collections.insert( collections.begin(), simdCollection.begin(), simdCollection.end() ); - } - { - std::unique_ptr> testSimd( FastSIMD::NewDispatchClass>( HEAD ) ); + // The largest finite value has nothing finite above it, use the gap below instead + return std::isfinite( above ) ? above - x : x - std::nextafter( x, T( 0 ) ); +} - TestCollection simdCollection = testSimd->RegisterTests(); +enum class LaneResult +{ + Equal, ZeroSign, Different +}; - collections.insert( collections.begin(), simdCollection.begin(), simdCollection.end() ); - } - } +struct LaneComparison +{ + LaneResult result; + double ulpsCovered = 0; // Relaxed difference only the ULP tolerance covered, for --stats + double absCovered = 0; // Relaxed difference only the absolute tolerance covered +}; - return collections; +template +static LaneComparison CompareLane( const TestData& test, const TestTier& tier, T expected, T actual ) +{ + if constexpr( std::is_floating_point_v ) + { + bool bitsEqual = ToBits( expected ) == ToBits( actual ); + bool bothNaN = std::isnan( expected ) && std::isnan( actual ); + + if( bitsEqual || ( bothNaN && !test.bitExact ) ) + { + return { LaneResult::Equal }; } - static TestSet GetSet() + // Relaxed results are compared by value, so +0 and -0 are equal + if( tier.kind == Tiers::Relaxed && !test.bitExact ) { - TestCollection collections = GetCollections(); - TestSet set; - - for( auto& collection : collections ) + if( expected == actual ) { - std::string_view& testName = collection.first; + return { LaneResult::Equal }; + } + + Tolerance tolerance = test.RelaxedTolerance( tier.featureSet ); + double absoluteDif = std::abs( static_cast( expected ) - static_cast( actual ) ); + double relativeDif = absoluteDif / DeltaUnit( expected ); - auto find = std::find_if( set.begin(), set.end(), [testName]( const auto& pair ){ return pair.first == testName; } ); + if( relativeDif <= tolerance.ulps || absoluteDif <= tolerance.absolute ) + { + LaneComparison equal = { LaneResult::Equal }; - if( find == set.end() ) + // Track what each tolerance had to cover on its own + if( std::isfinite( absoluteDif ) ) { - if( collection.second.featureSet != FastSIMD::FeatureSet::SCALAR ) + if( absoluteDif > tolerance.absolute ) { - throw std::runtime_error( "Scalar must be base test set" ); + equal.ulpsCovered = relativeDif; + } + if( relativeDif > tolerance.ulps ) + { + equal.absCovered = absoluteDif; } - - find = set.emplace( set.end(), testName, std::vector{} ); } - - find->second.emplace_back( collection.second ); + return equal; } - - return set; + return { LaneResult::Different }; } - }; - // get value of least significant bit - static float DeltaUnit( float x ) + return { expected == 0 && actual == 0 ? LaneResult::ZeroSign : LaneResult::Different }; + } + else { - union - { - float f; - uint32_t i; - } u; - x = std::fabs( x ); + return { expected == actual ? LaneResult::Equal : LaneResult::Different }; + } +} - if( !std::isfinite( x ) ) +template +static std::string Describe( T expected, T actual ) +{ + std::ostringstream sstr; + sstr << std::setprecision( std::is_floating_point_v ? std::numeric_limits::max_digits10 : 16 ) << std::boolalpha; + sstr << "Expected \"" << expected << "\" Actual \"" << actual << "\""; + + if constexpr( std::is_floating_point_v ) + { + T absoluteDif = std::abs( expected - actual ); + sstr << " Diff \"" << absoluteDif << "\""; + + if( std::isfinite( absoluteDif ) && absoluteDif != 0 ) { - return 1.f; + sstr << " (" << absoluteDif / DeltaUnit( expected ) << " ULP)"; } - if( x == 0.f || !std::isnormal( x ) ) + if( ToBits( expected ) != ToBits( actual ) && ( absoluteDif == 0 || std::isnan( absoluteDif ) ) ) { - u.i = 0x00800000; // smallest positive normal number - return u.f; + sstr << std::hex << " (bits 0x" << ToBits( expected ) << " vs 0x" << ToBits( actual ) << ")" << std::dec; } - float x1 = x; - u.f = x; - u.i++; - return u.f - x1; } - - template - static bool CompareTyped( std::string_view testName, FastSIMD::FeatureSet featureSet, float accuracy, size_t outputCount, void* scalarResults, void* simdResults ) + else if constexpr( std::is_same_v ) { - bool success = true; + sstr << " Diff \"" << static_cast( expected ) - static_cast( actual ) << "\""; + } + else if constexpr( std::is_same_v ) + { + sstr << std::hex << " (bits 0x" << static_cast( expected ) << " vs 0x" << static_cast( actual ) << ")" << std::dec; + } + return sstr.str(); +} - const T* typedScalar = reinterpret_cast( scalarResults ); - const T* typedSimd = reinterpret_cast( simdResults ); +// A tier's run state while one test runs +struct Tier : TestTier +{ + std::string name; + const std::function& testFunc; // The tier's test, or the oracle for the oracle tier + bool allDifferencesKnown; // A known failure covers any difference on this tier, so comparing more of its calls can't fail + size_t compared = 0; // Calls compared + int failures = 0; // Calls with differences not covered by a known failure + int knownCalls = 0; // Calls with only known differences + double maxUlps = 0; // Largest relaxed difference only the ULP tolerance covered, for --stats + double maxAbs = 0; // Largest relaxed difference only the absolute tolerance covered + + // A tier stops being checked after kMaxFailuresPerTier failing calls, or known failing calls if a known failure covers every difference + bool Stopped() const + { + return failures >= kMaxFailuresPerTier || ( allDifferencesKnown && knownCalls >= kMaxFailuresPerTier ); + } +}; - for( size_t idx = 0; idx < outputCount; idx++ ) - { - if( typedScalar[idx] != typedSimd[idx] ) - { - float relativeDif = 0; +// Ordered, a call's outcome is its worst lane's +enum class CallOutcome +{ + Pass, Known, Fail +}; - if constexpr( std::is_floating_point_v ) - { - if( std::isnan( typedScalar[idx] ) && std::isnan( typedSimd[idx] ) ) - { - continue; - } +struct CallResult +{ + CallOutcome outcome = CallOutcome::Pass; + std::string details; // A line for each difference no known failure covers +}; - relativeDif = std::abs( typedScalar[idx] - typedSimd[idx] ) / DeltaUnit( typedScalar[idx] ); +// Compares every tier of one test on every call, then reports what they found +class TestRun +{ +public: + TestRun( size_t registerBytes, const Test& test ) : + mWidth( std::to_string( registerBytes * 8 ) + "bit " ), mTestName( test.name ), mReference( test.reference ), mKnownSeen( test.reference.knownFailures.size() ) + { + auto addComparison = [this]( TestTier tier, const std::function& testFunc ) + { + bool allDifferencesKnown = std::any_of( mReference.knownFailures.begin(), mReference.knownFailures.end(), + [&]( const KnownFailure& knownFailure ) { return knownFailure.CoversAnyDifference() && knownFailure.Covers( tier ); } ); - if( relativeDif <= accuracy ) - { - continue; - } - } - if( success ) - { - std::cerr << std::setprecision( 16 ) << std::boolalpha; - std::cerr << "--- " << FastSIMD::GetFeatureSetString( featureSet ) << " FAILED ---" << std::endl; - } - std::cerr << "idx " << idx << ": " << testName - << " Expected \"" << typedScalar[idx] - << "\" Actual \"" << typedSimd[idx] - << "\" Diff \"" << std::abs( typedScalar[idx] - typedSimd[idx] ) << "\""; + mTiers.push_back( { tier, TierName( tier ), testFunc, allDifferencesKnown } ); + }; - if( relativeDif != 0.0f ) - { - std::cerr << " (" << relativeDif << ")"; - } - std::cerr << std::endl; - success = false; - } + if( mReference.oracleFunc ) + { + addComparison( { FastSIMD::FeatureSet::SCALAR, Tiers::Oracle }, mReference.oracleFunc ); + } + for( const TierTest& tierTest : test.tiers ) + { + addComparison( tierTest.tier, tierTest.testFunc ); } - - return success; } - static bool CompareOutputs( std::string_view testName, FastSIMD::FeatureSet featureSet, TestData::ReturnType returnType, float accuracy, size_t outputCount, void* scalarResults, void* simdResults ) + void Run( const TestOptions& options ) { - switch( returnType ) + switch( mReference.returnType ) { case TestData::ReturnType::boolean: - return CompareTyped( testName, featureSet, accuracy, outputCount, scalarResults, simdResults ); + return RunCalls( options ); case TestData::ReturnType::f32: - return CompareTyped( testName, featureSet, accuracy, outputCount, scalarResults, simdResults ); + return RunCalls( options ); + + case TestData::ReturnType::f64: + return RunCalls( options ); case TestData::ReturnType::i32: - return CompareTyped( testName, featureSet, accuracy, outputCount, scalarResults, simdResults ); + return RunCalls( options ); + + case TestData::ReturnType::i64: + return RunCalls( options ); } + } - return false; + void Report( const TestOptions& options, TestSummary& summary ) const + { + std::vector failedTiers; + bool comparedAny = false; + + for( const Tier& tier : mTiers ) + { + comparedAny |= tier.compared > 0; + + if( tier.failures ) + { + failedTiers.push_back( tier.name ); + } + if( options.stats && tier.kind == Tiers::Relaxed && ( tier.maxUlps > 0 || tier.maxAbs > 0 ) ) + { + Tolerance tolerance = mReference.RelaxedTolerance( tier.featureSet ); + std::cout << " Stats " << mWidth << mTestName << " " << tier.name << ": max " << tier.maxUlps << " / " << tolerance.ulps << " ULP, " + << tier.maxAbs << " / " << tolerance.absolute << " abs" << std::endl; + } + } + + for( size_t knownIdx = 0; knownIdx < mReference.knownFailures.size(); knownIdx++ ) + { + const KnownFailure& knownFailure = mReference.knownFailures[knownIdx]; + std::string reason( knownFailure.reason ); + + if( !mKnownSeen[knownIdx].empty() ) + { + summary.knownFailures.push_back( mWidth + std::string( mTestName ) + " (" + JoinNames( mKnownSeen[knownIdx] ) + "): " + reason ); + } + else if( std::any_of( mTiers.begin(), mTiers.end(), [&]( const Tier& tier ) { return tier.compared && knownFailure.Covers( tier ); } ) ) + { + summary.knownNotSeen.push_back( mWidth + std::string( mTestName ) + ": " + reason ); + } + } + + if( !comparedAny ) + { + std::cout << " Warning: no tiers to compare with the reference" << std::endl; + } + + if( !failedTiers.empty() ) + { + summary.failed.push_back( mWidth + std::string( mTestName ) + " (" + JoinNames( failedTiers ) + ")" ); + } + else + { + summary.passed++; + } } - static void DoTest( std::string_view testName, std::vector& tests ) +private: + template + void RunCalls( const TestOptions& options ) { - std::cout << "Testing: " << testName << std::endl; + InputGenerator generator( options.seed, mTestName, mReference ); + + // No argument tests always produce the same output + size_t inputCount = mReference.argCount ? generator.SpecialCount() + static_cast( options.count ) : mReference.lanes; - char* scalarResults = new char[RegisterBytes]; - char* simdResults = new char[RegisterBytes]; - int failed = 0; + auto allStopped = [this]() { return std::all_of( mTiers.begin(), mTiers.end(), []( const Tier& tier ) { return tier.Stopped(); } ); }; + std::vector referenceResults( kMaxTestResultBytes ), tierResults( kMaxTestResultBytes ); + const T* reference = reinterpret_cast( referenceResults.data() ); + const T* results = reinterpret_cast( tierResults.data() ); - for( size_t idx = 0; idx < TestCount; idx += RegisterBytes / sizeof( int ) ) + for( size_t chunkStart = 0; chunkStart < inputCount && !allStopped(); chunkStart += kChunkSize ) { - for( size_t testIdx = 0; testIdx < tests.size(); testIdx++ ) - { - TestData& test = tests[testIdx]; - - char* resultsOut = testIdx ? simdResults : scalarResults; - std::memset( resultsOut, (int)testIdx, RegisterBytes ); - - size_t outputCount = test.testFunc( resultsOut, idx, rndInts, rndFloats ); - - if( testIdx ) + const TestInputs& inputs = generator.Generate( chunkStart ); + size_t chunkCount = std::min( kChunkSize, inputCount - chunkStart ); + + for( size_t idx = 0; idx < chunkCount; idx += mReference.lanes ) + { + std::memset( referenceResults.data(), 0xCD, kMaxTestResultBytes ); + size_t referenceCount = mReference.testFunc( referenceResults.data(), idx, inputs ); + + for( Tier& tier : mTiers ) { - if( test.returnType != tests[0].returnType ) + if( tier.Stopped() ) { - std::cerr << "Tests do not match: " << testName; - throw std::exception(); - } - if( test.featureSet == FastSIMD::FeatureSet::SCALAR && !test.relaxed ) - { - std::cerr << "Multiple tests with same name: " << testName; - throw std::exception(); + continue; } - std::string testNameRelaxed = testName.data(); - float accuracy = 0; + std::memset( tierResults.data(), 0xAB, kMaxTestResultBytes ); + size_t outputCount = tier.testFunc( tierResults.data(), idx, inputs ); + + // The oracle is the expected value, the SCALAR reference is checked against it + bool oracle = tier.kind == Tiers::Oracle; + CallResult call = outputCount != referenceCount + ? CallResult{ CallOutcome::Fail, "Output count " + std::to_string( outputCount ) + ", reference " + std::to_string( referenceCount ) + "\n" } + : CompareCall( tier, inputs, idx, outputCount, oracle ? results : reference, oracle ? reference : results ); - if( test.relaxed ) + tier.compared++; + + if( call.outcome == CallOutcome::Fail ) { - testNameRelaxed += " RELAXED"; - accuracy = test.relaxedAccuracy; + if( !tier.failures ) + { + std::cerr << std::setprecision( 16 ) << "--- " << mWidth << mTestName << " " << tier.name << " FAILED ---" << std::endl; + std::cerr << call.details; + std::cerr << "Inputs (index " << chunkStart + idx << "): " << DescribeInputs( mReference, inputs, idx ) << std::endl; + } + tier.failures++; } - - if( !CompareOutputs( testNameRelaxed, test.featureSet, test.returnType, accuracy, outputCount, scalarResults, simdResults ) ) + else if( call.outcome == CallOutcome::Known ) { - std::cerr << "Inputs: " << tests[0].inputsFunc( idx, rndInts, rndFloats ) << std::endl; - failed++; + tier.knownCalls++; } } + } + } + } + + template + CallResult CompareCall( Tier& tier, const TestInputs& inputs, size_t inIdx, size_t outputCount, const T* expected, const T* actual ) + { + const std::vector& knownFailures = mReference.knownFailures; + CallResult call; - if( failed >= 3 ) + for( size_t idx = 0; idx < outputCount; idx++ ) + { + LaneComparison lane = CompareLane( mReference, tier, expected[idx], actual[idx] ); + tier.maxUlps = std::max( tier.maxUlps, lane.ulpsCovered ); + tier.maxAbs = std::max( tier.maxAbs, lane.absCovered ); + + if( lane.result == LaneResult::Equal ) + { + continue; + } + + LaneArgs laneArgs = GetLaneArgs( mReference, inputs, inIdx, idx ); + auto known = std::find_if( knownFailures.begin(), knownFailures.end(), [&]( const KnownFailure& knownFailure ) + { + return knownFailure.Matches( tier, lane.result == LaneResult::ZeroSign, laneArgs ); + } ); + + if( known != knownFailures.end() ) + { + std::vector& seen = mKnownSeen[known - knownFailures.begin()]; + + if( std::find( seen.begin(), seen.end(), tier.name ) == seen.end() ) { - break; + seen.push_back( tier.name ); } + call.outcome = std::max( call.outcome, CallOutcome::Known ); } - - if( failed >= 3 ) + else { - std::cerr << "Skipping test, fail limit reached" << std::endl; - break; + call.details += "idx " + std::to_string( idx ) + ": " + std::string( mTestName ) + ( tier.kind == Tiers::Relaxed ? " RELAXED " : " " ) + Describe( expected[idx], actual[idx] ) + "\n"; + call.outcome = CallOutcome::Fail; } } - - delete[] scalarResults; - delete[] simdResults; + return call; } - static void Run() + std::string mWidth; + std::string_view mTestName; + const TestData& mReference; + std::vector mTiers; + std::vector> mKnownSeen; // Tiers each known failure was seen on +}; + +static void DoTest( size_t registerBytes, const Test& test, const TestOptions& options, TestSummary& summary ) +{ + std::cout << "Testing: " << test.name << std::endl; + + TestRun run( registerBytes, test ); + run.Run( options ); + run.Report( options, summary ); +} + +template +static void RunRegisterSize( const TestOptions& options, TestSummary& summary ) +{ + if( options.registerBytes && options.registerBytes != RegisterBytes ) { - std::cout << "Starting Tests - Register Size: " << RegisterBytes * 8 << " (" << RegisterBytes << "b)" << std::endl; + return; + } - TestSet testSet = TestOrganiser::GetSet(); + std::cout << "Starting Tests - Register Size: " << RegisterBytes * 8 << " (" << RegisterBytes << "b)" << std::endl; - for( auto& test : testSet ) + for( const Test& test : CollectTests( options.maxFeatureSet ) ) + { + if( test.name.find( options.filter ) != std::string_view::npos ) { - //if( test.first.find( "sqrt" ) != std::string_view::npos ) + DoTest( RegisterBytes, test, options, summary ); + } + } +} + +template +static void RunTests( const TestOptions& options, TestSummary& summary, std::index_sequence ) +{ + ( RunRegisterSize( options, summary ), ... ); +} + +static void PrintUsage( const char* exe ) +{ + std::cout << "Usage: " << exe << " [options]\n" + << " --seed Seed for random inputs (default " << TestOptions{}.seed << ")\n" + << " --count Random input values per test argument (default " << TestOptions{}.count << ")\n" + << " --filter Only run tests with names containing text\n" + << " --width Only run one register size:"; + for( size_t bytes : kTestRegisterBytes ) + { + std::cout << " " << bytes; + } + std::cout << "\n --max-feature-set Only run feature sets up to this one, to match a less capable CPU:"; + for( FastSIMD::FeatureSet featureSet : TestFeatureSets::AsArray ) + { + std::cout << " " << FeatureSetName( featureSet ); + } + std::cout << "\n --stats Print the largest relaxed differences within tolerance" << std::endl; +} + +static bool ParseUnsigned( const char* text, uint64_t& value ) +{ + if( !text || !std::isdigit( static_cast( text[0] ) ) ) + { + return false; + } + + char* end = nullptr; + errno = 0; + unsigned long long parsed = std::strtoull( text, &end, 0 ); + + if( errno || *end ) + { + return false; + } + value = parsed; + return true; +} + +static std::string NormaliseName( std::string_view name ) +{ + std::string normalised; + for( char c : name ) + { + if( c != '.' && c != '_' ) + { + normalised += static_cast( std::toupper( static_cast( c ) ) ); + } + } + return normalised; +} + +static bool ParseArgs( int argc, char** argv, TestOptions& options ) +{ + for( int i = 1; i < argc; i++ ) + { + std::string_view arg = argv[i]; + + if( arg == "--stats" ) + { + options.stats = true; + continue; + } + + const char* value = i + 1 < argc ? argv[++i] : nullptr; + uint64_t number = 0; + bool valid = value != nullptr; + + if( arg == "--seed" && valid ) + { + if( std::string_view( value ) == "random" ) + { + options.seed = std::random_device()(); + } + else { - DoTest( test.first, test.second ); + valid = ParseUnsigned( value, options.seed ); } } + else if( arg == "--count" && valid ) + { + // Limited so the input index can't overflow size_t on 32bit + valid = ParseUnsigned( value, number ) && number <= ( 1ull << 30 ); + options.count = number; + } + else if( arg == "--filter" && valid ) + { + options.filter = value; + } + else if( arg == "--width" && valid ) + { + valid = ParseUnsigned( value, number ) && + std::find( std::begin( kTestRegisterBytes ), std::end( kTestRegisterBytes ), number ) != std::end( kTestRegisterBytes ); + options.registerBytes = static_cast( number ); + } + else if( arg == "--max-feature-set" && valid ) + { + const auto& featureSets = TestFeatureSets::AsArray; + auto find = std::find_if( std::begin( featureSets ), std::end( featureSets ), [value]( FastSIMD::FeatureSet featureSet ) { return NormaliseName( FeatureSetName( featureSet ) ) == NormaliseName( value ); } ); - std::cout << "Testing Complete!" << std::endl; + valid = find != std::end( featureSets ); + options.maxFeatureSet = valid ? *find : options.maxFeatureSet; + } + else + { + valid = false; + } + + if( !valid ) + { + std::cout << "Invalid argument: " << arg << ( value ? std::string( " " ) + value : std::string() ) << "\n" << std::endl; + PrintUsage( argv[0] ); + return false; + } } -}; + return true; +} -int main() +int main( int argc, char** argv ) { - GenerateRandomValues(); + TestOptions options; + + if( !ParseArgs( argc, argv, options ) ) + { + return 2; + } + + std::cout << "Seed: " << options.seed << ", Random inputs per test: " << options.count << std::endl; + std::cout << "CPU max feature set: " << FeatureSetName( FastSIMD::DetectCpuMaxFeatureSet() ) << ", Compiled:"; + for( FastSIMD::FeatureSet featureSet : TestFeatureSets::AsArray ) + { + std::cout << " " << FeatureSetName( featureSet ); + } + std::cout << std::endl; + if( options.maxFeatureSet != FastSIMD::FeatureSet::Max ) + { + std::cout << "Max feature set: " << FeatureSetName( options.maxFeatureSet ) << std::endl; + } + + TestSummary summary; - TestRunner::Run(); + try + { + RunTests( options, summary, std::make_index_sequence{} ); + } + catch( const std::exception& e ) + { + std::cerr << std::endl << "Test setup error: " << e.what() << std::endl; + return 1; + } + + std::cout << std::endl << "Passed: " << summary.passed << ", Known failures: " << summary.knownFailures.size() << ", Failed: " << summary.failed.size() << std::endl; + + for( const std::string& knownFailure : summary.knownFailures ) + { + std::cout << " Known failure: " << knownFailure << std::endl; + } + for( const std::string& notSeen : summary.knownNotSeen ) + { + std::cout << " Known failure not seen: " << notSeen << std::endl; + } + for( const std::string& failed : summary.failed ) + { + std::cout << " FAILED: " << failed << std::endl; + } + + if( !summary.passed && summary.failed.empty() ) + { + std::cout << "No tests matched filter \"" << options.filter << "\"" << std::endl; + return 1; + } - return 0; + return summary.failed.empty() ? 0 : 1; } diff --git a/tests/test.h b/tests/test.h index 24c3a19..0019398 100644 --- a/tests/test.h +++ b/tests/test.h @@ -1,35 +1,291 @@ #pragma once #include #include -#include +#include +#include #include +#include +#include #include +#include +#include #include #include -constexpr size_t kTestBytes = 512 / 8; +// Register sizes the tests are compiled for, each is a pair of dispatch libraries in tests/CMakeLists.txt +constexpr size_t kTestRegisterBytes[] = { 16, 32, 64, 128 }; +// Lanes per call is set by a test's widest lane type, 32bit lanes give the most +constexpr size_t kMaxTestLanes = 128 / sizeof( int32_t ); +// Converts keep the lane count, so 32bit lanes widened to 64bit need twice the register size +constexpr size_t kMaxTestResultBytes = kMaxTestLanes * sizeof( double ); -struct TestResult +constexpr size_t kMaxTestArgs = 3; + +template +struct FloatTraits; + +template<> +struct FloatTraits +{ + using Bits = uint32_t; + static constexpr int kSignificandBits = 24; +}; + +template<> +struct FloatTraits { - uint8_t returnCount; + using Bits = uint64_t; + static constexpr int kSignificandBits = 53; }; -using TestFunction = size_t ( void*, size_t, int32_t*, float* ); -using InputsFunction = std::string ( size_t, int32_t*, float* ); +template +typename FloatTraits::Bits ToBits( T f ) +{ + typename FloatTraits::Bits bits; + std::memcpy( &bits, &f, sizeof( f ) ); + return bits; +} + +template +T FromBits( typename FloatTraits::Bits bits ) +{ + T f; + std::memcpy( &f, &bits, sizeof( f ) ); + return f; +} + +// The type an argument's input values are stored as, one buffer per argument +enum class TestElement : uint8_t +{ + Int32, Float, Double +}; +struct TestArgType +{ + TestElement element; + bool mask; // Lanes are 0 or 1 in the element type, from lane patterns rather than an input spec, compared != 0 to make the mask + size_t lanes; // Values the argument takes per call, 1 for scalars which are the same value in every lane + + bool operator==( const TestArgType& other ) const { return element == other.element && mask == other.mask && lanes == other.lanes; } +}; + +// One pointer per argument to its values for the current chunk, indexed by input index and stored as the argument's TestElement +struct TestInputs +{ + const void* values[kMaxTestArgs]; + + template + const T* Get( size_t argIdx ) const + { + return static_cast( values[argIdx] ); + } +}; + +// A lane's input values as the oracle sees them, for known failure conditions. Arguments of other types are 0 +struct LaneArgs +{ + int32_t ints[kMaxTestArgs]; // Masks are 0 or 1 + float floats[kMaxTestArgs]; + double doubles[kMaxTestArgs]; + + void Set( size_t argIdx, int32_t value ) { ints[argIdx] = value; } + void Set( size_t argIdx, bool value ) { ints[argIdx] = value; } + void Set( size_t argIdx, float value ) { floats[argIdx] = value; } + void Set( size_t argIdx, double value ) { doubles[argIdx] = value; } +}; + +// How input values are generated for a test argument, the default spec is every value of the argument's type +// Every test first runs all combinations of the special values for its inputs, then random values +struct InputSpec +{ + enum class Kind + { + Any, // Values of every magnitude, +-inf and NaNs + NoNaN, // Values of every magnitude and +-inf + Finite, // Values of every magnitude + Range, // Values within [lo, hi], half uniform, half log scaled + Magnitude, // Values with magnitude within [lo, hi] log scaled, either sign, lo > 0 + }; + + Kind kind = Kind::Any; + double lo = 0; + double hi = 0; + int significandBits = 0; // Floats are truncated to this many significand bits, 0 for all of them + std::vector specials; // Extra special values, only used when inside the spec's domain + + static InputSpec NoNaN() { InputSpec spec; spec.kind = Kind::NoNaN; return spec; } + static InputSpec Finite() { InputSpec spec; spec.kind = Kind::Finite; return spec; } + static InputSpec Range( double lo, double hi ) { InputSpec spec; spec.kind = Kind::Range; spec.lo = lo; spec.hi = hi; return spec; } + static InputSpec Magnitude( double lo, double hi ) { InputSpec spec; spec.kind = Kind::Magnitude; spec.lo = lo; spec.hi = hi; return spec; } + + InputSpec Bits( int bits ) const { InputSpec spec = *this; spec.significandBits = bits; return spec; } + + // Adds each value and the floats (or doubles) either side of it + InputSpec Specials( std::initializer_list values ) const + { + InputSpec spec = *this; + spec.specials.insert( spec.specials.end(), values.begin(), values.end() ); + return spec; + } + + bool IsDefault() const + { + return kind == Kind::Any && significandBits == 0 && specials.empty(); + } +}; + +using TestFunction = size_t ( void* outResult, size_t inIdx, const TestInputs& inputs ); + +// Kinds of tier, each tier is one kind and known failures are scoped to a set of them +enum class Tiers : uint8_t +{ + None = 0, + NonRelaxed = 1 << 0, // A non relaxed feature set against the SCALAR reference, bit for bit + Relaxed = 1 << 1, // A relaxed feature set against the SCALAR reference, within the test's tolerance + Oracle = 1 << 2, // The SCALAR reference against the test's oracle + All = NonRelaxed | Relaxed, // Every tier compared with the reference, not the oracle +}; + +constexpr Tiers operator|( Tiers a, Tiers b ) +{ + return static_cast( static_cast( a ) | static_cast( b ) ); +} + +constexpr bool operator&( Tiers a, Tiers b ) +{ + return static_cast( a ) & static_cast( b ); +} + +// What a result is checked against: a feature set against the SCALAR reference, or the reference against the oracle +struct TestTier +{ + FastSIMD::FeatureSet featureSet; + Tiers kind; // One of NonRelaxed, Relaxed or Oracle +}; + +using LaneInputCondition = bool ( * )( const LaneArgs& args ); + +// Differences on tiers in scope are reported as known failures and don't fail the run, scoped by tier kind and feature set range +struct KnownFailure +{ + std::string_view reason; + Tiers tiers; + FastSIMD::FeatureSet minFeatureSet; + FastSIMD::FeatureSet maxFeatureSet; + bool zeroSignOnly = false; // Only results differing in the sign of zero are known, any other difference still fails + LaneInputCondition inputCondition = nullptr; // Only lanes with matching inputs are known, null for every lane + + constexpr explicit KnownFailure( std::string_view knownReason, Tiers knownTiers = Tiers::All, + FastSIMD::FeatureSet minKnown = FastSIMD::FeatureSet::Invalid, + FastSIMD::FeatureSet maxKnown = FastSIMD::FeatureSet::Max ) : + reason( knownReason ), tiers( knownTiers ), minFeatureSet( minKnown ), maxFeatureSet( maxKnown ) {} + + // Only results that differ in the sign of zero are known + constexpr KnownFailure ZeroSignOnly() const + { + KnownFailure known = *this; + known.zeroSignOnly = true; + return known; + } + + // Only lanes whose inputs match the condition are known, other inputs are still checked + constexpr KnownFailure ForInputs( LaneInputCondition condition ) const + { + KnownFailure known = *this; + known.inputCondition = condition; + return known; + } + + bool Covers( const TestTier& tier ) const + { + return ( tiers & tier.kind ) && tier.featureSet >= minFeatureSet && tier.featureSet <= maxFeatureSet; + } + + // Any difference on a covered tier is known, so comparing more of its calls can't fail + bool CoversAnyDifference() const + { + return !zeroSignOnly && !inputCondition; + } + + bool Matches( const TestTier& tier, bool zeroSignDifference, const LaneArgs& args ) const + { + return Covers( tier ) && ( !zeroSignOnly || zeroSignDifference ) && ( !inputCondition || inputCondition( args ) ); + } +}; + +// Allowed error for relaxed results +struct Tolerance +{ + double ulps = 0; + double absolute = 0; // For results near 0 + + bool operator==( const Tolerance& other ) const { return ulps == other.ulps && absolute == other.absolute; } + bool operator!=( const Tolerance& other ) const { return !( *this == other ); } +}; + +// A test as registered by one tier +// Only the SCALAR non relaxed registration is kept whole, the other tiers must register the same settings and only their test function is kept struct TestData { enum class ReturnType { - boolean, f32, i32 + boolean, f32, f64, i32, i64 // i64 holds 64bit mask lanes }; - - FastSIMD::FeatureSet featureSet; - bool relaxed; + ReturnType returnType; - float relaxedAccuracy = 0; + size_t lanes = 0; // Elements per call, the widest register in the test sets it + size_t argCount = 0; + TestArgType argTypes[kMaxTestArgs] = {}; + InputSpec inputs[kMaxTestArgs]; + Tolerance relaxedTolerance; + std::optional scalarRelaxedTolerance; // Relaxed SCALAR, when it differs from the relaxed tolerance + bool bitExact = false; // Float results must match bitwise on every tier, including NaN payloads + std::vector knownFailures; std::function testFunc; - std::function inputsFunc; + std::function oracleFunc; // Optional plain C++ version of the test, the SCALAR reference must match it exactly + + template + TestData& Inputs( SPECs... specs ) + { + static_assert( sizeof...( SPECs ) <= kMaxTestArgs ); + size_t idx = 0; + ( ( inputs[idx++] = specs ), ... ); + return *this; + } + + TestData& RelaxedAccuracy( double ulps, double absolute = 0 ) + { + relaxedTolerance = { ulps, absolute }; + return *this; + } + + // Relaxed SCALAR uses std:: math for some functions, so this is the accuracy of the FS implementation itself + // Relaxed SIMD tolerances cover estimate instructions, which would hide FS accuracy regressions + TestData& ScalarRelaxedAccuracy( double ulps, double absolute = 0 ) + { + scalarRelaxedTolerance = Tolerance{ ulps, absolute }; + return *this; + } + + Tolerance RelaxedTolerance( FastSIMD::FeatureSet tierFeatureSet ) const + { + return tierFeatureSet == FastSIMD::FeatureSet::SCALAR && scalarRelaxedTolerance ? *scalarRelaxedTolerance : relaxedTolerance; + } + + TestData& BitExact() + { + bitExact = true; + return *this; + } + + // Differences these cover are reported as known failures and don't fail the run + template + TestData& Known( const KNOWNs&... known ) + { + static_assert( ( std::is_same_v && ... ) ); + ( knownFailures.push_back( known ), ... ); + return *this; + } }; using TestCollection = std::vector>; diff --git a/tests/test.inl b/tests/test.inl index 716f40f..13d0bcf 100644 --- a/tests/test.inl +++ b/tests/test.inl @@ -1,12 +1,11 @@ #pragma once -#include - #include "test.h" +#include +#include +#include #include -#include -#include -#include +#include #include template @@ -36,44 +35,23 @@ template decltype( lambda_pack_helper( &T::operator() ) ) pack_helper( T ); -template -struct GetReturn; - -template<> -struct GetReturn -{ - static constexpr auto Type = TestData::ReturnType::i32; -}; - -template<> -struct GetReturn -{ - static constexpr auto Type = TestData::ReturnType::f32; -}; - -template<> -struct GetReturn -{ - static constexpr auto Type = TestData::ReturnType::boolean; -}; - -template -struct GetReturn> -{ - static constexpr auto Type = TestData::ReturnType::boolean; -}; - -template -struct GetReturn> +// The type an argument's inputs are stored as +template +constexpr TestElement ElementOf() { - static constexpr auto Type = GetReturn::Type; -}; + static_assert( std::is_same_v || std::is_same_v || std::is_same_v, "No test inputs for type" ); + return std::is_same_v ? TestElement::Float : std::is_same_v ? TestElement::Double : TestElement::Int32; +} +// The type a result's lanes are stored and compared as template -static constexpr size_t GetReturnCount = 1; - -template -static constexpr size_t GetReturnCount> = N; +constexpr TestData::ReturnType ReturnTypeOf() +{ + static_assert( std::is_same_v || std::is_same_v || std::is_same_v || std::is_same_v || std::is_same_v, + "No test results for type" ); + return std::is_same_v ? TestData::ReturnType::boolean : std::is_same_v ? TestData::ReturnType::f32 : + std::is_same_v ? TestData::ReturnType::f64 : std::is_same_v ? TestData::ReturnType::i64 : TestData::ReturnType::i32; +} template @@ -83,133 +61,128 @@ class FastSIMD::DispatchClass, SIMD> : publ using TestReg = FS::Register; - template - static std::tuple GetArg_Helper( Ret ( F::* )( Args... ) const ); - - template - using GetArg = std::tuple_element_t; - - - template - static std::string AsString( T arg ) - { - std::stringstream ss; - ss << std::setprecision( 16 ) << std::boolalpha << arg; - return ss.str(); - } - - template - static std::string AsString( const FS::Register& arg ) + // Stores an oracle lane as T, float/double lanes can be given as their bits in an unsigned int + // Any unsigned type of the right size, uint64_t is unsigned long on LP64 so 64bit literals have a different type + template + static void StoreOracleLane( void* outResult, size_t lane, V value ) { - T scalarValues[N]; - FS::Store( scalarValues, arg ); - - std::ostringstream sstr; - sstr << "[" << AsString( scalarValues[0] ); - - for( size_t idx = 1; idx < N; idx++ ) + if constexpr( std::is_floating_point_v && std::is_integral_v && std::is_unsigned_v && sizeof( V ) == sizeof( T ) ) { - sstr << ", " << AsString( scalarValues[idx] ); - } - - sstr << "]"; - return sstr.str(); + std::memcpy( static_cast( outResult ) + lane, &value, sizeof( T ) ); + } + else + { + static_assert( std::is_same_v, "Oracle must return the test's element type" ); + static_cast( outResult )[lane] = value; + } } - template - static std::string AsString( const FS::Register, N, S>& arg ) + // How each kind of test argument and result is read from the inputs, passed to the oracle and stored as results + // Load is a call's argument, Lane is one lane's argument for the oracle, Store and StoreLane write results as Element + // Scalars are one value, the oracle sees the same value in every lane + template + struct TestValue { - FS::BitStorage bitMask = FS::BitMask( arg ); - - std::ostringstream sstr; - sstr << "[" << AsString( static_cast( bitMask & 1 ) ); + using Element = T; + static constexpr size_t Lanes = 1; - for( size_t idx = 1; idx < N; idx++ ) - { - bitMask >>= 1; + static constexpr TestArgType ArgType() { return { ElementOf(), false, Lanes }; } - sstr << ", " << AsString( static_cast( bitMask & 1 ) ); - } + static T Load( const TestInputs& inputs, size_t argIdx, size_t inIdx ) + { + return inputs.Get( argIdx )[inIdx]; + } - sstr << "]"; - return sstr.str(); - } + static T Lane( const TestInputs& inputs, size_t argIdx, size_t inIdx, size_t ) + { + return Load( inputs, argIdx, inIdx ); + } + static void Store( void* outResult, T result ) + { + *static_cast( outResult ) = result; + } - template - static size_t StoreOutput( void* outResult, T result ) - { - *reinterpret_cast( outResult ) = result; - return 1; - } + template + static void StoreLane( void* outResult, size_t lane, V value ) + { + StoreOracleLane( outResult, lane, value ); + } + }; template - static size_t StoreOutput( void* outResult, FS::Register result ) + struct TestValue> { - FS::Store( reinterpret_cast( outResult ), result ); - return N; - } + static_assert( N * sizeof( T ) <= kMaxTestResultBytes, "Test result is larger than the result buffer" ); - template - static size_t StoreOutput( void* outResult, FS::Register, N, S> result ) - { - bool* boolOut = reinterpret_cast( outResult ); - auto bitMask = FS::BitMask( result ); + using Element = T; + static constexpr size_t Lanes = N; + + static constexpr TestArgType ArgType() { return { ElementOf(), false, Lanes }; } - for( size_t idx = 0; idx < N; idx++ ) + static FS::Register Load( const TestInputs& inputs, size_t argIdx, size_t inIdx ) { - *boolOut++ = bitMask & 1; - bitMask >>= 1; - } - return N; - } + return FS::Load( inputs.Get( argIdx ) + inIdx ); + } + static T Lane( const TestInputs& inputs, size_t argIdx, size_t inIdx, size_t lane ) + { + return inputs.Get( argIdx )[inIdx + lane]; + } - template - struct GenArg - { - template - static T Load( size_t inIdx, size_t argIdx, T* rnd, ARGs... ) + static void Store( void* outResult, FS::Register result ) { - return *( rnd + inIdx + argIdx ); + FS::Store( static_cast( outResult ), result ); } - template - static T Load( size_t inIdx, size_t argIdx, U, ARGs... args ) + template + static void StoreLane( void* outResult, size_t lane, V value ) { - return Load( inIdx, argIdx, args... ); + StoreOracleLane( outResult, lane, value ); } }; - template - struct GenArg> + // Mask lanes are generated independently as 0 or 1, then compared != 0 to make the mask. The oracle sees them as bool + // Results are stored as integer lanes of the mask's width so every bit of each lane is compared + template + struct TestValue, N, S>> { - template - static FS::Register Load( size_t inIdx, size_t argIdx, T* rnd, ARGs... ) + static_assert( MN == 32 || ( MN == 64 && MB ), "Masks are tested as 32bit lanes, or 64bit from doubles" ); + + // Inputs are stored as double for 64bit masks, float for float masks, int32 for int masks + using Stored = std::conditional_t>; + using Element = std::conditional_t; + static constexpr size_t Lanes = N; + + static constexpr TestArgType ArgType() { return { ElementOf(), true, Lanes }; } + + static FS::Register, N, S> Load( const TestInputs& inputs, size_t argIdx, size_t inIdx ) { - return FS::Load( rnd + inIdx + argIdx * N ); + return FS::Load( inputs.Get( argIdx ) + inIdx ) != FS::Register( Stored( 0 ) ); } - template - static FS::Register Load( size_t inIdx, size_t argIdx, U, ARGs... args ) + static bool Lane( const TestInputs& inputs, size_t argIdx, size_t inIdx, size_t lane ) { - return Load( inIdx, argIdx, args... ); + return inputs.Get( argIdx )[inIdx + lane] != 0; } - }; - template - struct GenArg, N, S>> - { - template - static FS::Register, N, S> Load( size_t inIdx, size_t argIdx, std::conditional_t* rnd, ARGs... ) + // BitMask only shows the high bit of each lane, Masked keeps all of them + static void Store( void* outResult, FS::Register, N, S> result ) { - return FS::Load( rnd + inIdx + argIdx * N ) > FS::Load( rnd + inIdx + argIdx * N + 1 ); + if constexpr( MN == 32 ) + { + FS::Store( static_cast( outResult ), FS::Masked( result, FS::Register( -1 ) ) ); + } + else + { + // No 64bit int registers, a double with every bit set keeps all the lane's bits through Masked + FS::Store( static_cast( outResult ), FS::Masked( result, FS::Register( FromBits( ~uint64_t( 0 ) ) ) ) ); + } } - template - static FS::Register, N, S> Load( size_t inIdx, size_t argIdx, U, ARGs... args ) + static void StoreLane( void* outResult, size_t lane, bool value ) { - return Load( inIdx, argIdx, args... ); + static_cast( outResult )[lane] = value ? -1 : 0; } }; @@ -220,39 +193,46 @@ class FastSIMD::DispatchClass, SIMD> : publ template struct TestFunctionFactory> { - using ReturnType = std::invoke_result_t; + using Result = TestValue>; + static constexpr size_t ArgCount = sizeof...( ARGs ); + static constexpr size_t Lanes = std::max( { Result::Lanes, TestValue>::Lanes... } ); + + static_assert( Lanes <= kMaxTestLanes, "Too many lanes per call" ); + + static_assert( ArgCount <= kMaxTestArgs, "Too many test arguments" ); + + static void ArgTypes( TestArgType* argTypes ) + { + [[maybe_unused]] size_t argIdx = 0; + ( ( argTypes[argIdx++] = TestValue>::ArgType() ), ... ); + } static std::function Create( FUNC func ) { - return [func]( void* outResult, size_t inIdx, int32_t* rndInts, float* rndFloats ) + return [func]( void* outResult, size_t inIdx, const TestInputs& inputs ) -> size_t { - size_t argIdx = 0; - auto tuple = std::tuple{ GenArg>::Load( inIdx, argIdx++, rndInts, rndFloats )... }; + [[maybe_unused]] size_t argIdx = 0; + auto tuple = std::tuple{ TestValue>::Load( inputs, argIdx++, inIdx )... }; - return StoreOutput( outResult, std::apply( func, tuple ) ); + Result::Store( outResult, std::apply( func, tuple ) ); + return Result::Lanes; }; } - static std::function GetInputs() + // The oracle is called once per output lane with that lane's argument values + template + static std::function CreateOracle( ORACLE oracle ) { - return []( size_t inIdx, int32_t* rndInts, float* rndFloats ) -> std::string + return [oracle]( void* outResult, size_t inIdx, const TestInputs& inputs ) -> size_t { - if( !sizeof...( ARGs ) ) + for( size_t lane = 0; lane < Result::Lanes; lane++ ) { - return "No Inputs"; - } - - size_t argIdx = 0; - std::initializer_list tuple = { AsString( GenArg>::Load( inIdx, argIdx++, rndInts, rndFloats ) )... }; - - std::ostringstream inputsString; - inputsString << "\n"; + [[maybe_unused]] size_t argIdx = 0; + auto tuple = std::tuple{ TestValue>::Lane( inputs, argIdx++, inIdx, lane )... }; - std::copy( tuple.begin(), tuple.end() - 1, std::ostream_iterator( inputsString, "\n" ) ); - - inputsString << *(tuple.end() - 1); - - return inputsString.str(); + Result::StoreLane( outResult, lane, std::apply( oracle, tuple ) ); + } + return Result::Lanes; }; } }; @@ -261,223 +241,405 @@ class FastSIMD::DispatchClass, SIMD> : publ template TestData& RegisterTest( TestCollection& testMap, const char* name, FUNC func ) { + using Factory = TestFunctionFactory; + TestData data; - data.featureSet = SIMD; - data.relaxed = FastSIMD::IsRelaxed(); - data.returnType = GetReturn::ReturnType>::Type; + data.returnType = ReturnTypeOf(); + data.lanes = Factory::Lanes; + data.argCount = Factory::ArgCount; + Factory::ArgTypes( data.argTypes ); + data.testFunc = Factory::Create( func ); - data.testFunc = TestFunctionFactory::Create( func ); - data.inputsFunc = TestFunctionFactory::GetInputs(); + return testMap.emplace_back( name, std::move( data ) ).second; + } - return testMap.emplace_back( name, data ).second; + // oracle is a plain C++ version of the test, called per lane with plain values (masks as bool) + template + TestData& RegisterTest( TestCollection& testMap, const char* name, FUNC func, ORACLE oracle ) + { + TestData& data = RegisterTest( testMap, name, func ); + data.oracleFunc = TestFunctionFactory::CreateOracle( oracle ); + return data; } + // int32 arithmetic wraps on every tier, done in uint32 to avoid signed overflow + static int32_t Wrap( uint32_t v ) { return static_cast( v ); } + static uint32_t U( int32_t v ) { return static_cast( v ); } + + // x + 0.0f as the masked add/decrement implementations do: -0 becomes +0, everything else is unchanged + static float PlusZero( float a ) { return a == 0.0f ? 0.0f : a; } + + using TestRegi32 = TestReg; + using TestRegf32 = TestReg; + using TestRegm32 = typename TestRegf32::MaskType; + using TestRegm32i = typename TestRegi32::MaskType; + using In = InputSpec; + + static constexpr double denormMin = std::numeric_limits::denorm_min(); + static constexpr double floatMax = std::numeric_limits::max(); + static constexpr double pi = 3.14159265358979323846; + + // Lane input conditions for known failures + static bool DenormalArg0( const LaneArgs& args ) { return std::fpclassify( args.floats[0] ) == FP_SUBNORMAL; } + static bool DenormalArg1( const LaneArgs& args ) { return std::fpclassify( args.floats[1] ) == FP_SUBNORMAL; } + static bool QuotientOverflows( const LaneArgs& args ) { return std::isfinite( args.floats[0] ) && args.floats[1] != 0 && std::isinf( args.floats[0] / args.floats[1] ); } + static bool X86ReciprocalInput( const LaneArgs& args ) { return std::fpclassify( args.floats[0] ) == FP_SUBNORMAL || std::abs( args.floats[0] ) >= 0x1.ffcp125f; } + static bool ArmEstimateInput( const LaneArgs& args ) { return args.floats[0] == 0 || std::isinf( args.floats[0] ) || std::fpclassify( args.floats[0] ) == FP_SUBNORMAL; } + + // Known library behaviour, each scoped to the tiers that show it + static constexpr KnownFailure sse2RoundZeroSign = KnownFailure( + "Below SSE4.1 rounding converts through int32 (cvtps2dq/cvttps2dq), which has no -0, so negative results that round to 0 are +0", + Tiers::NonRelaxed, FeatureSet::SSE, FeatureSet::SSSE3 ).ZeroSignOnly(); + static constexpr KnownFailure sinReduceZeroSign = KnownFailure( + "Sin_Reduce uses Trunc, see f32 trunc: Sin( -0 ) is -0 below SSE4.1 and +0 elsewhere", + Tiers::NonRelaxed, FeatureSet::SSE, FeatureSet::SSSE3 ).ZeroSignOnly(); + static constexpr KnownFailure neonDivide = KnownFailure( + "NEON (ARMv7) has no divide, division is a reciprocal estimate plus Newton-Raphson steps which isn't correctly rounded", + Tiers::NonRelaxed, FeatureSet::NEON, FeatureSet::NEON ); + // vrecpe overflows to inf for |x| < 2^-128, then vrecps( x, inf ) is -inf, so the Newton-Raphson steps give inf of the wrong sign, or NaN for 0 / x + static constexpr KnownFailure neonDivideDenormal = KnownFailure( + "NEON (ARMv7) divide by a denormal: the reciprocal estimate overflows to inf and the Newton-Raphson steps flip its sign, giving -inf for +inf and NaN for 0", + Tiers::Relaxed, FeatureSet::NEON, FeatureSet::NEON ).ForInputs( DenormalArg1 ); + // The refined reciprocal can be 1 ULP low, e.g. 1 / 0.99999994 becomes 1, so FLT_MAX / 0.99999994 is FLT_MAX instead of inf + static constexpr KnownFailure neonDivideOverflow = KnownFailure( + "NEON (ARMv7) divide near overflow: the Newton-Raphson reciprocal can be 1 ULP low, so quotients that overflow to +-inf can give +-FLT_MAX", + Tiers::Relaxed, FeatureSet::NEON, FeatureSet::NEON ).ForInputs( QuotientOverflows ); + // Generic masked add is a + Masked( m, b ), so masked off lanes add +0. AVX512 512bit uses a native masked add that leaves them unchanged + static constexpr KnownFailure avx512MaskedAddZeroSign = KnownFailure( + "AVX512 512bit masked add leaves masked off lanes unchanged, the generic a + Masked( m, b ) turns -0 into +0", + RegisterBytes >= 64 ? Tiers::NonRelaxed : Tiers::None, FeatureSet::AVX512, FeatureSet::AVX512 ).ZeroSignOnly(); + // Relaxed x86 uses rcpps/rsqrtps, AVX512 512bit registers use rcp14/rsqrt14 which handle denormals + static constexpr FastSIMD::FeatureSet x86EstimateMax = RegisterBytes >= 64 ? FeatureSet::AVX2 : FeatureSet::AVX512; + static constexpr KnownFailure x86EstimateDenormals = KnownFailure( + "Relaxed rcpps/rsqrtps estimates treat denormal inputs as 0 and return inf", + Tiers::Relaxed, FeatureSet::SSE, x86EstimateMax ).ForInputs( DenormalArg0 ); + // The Intel SDM allows rcpps to flush results just above FLT_MIN to 0, for |x| just below 2^126 + static constexpr KnownFailure x86ReciprocalEstimate = KnownFailure( + "Relaxed rcpps treats denormal inputs as 0 and returns inf, and may flush results near FLT_MIN to 0", + Tiers::Relaxed, FeatureSet::SSE, x86EstimateMax ).ForInputs( X86ReciprocalInput ); + // Relaxed ARM refines vrsqrte/vrecpe with one Newton-Raphson step, which breaks where the estimate is 0 or inf + static constexpr KnownFailure armEstimateSpecials = KnownFailure( + "Relaxed ARM estimate + Newton-Raphson step: InvSqrt( +-0 ) and InvSqrt( inf ) are NaN (0 * inf), Reciprocal of a denormal is inf of the wrong sign", + Tiers::Relaxed, FeatureSet::NEON, FeatureSet::AARCH64 ).ForInputs( ArmEstimateInput ); +#if defined( _MSC_VER ) && !defined( __clang__ ) && defined( _DEBUG ) + // The scalar reference is wrong, so every tier and the oracle comparison see it + static constexpr Tiers msvcDebugAbsTiers = Tiers::All | Tiers::Oracle; +#else + static constexpr Tiers msvcDebugAbsTiers = Tiers::None; +#endif + static constexpr KnownFailure msvcDebugAbs = KnownFailure( + "Unoptimised MSVC calls the CRT fabsf for std::abs in the scalar FS::Abs, which quiets signalling NaNs", msvcDebugAbsTiers ); + TestCollection RegisterTests() override { TestCollection tests; + RegisterMaskTests( tests ); + RegisterIntTests( tests ); + RegisterFloatTests( tests ); + RegisterFloatMathTests( tests ); + RegisterConvertTests( tests ); + return tests; + } - using TestRegi32 = TestReg; - using TestRegf32 = TestReg; - using TestRegm32 = typename TestRegf32::MaskType; - using TestRegm32i = typename TestRegi32::MaskType; - - - RegisterTest( tests, "m32 bit mask", []( TestRegm32 a ) { return a; } ); + void RegisterMaskTests( TestCollection& tests ) + { + RegisterTest( tests, "m32 bit mask", []( TestRegm32 a ) { return a; }, []( bool a ) { return a; } ); RegisterTest( tests, "m32 any mask", []( TestRegm32 a ) { return FS::AnyMask( a ); } ); + RegisterTest( tests, "m32 bit mask to int", []( TestRegm32 a ) { return static_cast( FS::BitMask( a ) ); } ); + + RegisterTest( tests, "m32 bit and operator", std::bit_and(), []( bool a, bool b ) { return a && b; } ); + RegisterTest( tests, "m32 bit or operator", std::bit_or(), []( bool a, bool b ) { return a || b; } ); + RegisterTest( tests, "m32 bit xor operator", std::bit_xor(), []( bool a, bool b ) { return a != b; } ); + RegisterTest( tests, "m32 bit not operator", std::bit_not(), []( bool a ) { return !a; } ); + RegisterTest( tests, "m32 bit and not", []( TestRegm32 a, TestRegm32 b ) { return FS::BitwiseAndNot( a, b ); }, []( bool a, bool b ) { return a && !b; } ); + + RegisterTest( tests, "m32i bit mask", []( TestRegm32i a ) { return a; }, []( bool a ) { return a; } ); + RegisterTest( tests, "m32i any mask", []( TestRegm32i a ) { return FS::AnyMask( a ); } ); + RegisterTest( tests, "m32i bit mask to int", []( TestRegm32i a ) { return static_cast( FS::BitMask( a ) ); } ); + + RegisterTest( tests, "m32i bit and operator", std::bit_and(), []( bool a, bool b ) { return a && b; } ); + RegisterTest( tests, "m32i bit or operator", std::bit_or(), []( bool a, bool b ) { return a || b; } ); + RegisterTest( tests, "m32i bit xor operator", std::bit_xor(), []( bool a, bool b ) { return a != b; } ); + RegisterTest( tests, "m32i bit not operator", std::bit_not(), []( bool a ) { return !a; } ); + RegisterTest( tests, "m32i bit and not", []( TestRegm32i a, TestRegm32i b ) { return FS::BitwiseAndNot( a, b ); }, []( bool a, bool b ) { return a && !b; } ); + } - RegisterTest( tests, "m32 bit and operator", std::bit_and() ); - RegisterTest( tests, "m32 bit or operator", std::bit_or() ); - RegisterTest( tests, "m32 bit xor operator", std::bit_xor() ); - RegisterTest( tests, "m32 bit not operator", std::bit_not() ); - RegisterTest( tests, "m32 bit and not", []( TestRegm32 a, TestRegm32 b ) { return FS::BitwiseAndNot( a, b ); } ); - - RegisterTest( tests, "m32i bit mask", []( TestRegm32 a ) { return a; } ); - RegisterTest( tests, "m32i any mask", []( TestRegm32 a ) { return FS::AnyMask( a ); } ); - - RegisterTest( tests, "m32i bit and operator", std::bit_and() ); - RegisterTest( tests, "m32i bit or operator", std::bit_or() ); - RegisterTest( tests, "m32i bit xor operator", std::bit_xor() ); - RegisterTest( tests, "m32i bit not operator", std::bit_not() ); - RegisterTest( tests, "m32i bit and not", []( TestRegm32i a, TestRegm32i b ) { return FS::BitwiseAndNot( a, b ); } ); - - RegisterTest( tests, "i32 load store", []( TestRegi32 a ) { return a; } ); - RegisterTest( tests, "i32 load scalar", []( int32_t a ) { return TestRegi32( a ); } ); - RegisterTest( tests, "i32 splat", []( int32_t a ) { return FS::Splat( a ); } ); - RegisterTest( tests, "i32 extract 0", []( TestRegi32 a ) { return FS::Extract0( a ); } ); + void RegisterIntTests( TestCollection& tests ) + { + RegisterTest( tests, "i32 load store", []( TestRegi32 a ) { return a; }, []( int32_t a ) { return a; } ); + RegisterTest( tests, "i32 load scalar", []( int32_t a ) { return TestRegi32( a ); }, []( int32_t a ) { return a; } ); + RegisterTest( tests, "i32 splat", []( int32_t a ) { return FS::Splat( a ); }, []( int32_t a ) { return a; } ); + RegisterTest( tests, "i32 extract 0", []( TestRegi32 a ) { return FS::Extract0( a ); }, []( int32_t a ) { return a; } ); RegisterTest( tests, "i32 load incremented", []() { return FS::LoadIncremented(); } ); - RegisterTest( tests, "i32 plus operator", std::plus() ); - RegisterTest( tests, "i32 minus operator", std::minus() ); - RegisterTest( tests, "i32 multiply operator", std::multiplies() ); - - RegisterTest( tests, "i32 bit and operator", std::bit_and() ); - RegisterTest( tests, "i32 bit or operator", std::bit_or() ); - RegisterTest( tests, "i32 bit xor operator", std::bit_xor() ); - RegisterTest( tests, "i32 bit not operator", std::bit_not() ); - RegisterTest( tests, "i32 negate operator", std::negate() ); - RegisterTest( tests, "i32 bit and not", []( TestRegi32 a, TestRegi32 b ) { return FS::BitwiseAndNot( a, b ); } ); - - RegisterTest( tests, "i32 increment", []( TestRegi32 a ) { return FS::Increment( a ); } ); - RegisterTest( tests, "i32 decrement", []( TestRegi32 a ) { return FS::Decrement( a ); } ); - RegisterTest( tests, "i32 abs", []( TestRegi32 a ) { return FS::Abs( a ); } ); - RegisterTest( tests, "i32 min", []( TestRegi32 a, TestRegi32 b ) { return FS::Min( a, b ); } ); - RegisterTest( tests, "i32 max", []( TestRegi32 a, TestRegi32 b ) { return FS::Max( a, b ); } ); - - RegisterTest( tests, "i32 bit shift left scalar", []( TestRegi32 a, int b ) { return a << ( b & 31 ); } ); - RegisterTest( tests, "i32 bit shift right scalar", []( TestRegi32 a, int b ) { return a >> ( b & 31 ); } ); - RegisterTest( tests, "i32 bit shift right zero extend scalar", []( TestRegi32 a, int b ) { return FS::BitShiftRightZeroExtend( a, b & 31 ); } ); + RegisterTest( tests, "i32 plus operator", std::plus(), []( int32_t a, int32_t b ) { return Wrap( U( a ) + U( b ) ); } ); + RegisterTest( tests, "i32 minus operator", std::minus(), []( int32_t a, int32_t b ) { return Wrap( U( a ) - U( b ) ); } ); + RegisterTest( tests, "i32 multiply operator", std::multiplies(), []( int32_t a, int32_t b ) { return Wrap( U( a ) * U( b ) ); } ); + + RegisterTest( tests, "i32 bit and operator", std::bit_and(), []( int32_t a, int32_t b ) { return a & b; } ); + RegisterTest( tests, "i32 bit or operator", std::bit_or(), []( int32_t a, int32_t b ) { return a | b; } ); + RegisterTest( tests, "i32 bit xor operator", std::bit_xor(), []( int32_t a, int32_t b ) { return a ^ b; } ); + RegisterTest( tests, "i32 bit not operator", std::bit_not(), []( int32_t a ) { return ~a; } ); + RegisterTest( tests, "i32 negate operator", std::negate(), []( int32_t a ) { return Wrap( 0u - U( a ) ); } ); + // MSVC x86 has miscompiled operators taking an over-aligned lhs by value, the copy was elided and the result written into the caller's a + // Select( m, a OP b, a ) reads a again after each operator, the shape that showed it + RegisterTest( tests, "i32 operators keep lhs", []( TestRegm32 m, TestRegi32 a, TestRegi32 b ) { + return FS::Select( m, a + b, a ) ^ FS::Select( m, a - b, a ) ^ FS::Select( m, a * b, a ) ^ FS::Select( m, a & b, a ) ^ + FS::Select( m, a | b, a ) ^ FS::Select( m, a ^ b, a ) ^ FS::Select( m, a << 3, a ) ^ FS::Select( m, a >> 3, a ) ^ a; + }, []( bool m, int32_t a, int32_t b ) { + int32_t results[] = { Wrap( U( a ) + U( b ) ), Wrap( U( a ) - U( b ) ), Wrap( U( a ) * U( b ) ), a & b, a | b, a ^ b, Wrap( U( a ) << 3 ), a < 0 ? ~( ~a >> 3 ) : a >> 3 }; + int32_t combined = a; + for( int32_t result : results ) + { + combined ^= m ? result : a; + } + return combined; + } ); + RegisterTest( tests, "i32 bit and not", []( TestRegi32 a, TestRegi32 b ) { return FS::BitwiseAndNot( a, b ); }, []( int32_t a, int32_t b ) { return a & ~b; } ); + + RegisterTest( tests, "i32 increment", []( TestRegi32 a ) { return FS::Increment( a ); }, []( int32_t a ) { return Wrap( U( a ) + 1u ); } ); + RegisterTest( tests, "i32 decrement", []( TestRegi32 a ) { return FS::Decrement( a ); }, []( int32_t a ) { return Wrap( U( a ) - 1u ); } ); + RegisterTest( tests, "i32 abs", []( TestRegi32 a ) { return FS::Abs( a ); }, []( int32_t a ) { return a < 0 ? Wrap( 0u - U( a ) ) : a; } ); + RegisterTest( tests, "i32 min", []( TestRegi32 a, TestRegi32 b ) { return FS::Min( a, b ); }, []( int32_t a, int32_t b ) { return std::min( a, b ); } ); + RegisterTest( tests, "i32 max", []( TestRegi32 a, TestRegi32 b ) { return FS::Max( a, b ); }, []( int32_t a, int32_t b ) { return std::max( a, b ); } ); + + RegisterTest( tests, "i32 bit shift left scalar", []( TestRegi32 a, int b ) { return a << b; }, []( int32_t a, int32_t b ) { return Wrap( U( a ) << b ); } ).Inputs( In(), In::Range( 0, 31 ) ); + RegisterTest( tests, "i32 bit shift right scalar", []( TestRegi32 a, int b ) { return a >> b; }, []( int32_t a, int32_t b ) { return a < 0 ? ~( ~a >> b ) : a >> b; } ).Inputs( In(), In::Range( 0, 31 ) ); + RegisterTest( tests, "i32 bit shift right zero extend scalar", []( TestRegi32 a, int b ) { return FS::BitShiftRightZeroExtend( a, b ); }, []( int32_t a, int32_t b ) { return Wrap( U( a ) >> b ); } ).Inputs( In(), In::Range( 0, 31 ) ); //RegisterTest( tests, "i32 bit shift left", []( TestRegi32 a, TestRegi32 b ) { return a << FS::Min( TestRegi32( 31 ), FS::Abs( b ) ); } ); //RegisterTest( tests, "i32 bit shift right", []( TestRegi32 a, TestRegi32 b ) { return a >> FS::Min( TestRegi32( 31 ), FS::Abs( b ) ); } ); - RegisterTest( tests, "i32 equals operator", []( TestRegi32 a, TestRegi32 b ) { return a == b; } ); - RegisterTest( tests, "i32 equals operator alt", []( TestRegi32 a ) { return a == a; } ); - RegisterTest( tests, "i32 not equals operator", []( TestRegi32 a, TestRegi32 b ) { return a != b; } ); - RegisterTest( tests, "i32 not equals operator alt", []( TestRegi32 a ) { return a != a; } ); - RegisterTest( tests, "i32 less than operator", []( TestRegi32 a, TestRegi32 b ) { return a < b; } ); - RegisterTest( tests, "i32 greater than operator", []( TestRegi32 a, TestRegi32 b ) { return a > b; } ); - RegisterTest( tests, "i32 less equal than operator", []( TestRegi32 a, TestRegi32 b ) { return a <= b; } ); - RegisterTest( tests, "i32 greater equal than operator", []( TestRegi32 a, TestRegi32 b ) { return a >= b; } ); - - RegisterTest( tests, "i32 select", []( TestRegm32 m, TestRegi32 a, TestRegi32 b ) { return FS::Select( m, a, b ); } ); - RegisterTest( tests, "i32 select high bit", []( TestRegf32 m, TestRegi32 a, TestRegi32 b ) { return FS::SelectHighBit( m, a, b ); } ); - RegisterTest( tests, "i32 masked", []( TestRegm32 m, TestRegi32 a ) { return FS::Masked( m, a ); } ); - RegisterTest( tests, "i32 inv masked", []( TestRegm32 m, TestRegi32 a ) { return FS::InvMasked( m, a ); } ); - RegisterTest( tests, "i32 masked increment", []( TestRegm32 m, TestRegi32 a ) { return FS::MaskedIncrement( m, a ); } ); - RegisterTest( tests, "i32 masked decrement", []( TestRegm32 m, TestRegi32 a ) { return FS::MaskedDecrement( m, a ); } ); - RegisterTest( tests, "i32 masked add", []( TestRegm32 m, TestRegi32 a, TestRegi32 b ) { return FS::MaskedAdd( m, a, b ); } ); - RegisterTest( tests, "i32 masked sub", []( TestRegm32 m, TestRegi32 a, TestRegi32 b ) { return FS::MaskedSub( m, a, b ); } ); - RegisterTest( tests, "i32 masked mul", []( TestRegm32 m, TestRegi32 a, TestRegi32 b ) { return FS::MaskedMul( m, a, b ); } ); - RegisterTest( tests, "i32 inv masked add", []( TestRegm32 m, TestRegi32 a, TestRegi32 b ) { return FS::InvMaskedAdd( m, a, b ); } ); - RegisterTest( tests, "i32 inv masked sub", []( TestRegm32 m, TestRegi32 a, TestRegi32 b ) { return FS::InvMaskedSub( m, a, b ); } ); - RegisterTest( tests, "i32 inv masked mul", []( TestRegm32 m, TestRegi32 a, TestRegi32 b ) { return FS::InvMaskedMul( m, a, b ); } ); - - RegisterTest( tests, "f32 load store", []( TestRegf32 a ) { return a; } ); - RegisterTest( tests, "f32 load scalar", []( float a ) { return TestRegf32( a ); } ); - RegisterTest( tests, "f32 splat", []( float a ) { return FS::Splat( a ); } ); - RegisterTest( tests, "f32 extract 0", []( TestRegf32 a ) { return FS::Extract0( a ); } ); + RegisterTest( tests, "i32 equals operator", []( TestRegi32 a, TestRegi32 b ) { return a == b; }, []( int32_t a, int32_t b ) { return a == b; } ); + RegisterTest( tests, "i32 equals operator alt", []( TestRegi32 a ) { return a == a; }, []( int32_t ) { return true; } ); + RegisterTest( tests, "i32 not equals operator", []( TestRegi32 a, TestRegi32 b ) { return a != b; }, []( int32_t a, int32_t b ) { return a != b; } ); + RegisterTest( tests, "i32 not equals operator alt", []( TestRegi32 a ) { return a != a; }, []( int32_t ) { return false; } ); + RegisterTest( tests, "i32 less than operator", []( TestRegi32 a, TestRegi32 b ) { return a < b; }, []( int32_t a, int32_t b ) { return a < b; } ); + RegisterTest( tests, "i32 greater than operator", []( TestRegi32 a, TestRegi32 b ) { return a > b; }, []( int32_t a, int32_t b ) { return a > b; } ); + RegisterTest( tests, "i32 less equal than operator", []( TestRegi32 a, TestRegi32 b ) { return a <= b; }, []( int32_t a, int32_t b ) { return a <= b; } ); + RegisterTest( tests, "i32 greater equal than operator", []( TestRegi32 a, TestRegi32 b ) { return a >= b; }, []( int32_t a, int32_t b ) { return a >= b; } ); + + RegisterTest( tests, "i32 select", []( TestRegm32 m, TestRegi32 a, TestRegi32 b ) { return FS::Select( m, a, b ); }, []( bool m, int32_t a, int32_t b ) { return m ? a : b; } ); + // i32's own mask type, as int compares return + RegisterTest( tests, "i32 select int mask", []( TestRegm32i m, TestRegi32 a, TestRegi32 b ) { return FS::Select( m, a, b ); }, []( bool m, int32_t a, int32_t b ) { return m ? a : b; } ); + RegisterTest( tests, "i32 masked int mask", []( TestRegm32i m, TestRegi32 a ) { return FS::Masked( m, a ); }, []( bool m, int32_t a ) { return m ? a : 0; } ); + RegisterTest( tests, "i32 masked increment int mask", []( TestRegm32i m, TestRegi32 a ) { return FS::MaskedIncrement( m, a ); }, []( bool m, int32_t a ) { return m ? Wrap( U( a ) + 1u ) : a; } ); + RegisterTest( tests, "i32 select high bit", []( TestRegf32 m, TestRegi32 a, TestRegi32 b ) { return FS::SelectHighBit( m, a, b ); }, + []( float m, int32_t a, int32_t b ) { return ( ToBits( m ) >> 31 ) ? a : b; } ); + RegisterTest( tests, "i32 masked", []( TestRegm32 m, TestRegi32 a ) { return FS::Masked( m, a ); }, []( bool m, int32_t a ) { return m ? a : 0; } ); + RegisterTest( tests, "i32 inv masked", []( TestRegm32 m, TestRegi32 a ) { return FS::InvMasked( m, a ); }, []( bool m, int32_t a ) { return m ? 0 : a; } ); + RegisterTest( tests, "i32 masked increment", []( TestRegm32 m, TestRegi32 a ) { return FS::MaskedIncrement( m, a ); }, []( bool m, int32_t a ) { return m ? Wrap( U( a ) + 1u ) : a; } ); + RegisterTest( tests, "i32 masked decrement", []( TestRegm32 m, TestRegi32 a ) { return FS::MaskedDecrement( m, a ); }, []( bool m, int32_t a ) { return m ? Wrap( U( a ) - 1u ) : a; } ); + RegisterTest( tests, "i32 masked add", []( TestRegm32 m, TestRegi32 a, TestRegi32 b ) { return FS::MaskedAdd( m, a, b ); }, []( bool m, int32_t a, int32_t b ) { return m ? Wrap( U( a ) + U( b ) ) : a; } ); + RegisterTest( tests, "i32 masked sub", []( TestRegm32 m, TestRegi32 a, TestRegi32 b ) { return FS::MaskedSub( m, a, b ); }, []( bool m, int32_t a, int32_t b ) { return m ? Wrap( U( a ) - U( b ) ) : a; } ); + RegisterTest( tests, "i32 masked mul", []( TestRegm32 m, TestRegi32 a, TestRegi32 b ) { return FS::MaskedMul( m, a, b ); }, []( bool m, int32_t a, int32_t b ) { return m ? Wrap( U( a ) * U( b ) ) : a; } ); + RegisterTest( tests, "i32 inv masked add", []( TestRegm32 m, TestRegi32 a, TestRegi32 b ) { return FS::InvMaskedAdd( m, a, b ); }, []( bool m, int32_t a, int32_t b ) { return m ? a : Wrap( U( a ) + U( b ) ); } ); + RegisterTest( tests, "i32 inv masked sub", []( TestRegm32 m, TestRegi32 a, TestRegi32 b ) { return FS::InvMaskedSub( m, a, b ); }, []( bool m, int32_t a, int32_t b ) { return m ? a : Wrap( U( a ) - U( b ) ); } ); + RegisterTest( tests, "i32 inv masked mul", []( TestRegm32 m, TestRegi32 a, TestRegi32 b ) { return FS::InvMaskedMul( m, a, b ); }, []( bool m, int32_t a, int32_t b ) { return m ? a : Wrap( U( a ) * U( b ) ); } ); + } + + void RegisterFloatTests( TestCollection& tests ) + { + RegisterTest( tests, "f32 load store", []( TestRegf32 a ) { return a; }, []( float a ) { return ToBits( a ); } ).BitExact(); + RegisterTest( tests, "f32 load scalar", []( float a ) { return TestRegf32( a ); }, []( float a ) { return ToBits( a ); } ).BitExact(); + RegisterTest( tests, "f32 splat", []( float a ) { return FS::Splat( a ); }, []( float a ) { return ToBits( a ); } ).BitExact(); + RegisterTest( tests, "f32 extract 0", []( TestRegf32 a ) { return FS::Extract0( a ); }, []( float a ) { return ToBits( a ); } ).BitExact(); RegisterTest( tests, "f32 load incremented", []() { return FS::LoadIncremented(); } ); - RegisterTest( tests, "f32 plus operator", std::plus() ); - RegisterTest( tests, "f32 minus operator", std::minus() ); - RegisterTest( tests, "f32 multiply operator", std::multiplies() ); - RegisterTest( tests, "f32 divide operator", std::divides() ).relaxedAccuracy = 64; - - RegisterTest( tests, "f32 fused multiply add", []( TestRegf32 a, TestRegf32 b ) { return FS::FMulAdd( a, TestRegf32( -1 ), b ); } ); - RegisterTest( tests, "f32 fused multiply sub", []( TestRegf32 a, TestRegf32 b ) { return FS::FMulSub( a, TestRegf32( -1 ), b ); } ); - RegisterTest( tests, "f32 fused negative multiply add", []( TestRegf32 a, TestRegf32 b ) { return FS::FNMulAdd( a, TestRegf32( -1 ), b ); } ); - RegisterTest( tests, "f32 fused negative multiply sub", []( TestRegf32 a, TestRegf32 b ) { return FS::FNMulSub( a, TestRegf32( -1 ), b ); } ); - - RegisterTest( tests, "f32 bit and operator", std::bit_and() ); - RegisterTest( tests, "f32 bit or operator", std::bit_or() ); - RegisterTest( tests, "f32 bit xor operator", std::bit_xor() ); - RegisterTest( tests, "f32 bit not operator", std::bit_not() ); - RegisterTest( tests, "f32 negate operator", std::negate() ); - RegisterTest( tests, "f32 bit and not", []( TestRegf32 a, TestRegf32 b ) { return FS::BitwiseAndNot( a, b ); } ); - - RegisterTest( tests, "f32 equals operator", []( TestRegf32 a, TestRegf32 b ) { return a == b; } ); - RegisterTest( tests, "f32 greater equal than operator", []( TestRegf32 a, TestRegf32 b ) { return a >= b; } ); - RegisterTest( tests, "f32 not equals operator", []( TestRegf32 a, TestRegf32 b ) { return a != b; } ); - RegisterTest( tests, "f32 less than operator", []( TestRegf32 a, TestRegf32 b ) { return a < b; } ); - RegisterTest( tests, "f32 greater than operator", []( TestRegf32 a, TestRegf32 b ) { return a > b; } ); - RegisterTest( tests, "f32 less equal than operator", []( TestRegf32 a, TestRegf32 b ) { return a <= b; } ); - - RegisterTest( tests, "f32 increment", []( TestRegf32 a ) { return FS::Increment( a ); } ); - RegisterTest( tests, "f32 decrement", []( TestRegf32 a ) { return FS::Decrement( a ); } ); - RegisterTest( tests, "f32 abs", []( TestRegf32 a ) { return FS::Abs( a ); } ); - RegisterTest( tests, "f32 min", []( TestRegf32 a, TestRegf32 b ) { return FS::Min( a, b ); } ); - RegisterTest( tests, "f32 max", []( TestRegf32 a, TestRegf32 b ) { return FS::Max( a, b ); } ); - - RegisterTest( tests, "f32 select", []( TestRegm32 m, TestRegf32 a, TestRegf32 b ) { return FS::Select( m, a, b ); } ); - RegisterTest( tests, "f32 select high bit", []( TestRegi32 m, TestRegf32 a, TestRegf32 b ) { return FS::SelectHighBit( m, a, b ); } ); - RegisterTest( tests, "f32 masked", []( TestRegm32 m, TestRegf32 a ) { return FS::Masked( m, a ); } ); - RegisterTest( tests, "f32 inv masked", []( TestRegm32 m, TestRegf32 a ) { return FS::InvMasked( m, a ); } ); - RegisterTest( tests, "f32 masked increment", []( TestRegm32 m, TestRegf32 a ) { return FS::MaskedIncrement( m, a ); } ); - RegisterTest( tests, "f32 masked decrement", []( TestRegm32 m, TestRegf32 a ) { return FS::MaskedDecrement( m, a ); } ); - RegisterTest( tests, "f32 masked add", []( TestRegm32 m, TestRegf32 a, TestRegf32 b ) { return FS::MaskedAdd( m, a, b ); } ); - RegisterTest( tests, "f32 masked sub", []( TestRegm32 m, TestRegf32 a, TestRegf32 b ) { return FS::MaskedSub( m, a, b ); } ); - RegisterTest( tests, "f32 masked mul", []( TestRegm32 m, TestRegf32 a, TestRegf32 b ) { return FS::MaskedMul( m, a, b ); } ); - RegisterTest( tests, "f32 inv masked add", []( TestRegm32 m, TestRegf32 a, TestRegf32 b ) { return FS::InvMaskedAdd( m, a, b ); } ); - RegisterTest( tests, "f32 inv masked sub", []( TestRegm32 m, TestRegf32 a, TestRegf32 b ) { return FS::InvMaskedSub( m, a, b ); } ); - RegisterTest( tests, "f32 inv masked mul", []( TestRegm32 m, TestRegf32 a, TestRegf32 b ) { return FS::InvMaskedMul( m, a, b ); } ); - - RegisterTest( tests, "f32 round", []( TestRegf32 a ) { return FS::Round( a ); } ); - RegisterTest( tests, "f32 ceil", []( TestRegf32 a ) { return FS::Ceil( a ); } ); - RegisterTest( tests, "f32 floor", []( TestRegf32 a ) { return FS::Floor( a ); } ); - RegisterTest( tests, "f32 trunc", []( TestRegf32 a ) { return FS::Trunc( a ); } ); - RegisterTest( tests, "f32 signbit", []( TestRegf32 a, TestRegf32 b ) { return FS::SignBit( a ) ^ b; } ); - RegisterTest( tests, "f32 modulus", []( TestRegf32 a, TestRegf32 b ) { - a = FS::Min( FS::Max( a, TestRegf32( -100.0f ) ), TestRegf32( 100.0f ) ); - b = FS::Min( FS::Max( FS::Abs( b ), TestRegf32( 0.25f ) ), TestRegf32( 100.0f ) ) | FS::SignBit( b ); - return FS::Modulus( a, b ); - } ).relaxedAccuracy = 16384; - - RegisterTest( tests, "f32 sqrt", []( TestRegf32 a ) { return FS::Sqrt( FS::Min( FS::Max( FS::Abs( a ), TestRegf32( 1.e-16f ) ), TestRegf32( 1.e+16f ) ) ); } ); - RegisterTest( tests, "f32 inv sqrt", []( TestRegf32 a ) { return FS::InvSqrt( FS::Min( FS::Max( FS::Abs( a ), TestRegf32( 1.e-16f ) ), TestRegf32( 1.e+16f ) ) ); } ).relaxedAccuracy = 8192; - RegisterTest( tests, "f32 reciprocal", []( TestRegf32 a ) { - a = FS::Min( FS::Max( FS::Abs( a ), TestRegf32( 1.e-16f ) ), TestRegf32( 1.e+16f ) ) | FS::SignBit( a ); - return FS::Reciprocal( a ); - } ).relaxedAccuracy = 8192; - - RegisterTest( tests, "f32 cos", []( TestRegf32 a ) { return FS::Cos( FS::Min( FS::Max( a, TestRegf32( -4096.0f ) ), TestRegf32( 4096.0f ) ) ); } ).relaxedAccuracy = 16384; - RegisterTest( tests, "f32 sin", []( TestRegf32 a ) { return FS::Sin( FS::Min( FS::Max( a, TestRegf32( -4096.0f ) ), TestRegf32( 4096.0f ) ) ); } ).relaxedAccuracy = 16384; - RegisterTest( tests, "f32 tan", []( TestRegf32 a ) { return FS::Tan( FS::Min( FS::Max( a, TestRegf32( -4096.0f ) ), TestRegf32( 4096.0f ) ) ); } ).relaxedAccuracy = 16384; - RegisterTest( tests, "f32 acos", []( TestRegf32 a ) { return FS::ACos( FS::Min( FS::Max( a, TestRegf32( -0.999f ) ), TestRegf32( 0.999f ) ) ); } ).relaxedAccuracy = 8192; - RegisterTest( tests, "f32 asin", []( TestRegf32 a ) { - // Clamp to avoid extreme near-zero and near-1 values - a = FS::Min( FS::Max( a, TestRegf32( -0.999f ) ), TestRegf32( 0.999f ) ); - auto tooSmall = FS::Abs( a ) < TestRegf32( 1.e-6f ); - a = FS::Select( tooSmall, TestRegf32( 0.0f ), a ); - return FS::ASin( a ); - } ).relaxedAccuracy = 8192; - RegisterTest( tests, "f32 atan", []( TestRegf32 a ) { return FS::ATan( FS::Min( FS::Max( a, TestRegf32( -100.0f ) ), TestRegf32( 100.0f ) ) ); } ).relaxedAccuracy = 16384; - RegisterTest( tests, "f32 atan2", []( TestRegf32 a, TestRegf32 b ) { - a = FS::Min( FS::Max( a, TestRegf32( -1000.0f ) ), TestRegf32( 1000.0f ) ); - b = FS::Min( FS::Max( b, TestRegf32( -1000.0f ) ), TestRegf32( 1000.0f ) ); - // Avoid very small denominators that cause precision issues - auto bTooSmall = FS::Abs( b ) < TestRegf32( 0.001f ); - b = FS::Select( bTooSmall, TestRegf32( 0.001f ) | FS::SignBit( b ), b ); - return FS::ATan2( a, b ); - } ).relaxedAccuracy = 16384; - RegisterTest( tests, "f32 exp", []( TestRegf32 a ) { return FS::Exp( FS::Min( FS::Max( a, TestRegf32( -87.0f ) ), TestRegf32( 88.0f ) ) ); } ).relaxedAccuracy = 8192; - RegisterTest( tests, "f32 exp2", []( TestRegf32 a ) { return FS::Exp2( FS::Min( FS::Max( a, TestRegf32( -126.0f ) ), TestRegf32( 127.0f ) ) ); } ).relaxedAccuracy = 8192; - RegisterTest( tests, "f32 log", []( TestRegf32 a ) { return FS::Log( FS::Min( FS::Max( FS::Abs( a ), TestRegf32( 1.e-20f ) ), TestRegf32( 1.e+20f ) ) ); } ).relaxedAccuracy = 16384; - RegisterTest( tests, "f32 log2", []( TestRegf32 a ) { - a = FS::Min( FS::Max( FS::Abs( a ), TestRegf32( 1.e-20f ) ), TestRegf32( 1.e+20f ) ); - // Avoid values very close to 1.0 which cause precision issues - auto nearOne = FS::Abs( a - TestRegf32( 1.0f ) ) < TestRegf32( 1.e-5f ); - a = FS::Select( nearOne, TestRegf32( 1.0f ), a ); - return FS::Log2( a ); - } ).relaxedAccuracy = 16384; - RegisterTest( tests, "f32 pow", []( TestRegf32 a, TestRegf32 b ) { - a = FS::Min( FS::Max( FS::Abs( a ), TestRegf32( 1.e-8f ) ), TestRegf32( 1.e+8f ) ); - b = FS::Min( FS::Max( b, TestRegf32( -8.0f ) ), TestRegf32( 8.0f ) ); - return FS::Pow( a, b ); - } ).relaxedAccuracy = 16384; - RegisterTest( tests, "f32 pow zero base", []( TestRegf32 a, TestRegf32 b ) { - // Negative inputs become a zero base, ranges avoid Exp overflow - auto zero = a < TestRegf32( 0.0f ); - a = FS::Min( FS::Max( a, TestRegf32( 1.e-4f ) ), TestRegf32( 1.e+4f ) ); - a = FS::Select( zero, TestRegf32( 0.0f ), a ); - b = FS::Min( FS::Max( FS::Abs( b ), TestRegf32( 0.1f ) ), TestRegf32( 8.0f ) ); - return FS::Pow( a, b ); - } ).relaxedAccuracy = 16384; - - RegisterTest( tests, "i32 convert to f32", []( TestRegi32 a ) { return FS::Convert( a ); } ); - RegisterTest( tests, "f32 convert to i32", []( TestRegf32 a ) { return FS::Convert( FS::Min( FS::Max( a, TestRegf32( -2147483647 - 1 ) ), TestRegf32( 2147483520 ) ) ); } ); - - RegisterTest( tests, "f32 cast to i32", []( TestRegf32 a ) { return FS::Cast( a ); } ); - RegisterTest( tests, "i32 cast to f32", []( TestRegi32 a ) { return FS::Cast( a ); } ); - - if constexpr( !( SIMD & FeatureFlag::AVX512_F ) ) + RegisterTest( tests, "f32 plus operator", std::plus(), []( float a, float b ) { return a + b; } ); + RegisterTest( tests, "f32 minus operator", std::minus(), []( float a, float b ) { return a - b; } ); + RegisterTest( tests, "f32 multiply operator", std::multiplies(), []( float a, float b ) { return a * b; } ); + RegisterTest( tests, "f32 divide operator", std::divides(), []( float a, float b ) { return a / b; } ).RelaxedAccuracy( 64 ) + .Known( neonDivide, neonDivideDenormal, neonDivideOverflow ); + + // b = -1 keeps a * b exact, so fused and unfused results match + RegisterTest( tests, "f32 fused multiply add", []( TestRegf32 a, TestRegf32 b ) { return FS::FMulAdd( a, TestRegf32( -1 ), b ); }, []( float a, float b ) { return -a + b; } ); + RegisterTest( tests, "f32 fused multiply sub", []( TestRegf32 a, TestRegf32 b ) { return FS::FMulSub( a, TestRegf32( -1 ), b ); }, []( float a, float b ) { return -a - b; } ); + RegisterTest( tests, "f32 fused negative multiply add", []( TestRegf32 a, TestRegf32 b ) { return FS::FNMulAdd( a, TestRegf32( -1 ), b ); }, []( float a, float b ) { return b + a; } ); + RegisterTest( tests, "f32 fused negative multiply sub", []( TestRegf32 a, TestRegf32 b ) { return FS::FNMulSub( a, TestRegf32( -1 ), b ); }, []( float a, float b ) { return -b + a; } ); + + // 12 bit significands keep a * b exact, so fused and unfused results match while every argument is independent + const In exactProduct = In::Magnitude( 1.e-15, 1.e+15 ).Bits( 12 ); + RegisterTest( tests, "f32 fused multiply add exact product", []( TestRegf32 a, TestRegf32 b, TestRegf32 c ) { return FS::FMulAdd( a, b, c ); }, + []( float a, float b, float c ) { float ab = a * b; return ab + c; } ).Inputs( exactProduct, exactProduct, In::Range( -1.e+30, 1.e+30 ) ); + RegisterTest( tests, "f32 fused multiply sub exact product", []( TestRegf32 a, TestRegf32 b, TestRegf32 c ) { return FS::FMulSub( a, b, c ); }, + []( float a, float b, float c ) { float ab = a * b; return ab - c; } ).Inputs( exactProduct, exactProduct, In::Range( -1.e+30, 1.e+30 ) ); + RegisterTest( tests, "f32 fused negative multiply add exact product", []( TestRegf32 a, TestRegf32 b, TestRegf32 c ) { return FS::FNMulAdd( a, b, c ); }, + []( float a, float b, float c ) { float ab = a * b; return c - ab; } ).Inputs( exactProduct, exactProduct, In::Range( -1.e+30, 1.e+30 ) ); + RegisterTest( tests, "f32 fused negative multiply sub exact product", []( TestRegf32 a, TestRegf32 b, TestRegf32 c ) { return FS::FNMulSub( a, b, c ); }, + []( float a, float b, float c ) { float ab = a * b; return -c - ab; } ).Inputs( exactProduct, exactProduct, In::Range( -1.e+30, 1.e+30 ) ); + + RegisterTest( tests, "f32 bit and operator", std::bit_and(), []( float a, float b ) { return ToBits( a ) & ToBits( b ); } ).BitExact(); + RegisterTest( tests, "f32 bit or operator", std::bit_or(), []( float a, float b ) { return ToBits( a ) | ToBits( b ); } ).BitExact(); + RegisterTest( tests, "f32 bit xor operator", std::bit_xor(), []( float a, float b ) { return ToBits( a ) ^ ToBits( b ); } ).BitExact(); + RegisterTest( tests, "f32 bit not operator", std::bit_not(), []( float a ) { return ~ToBits( a ); } ).BitExact(); + RegisterTest( tests, "f32 negate operator", std::negate(), []( float a ) { return ToBits( a ) ^ 0x80000000u; } ).BitExact(); + // See i32 operators keep lhs. Divide isn't exact on NEON, its lhs is read again by FS::Modulus, see f32 modulus exact + RegisterTest( tests, "f32 operators keep lhs", []( TestRegm32 m, TestRegf32 a, TestRegf32 b ) { + return FS::Select( m, a + b, a ) ^ FS::Select( m, a - b, a ) ^ FS::Select( m, a * b, a ) ^ + FS::Select( m, a & b, a ) ^ FS::Select( m, a | b, a ) ^ FS::Select( m, a ^ b, a ) ^ a; + }, []( bool m, float a, float b ) { + uint32_t results[] = { ToBits( a + b ), ToBits( a - b ), ToBits( a * b ), ToBits( a ) & ToBits( b ), ToBits( a ) | ToBits( b ), ToBits( a ) ^ ToBits( b ) }; + uint32_t combined = ToBits( a ); + for( uint32_t result : results ) + { + combined ^= m ? result : ToBits( a ); + } + return combined; + } ).Inputs( In(), In::Finite(), In::Finite() ).BitExact(); + RegisterTest( tests, "f32 bit and not", []( TestRegf32 a, TestRegf32 b ) { return FS::BitwiseAndNot( a, b ); }, []( float a, float b ) { return ToBits( a ) & ~ToBits( b ); } ).BitExact(); + + RegisterTest( tests, "f32 equals operator", []( TestRegf32 a, TestRegf32 b ) { return a == b; }, []( float a, float b ) { return a == b; } ); + RegisterTest( tests, "f32 greater equal than operator", []( TestRegf32 a, TestRegf32 b ) { return a >= b; }, []( float a, float b ) { return a >= b; } ); + RegisterTest( tests, "f32 not equals operator", []( TestRegf32 a, TestRegf32 b ) { return a != b; }, []( float a, float b ) { return a != b; } ); + RegisterTest( tests, "f32 less than operator", []( TestRegf32 a, TestRegf32 b ) { return a < b; }, []( float a, float b ) { return a < b; } ); + RegisterTest( tests, "f32 greater than operator", []( TestRegf32 a, TestRegf32 b ) { return a > b; }, []( float a, float b ) { return a > b; } ); + RegisterTest( tests, "f32 less equal than operator", []( TestRegf32 a, TestRegf32 b ) { return a <= b; }, []( float a, float b ) { return a <= b; } ); + + RegisterTest( tests, "f32 increment", []( TestRegf32 a ) { return FS::Increment( a ); }, []( float a ) { return a + 1.0f; } ); + RegisterTest( tests, "f32 decrement", []( TestRegf32 a ) { return FS::Decrement( a ); }, []( float a ) { return a - 1.0f; } ); + RegisterTest( tests, "f32 abs", []( TestRegf32 a ) { return FS::Abs( a ); }, []( float a ) { return ToBits( a ) & 0x7FFFFFFFu; } ).BitExact().Known( msvcDebugAbs ); + // Relaxed min/max may use instructions with different NaN handling, those are checked non relaxed in the full range tests + RegisterTest( tests, "f32 min", []( TestRegf32 a, TestRegf32 b ) { return FS::Min( a, b ); }, []( float a, float b ) { return std::min( a, b ); } ).Inputs( In::NoNaN(), In::NoNaN() ); + RegisterTest( tests, "f32 max", []( TestRegf32 a, TestRegf32 b ) { return FS::Max( a, b ); }, []( float a, float b ) { return std::max( a, b ); } ).Inputs( In::NoNaN(), In::NoNaN() ); + + RegisterTest( tests, "f32 select", []( TestRegm32 m, TestRegf32 a, TestRegf32 b ) { return FS::Select( m, a, b ); }, []( bool m, float a, float b ) { return ToBits( m ? a : b ); } ).BitExact(); + RegisterTest( tests, "f32 select high bit", []( TestRegi32 m, TestRegf32 a, TestRegf32 b ) { return FS::SelectHighBit( m, a, b ); }, []( int32_t m, float a, float b ) { return ToBits( m < 0 ? a : b ); } ).BitExact(); + RegisterTest( tests, "f32 masked", []( TestRegm32 m, TestRegf32 a ) { return FS::Masked( m, a ); }, []( bool m, float a ) { return m ? ToBits( a ) : 0u; } ).BitExact(); + RegisterTest( tests, "f32 inv masked", []( TestRegm32 m, TestRegf32 a ) { return FS::InvMasked( m, a ); }, []( bool m, float a ) { return m ? 0u : ToBits( a ); } ).BitExact(); + RegisterTest( tests, "f32 masked increment", []( TestRegm32 m, TestRegf32 a ) { return FS::MaskedIncrement( m, a ); }, []( bool m, float a ) { return m ? a + 1.0f : a; } ); + RegisterTest( tests, "f32 masked decrement", []( TestRegm32 m, TestRegf32 a ) { return FS::MaskedDecrement( m, a ); }, []( bool m, float a ) { return m ? a - 1.0f : PlusZero( a ); } ); + + RegisterTest( tests, "f32 masked add", []( TestRegm32 m, TestRegf32 a, TestRegf32 b ) { return FS::MaskedAdd( m, a, b ); }, + []( bool m, float a, float b ) { return m ? a + b : PlusZero( a ); } ).Known( avx512MaskedAddZeroSign ); + RegisterTest( tests, "f32 masked sub", []( TestRegm32 m, TestRegf32 a, TestRegf32 b ) { return FS::MaskedSub( m, a, b ); }, []( bool m, float a, float b ) { return m ? a - b : a; } ); + RegisterTest( tests, "f32 masked mul", []( TestRegm32 m, TestRegf32 a, TestRegf32 b ) { return FS::MaskedMul( m, a, b ); }, []( bool m, float a, float b ) { return m ? a * b : a; } ); + RegisterTest( tests, "f32 inv masked add", []( TestRegm32 m, TestRegf32 a, TestRegf32 b ) { return FS::InvMaskedAdd( m, a, b ); }, + []( bool m, float a, float b ) { return m ? PlusZero( a ) : a + b; } ).Known( avx512MaskedAddZeroSign ); + RegisterTest( tests, "f32 inv masked sub", []( TestRegm32 m, TestRegf32 a, TestRegf32 b ) { return FS::InvMaskedSub( m, a, b ); }, []( bool m, float a, float b ) { return m ? a : a - b; } ); + RegisterTest( tests, "f32 inv masked mul", []( TestRegm32 m, TestRegf32 a, TestRegf32 b ) { return FS::InvMaskedMul( m, a, b ); }, []( bool m, float a, float b ) { return m ? a : a * b; } ); + + RegisterTest( tests, "f32 round", []( TestRegf32 a ) { return FS::Round( a ); }, []( float a ) { return std::rint( a ); } ) + .Known( sse2RoundZeroSign ); + RegisterTest( tests, "f32 ceil", []( TestRegf32 a ) { return FS::Ceil( a ); }, []( float a ) { return std::ceil( a ); } ) + .Known( sse2RoundZeroSign ); + RegisterTest( tests, "f32 floor", []( TestRegf32 a ) { return FS::Floor( a ); }, []( float a ) { return std::floor( a ); } ) + .Known( sse2RoundZeroSign ); + RegisterTest( tests, "f32 trunc", []( TestRegf32 a ) { return FS::Trunc( a ); }, []( float a ) { return std::trunc( a ); } ) + .Known( sse2RoundZeroSign ); + RegisterTest( tests, "f32 signbit", []( TestRegf32 a, TestRegf32 b ) { return FS::SignBit( a ) ^ b; }, []( float a, float b ) { return ( ToBits( a ) & 0x80000000u ) ^ ToBits( b ); } ).BitExact(); + RegisterTest( tests, "f32 modulus", []( TestRegf32 a, TestRegf32 b ) { return FS::Modulus( a, b ); } ) + .Inputs( In::Range( -100, 100 ), In::Magnitude( 0.25, 100 ) ).RelaxedAccuracy( 16384 ) + .Known( KnownFailure( "Scalar uses std::fmod, SIMD and relaxed use a - trunc( a / b ) * b which isn't exact" ) ); + // b is a power of 2 and a is never tiny, so a - trunc( a / b ) * b is exact and must match std::fmod + RegisterTest( tests, "f32 modulus exact", []( TestRegf32 a, TestRegf32 b ) { return FS::Modulus( a, b ); }, []( float a, float b ) { return std::fmod( a, b ); } ) + .Inputs( In::Magnitude( 1.e-30, 100 ), In::Magnitude( 0.25, 64 ).Bits( 1 ) ) + .Known( KnownFailure( "a - trunc( a / b ) * b gives +0 for exact negative multiples, std::fmod gives -0", Tiers::NonRelaxed ).ZeroSignOnly() ); + } + + void RegisterFloatMathTests( TestCollection& tests ) + { + // Range reduction boundaries for trig + const In trigRange = In::Range( -4096, 4096 ).Specials( { pi / 4, pi / 2, pi, 3 * pi / 2, 2 * pi, 4 * pi, 100 * pi, 1000 * pi, + -pi / 4, -pi / 2, -pi, -3 * pi / 2, -2 * pi, -4 * pi, -100 * pi, -1000 * pi } ); + const In smallTrigRange = In::Range( -2 * pi, 2 * pi ).Specials( { pi / 4, pi / 2, pi, 3 * pi / 2, 2 * pi, + -pi / 4, -pi / 2, -pi, -3 * pi / 2, -2 * pi } ); + + RegisterTest( tests, "f32 sqrt", []( TestRegf32 a ) { return FS::Sqrt( a ); }, []( float a ) { return std::sqrt( a ); } ); + // Relaxed x86 uses the rcpps/rsqrtps estimates, their 1.5 * 2^-12 relative error is up to ~6144 ULP + RegisterTest( tests, "f32 inv sqrt", []( TestRegf32 a ) { return FS::InvSqrt( a ); }, []( float a ) { return 1.0f / std::sqrt( a ); } ).Inputs( In::Range( 1.e-16, 1.e+16 ) ).RelaxedAccuracy( 8192 ) + .Known( neonDivide ); + RegisterTest( tests, "f32 reciprocal", []( TestRegf32 a ) { return FS::Reciprocal( a ); }, []( float a ) { return 1.0f / a; } ).Inputs( In::Magnitude( 1.e-16, 1.e+16 ) ).RelaxedAccuracy( 8192 ) + .Known( neonDivide ); + + // Absolute error allowance is ~1 ULP of the largest input, float inputs that large are only that precise + RegisterTest( tests, "f32 cos", []( TestRegf32 a ) { return FS::Cos( a ); } ).Inputs( trigRange ).RelaxedAccuracy( 4, 5.e-4f ); + RegisterTest( tests, "f32 sin", []( TestRegf32 a ) { return FS::Sin( a ); } ).Inputs( trigRange ).RelaxedAccuracy( 4, 5.e-4f ) + .Known( sinReduceZeroSign ); + // Within +-2pi range reduction is exact apart from the float rounding of pi and 2pi (<= ~2.6e-7), so the polynomials are checked closely + RegisterTest( tests, "f32 cos small range", []( TestRegf32 a ) { return FS::Cos( a ); } ).Inputs( smallTrigRange ).RelaxedAccuracy( 4, 5.e-7f ); + RegisterTest( tests, "f32 sin small range", []( TestRegf32 a ) { return FS::Sin( a ); } ).Inputs( smallTrigRange ).RelaxedAccuracy( 4, 5.e-7f ) + .Known( sinReduceZeroSign ); + // The absolute allowances above hide relative accuracy near 0 + RegisterTest( tests, "f32 sin near 0", []( TestRegf32 a ) { return FS::Sin( a ); } ).Inputs( In::Magnitude( denormMin, 0.01 ) ).RelaxedAccuracy( 4 ); + // Tan is ill conditioned near its poles, so relaxed accuracy is checked between them + RegisterTest( tests, "f32 tan", []( TestRegf32 a ) { return FS::Tan( a ); } ).Inputs( In::Range( -1.5, 1.5 ).Specials( { pi / 4, -pi / 4 } ) ).RelaxedAccuracy( 64 ) + .Known( neonDivide ); + // Between the poles at +-pi/2 and +-3pi/2, Tan_Reduce subtracts pi and folds the sign, near +-pi the float rounding of pi is absolute error + RegisterTest( tests, "f32 tan reduced", []( TestRegf32 a ) { return FS::Tan( a ); } ).Inputs( In::Magnitude( 1.6, 4.6 ).Specials( { pi, 3 * pi / 4, 5 * pi / 4 } ) ).RelaxedAccuracy( 128, 3.e-7f ) + .Known( neonDivide ); + RegisterTest( tests, "f32 acos", []( TestRegf32 a ) { return FS::ACos( a ); } ).Inputs( In::Range( -1, 1 ) ).RelaxedAccuracy( 8 ); + // ASin is K_HALF_PI - ACos( x ), near 0 that cancels and ACos's error (~2 ULP of pi/2) becomes absolute error, +0 for |x| <= 2^-25 + RegisterTest( tests, "f32 asin", []( TestRegf32 a ) { return FS::ASin( a ); } ).Inputs( In::Range( -1, 1 ) ).RelaxedAccuracy( 32, 3.e-7f ); + // Relaxed x86 ATan uses the rcpps estimate for |x| > 1, its 1.5 * 2^-12 relative error is up to ~3072 ULP of atan near 1 + // Relaxed scalar is std::atan, so FS::ATan itself is checked much more closely + RegisterTest( tests, "f32 atan", []( TestRegf32 a ) { return FS::ATan( a ); } ).Inputs( In::NoNaN() ).RelaxedAccuracy( 4096 ).ScalarRelaxedAccuracy( 64 ) + .Known( neonDivide ); + RegisterTest( tests, "f32 atan2", []( TestRegf32 a, TestRegf32 b ) { return FS::ATan2( a, b ); } ).Inputs( In::NoNaN(), In::NoNaN() ).RelaxedAccuracy( 4096 ).ScalarRelaxedAccuracy( 64 ) + .Known( KnownFailure( "Relaxed scalar uses std::atan2: for atan2( +-0, -0 ) it returns +-pi where FS::ATan2 returns 0, and for atan2( +-inf, +-inf ) +-pi/4 or +-3pi/4 where FS::ATan2 returns NaN (inf / inf)", + Tiers::Relaxed, FeatureSet::SCALAR, FeatureSet::SCALAR ), + neonDivide, neonDivideDenormal ); + // Keep relaxed scalar checking FS::ATan2 against std::atan2 everywhere except both inputs 0 + RegisterTest( tests, "f32 atan2 nonzero y", []( TestRegf32 a, TestRegf32 b ) { return FS::ATan2( a, b ); } ).Inputs( In::Magnitude( denormMin, floatMax ), In::NoNaN() ).RelaxedAccuracy( 4096 ).ScalarRelaxedAccuracy( 64 ) + .Known( neonDivide, neonDivideDenormal ); + RegisterTest( tests, "f32 atan2 nonzero x", []( TestRegf32 a, TestRegf32 b ) { return FS::ATan2( a, b ); } ).Inputs( In::NoNaN(), In::Magnitude( denormMin, floatMax ) ).RelaxedAccuracy( 4096 ).ScalarRelaxedAccuracy( 64 ) + .Known( neonDivide, neonDivideDenormal ); + // FS::Exp clamps its input to +-88.3762626647949, the range reaches just below that and keeps results above FLT_MIN + RegisterTest( tests, "f32 exp", []( TestRegf32 a ) { return FS::Exp( a ); } ).Inputs( In::Range( -87.3, 88.37 ) ).RelaxedAccuracy( 8 ); + RegisterTest( tests, "f32 exp2", []( TestRegf32 a ) { return FS::Exp2( a ); } ).Inputs( In::Range( -126, 127 ) ).RelaxedAccuracy( 8 ); + RegisterTest( tests, "f32 log", []( TestRegf32 a ) { return FS::Log( a ); } ).Inputs( In::Range( 1.e-20, 1.e+20 ) ).RelaxedAccuracy( 8 ); + // Log2 reduces the mantissa to [1,2), just below 1 it cancels to ~2e-7 absolute error + RegisterTest( tests, "f32 log2", []( TestRegf32 a ) { return FS::Log2( a ); } ).Inputs( In::Range( 1.e-20, 1.e+20 ) ).RelaxedAccuracy( 8, 3.e-7f ); + // Results stay within float range, FS::Pow saturates where std::pow returns inf + // Pow is Exp( b * Log( a ) ), so Log's relative error is scaled by b * ln( a ), up to ~74 here + RegisterTest( tests, "f32 pow", []( TestRegf32 a, TestRegf32 b ) { return FS::Pow( a, b ); } ) + .Inputs( In::Range( 1.e-4, 1.e+4 ), In::Range( -8, 8 ) ).RelaxedAccuracy( 256 ); + RegisterTest( tests, "f32 pow zero base", []( TestRegf32 b ) { return FS::Pow( TestRegf32( 0.0f ), b ); }, []( float ) { return 0.0f; } ).Inputs( In::Range( 0.1, 8 ) ); + + RegisterTest( tests, "f32 inv sqrt full range", []( TestRegf32 a ) { return FS::InvSqrt( a ); }, []( float a ) { return 1.0f / std::sqrt( a ); } ).RelaxedAccuracy( 8192 ) + .Known( neonDivide, x86EstimateDenormals, armEstimateSpecials ); + RegisterTest( tests, "f32 reciprocal full range", []( TestRegf32 a ) { return FS::Reciprocal( a ); }, []( float a ) { return 1.0f / a; } ).RelaxedAccuracy( 8192 ) + .Known( neonDivide, x86ReciprocalEstimate, armEstimateSpecials ); + + // Relaxed accuracy is only meaningful over the ranges above, non relaxed results must still match exactly over the whole domain + if constexpr( !FastSIMD::IsRelaxed() ) { - RegisterTest( tests, "m32 cast to i32", []( TestRegm32 a ) { return FS_BIND_INTRINSIC( FS::Cast> )( FS_BIND_INTRINSIC( FS::Cast )( a ) ); } ); - RegisterTest( tests, "m32 cast to f32", []( TestRegm32 a ) { return FS_BIND_INTRINSIC( FS::Cast> )( FS_BIND_INTRINSIC( FS::Cast )( a ) ); } ); - RegisterTest( tests, "m32i cast to i32", []( TestRegm32i a ) { return FS_BIND_INTRINSIC( FS::Cast )( FS_BIND_INTRINSIC( FS::Cast )( a ) ); } ); - RegisterTest( tests, "m32i cast to f32", []( TestRegm32i a ) { return FS_BIND_INTRINSIC( FS::Cast )( FS_BIND_INTRINSIC( FS::Cast )( a ) ); } ); + RegisterTest( tests, "f32 min full range", []( TestRegf32 a, TestRegf32 b ) { return FS::Min( a, b ); }, []( float a, float b ) { return std::min( a, b ); } ); + RegisterTest( tests, "f32 max full range", []( TestRegf32 a, TestRegf32 b ) { return FS::Max( a, b ); }, []( float a, float b ) { return std::max( a, b ); } ); + RegisterTest( tests, "f32 cos full range", []( TestRegf32 a ) { return FS::Cos( a ); } ); + RegisterTest( tests, "f32 sin full range", []( TestRegf32 a ) { return FS::Sin( a ); } ) + .Known( sinReduceZeroSign ); + RegisterTest( tests, "f32 tan full range", []( TestRegf32 a ) { return FS::Tan( a ); } ) + .Known( neonDivide ); + // Exp clamps its input, no NaN since converting it to int is undefined in the scalar reference + RegisterTest( tests, "f32 exp full range", []( TestRegf32 a ) { return FS::Exp( a ); } ) + .Inputs( In::NoNaN().Specials( { 88.3762626647949, -88.3762626647949, 88.72283935546875, -87.33654022216797, -103.97207641601562 } ) ); + RegisterTest( tests, "f32 log full range", []( TestRegf32 a ) { return FS::Log( a ); } ).Inputs( In::Range( denormMin, floatMax ) ); + RegisterTest( tests, "f32 log2 full range", []( TestRegf32 a ) { return FS::Log2( a ); } ).Inputs( In::Range( denormMin, floatMax ) ); + RegisterTest( tests, "f32 pow full range", []( TestRegf32 a, TestRegf32 b ) { return FS::Pow( a, b ); } ).Inputs( In::Range( denormMin, floatMax ), In::Finite() ); + } + } + + void RegisterConvertTests( TestCollection& tests ) + { + RegisterTest( tests, "i32 convert to f32", []( TestRegi32 a ) { return FS::Convert( a ); }, []( int32_t a ) { return static_cast( a ); } ); + RegisterTest( tests, "f32 convert to i32", []( TestRegf32 a ) { return FS::Convert( a ); }, []( float a ) { return static_cast( std::rint( a ) ); } ) + .Inputs( In::Range( -2147483648.0, 2147483520.0 ) ); + + RegisterTest( tests, "f32 cast to i32", []( TestRegf32 a ) { return FS::Cast( a ); }, []( float a ) { return static_cast( ToBits( a ) ); } ); + RegisterTest( tests, "i32 cast to f32", []( TestRegi32 a ) { return FS::Cast( a ); }, []( int32_t a ) { return U( a ); } ).BitExact(); + + // Cast is a raw reinterpret, a mask's bits as another type differ between SIMD levels (and k-registers can't be cast) + // Only the round trip back to the mask is defined, so that's all these test + if constexpr( !( SIMD & FeatureFlag::AVX512_F ) || RegisterBytes < 64 ) + { + RegisterTest( tests, "m32 cast to i32", []( TestRegm32 a ) { return FS_BIND_INTRINSIC( FS::Cast> )( FS_BIND_INTRINSIC( FS::Cast )( a ) ); }, []( bool a ) { return a; } ); + RegisterTest( tests, "m32 cast to f32", []( TestRegm32 a ) { return FS_BIND_INTRINSIC( FS::Cast> )( FS_BIND_INTRINSIC( FS::Cast )( a ) ); }, []( bool a ) { return a; } ); + RegisterTest( tests, "m32i cast to i32", []( TestRegm32i a ) { return FS_BIND_INTRINSIC( FS::Cast )( FS_BIND_INTRINSIC( FS::Cast )( a ) ); }, []( bool a ) { return a; } ); + RegisterTest( tests, "m32i cast to f32", []( TestRegm32i a ) { return FS_BIND_INTRINSIC( FS::Cast )( FS_BIND_INTRINSIC( FS::Cast )( a ) ); }, []( bool a ) { return a; } ); } - return tests; } }; -template class FastSIMD::RegisterDispatchClass>; +// Each register size is built as its own dispatch library, see tests/CMakeLists.txt +template class FastSIMD::RegisterDispatchClass>; diff --git a/tests/test_inputs.cpp b/tests/test_inputs.cpp new file mode 100644 index 0000000..984af37 --- /dev/null +++ b/tests/test_inputs.cpp @@ -0,0 +1,652 @@ +#include "test_inputs.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// Calls func with a value of the type an argument's inputs are stored as +template +static void WithStoredType( TestElement element, FUNC&& func ) +{ + switch( element ) + { + case TestElement::Int32: + return func( int32_t() ); + case TestElement::Float: + return func( float() ); + case TestElement::Double: + return func( double() ); + } +} + +static uint64_t SplitMix64( uint64_t& state ) +{ + uint64_t z = ( state += 0x9E3779B97F4A7C15ull ); + z = ( z ^ ( z >> 30 ) ) * 0xBF58476D1CE4E5B9ull; + z = ( z ^ ( z >> 27 ) ) * 0x94D049BB133111EBull; + return z ^ ( z >> 31 ); +} + +// FNV-1a, stable across compilers unlike std::hash +static uint64_t HashName( std::string_view name ) +{ + uint64_t hash = 0xCBF29CE484222325ull; + for( char c : name ) + { + hash = ( hash ^ static_cast( c ) ) * 0x100000001B3ull; + } + return hash; +} + +void ValidateInputs( std::string_view testName, const TestData& test ) +{ + auto fail = [&]( size_t argIdx, const char* problem ) + { + std::string message = "Invalid input spec for \"" + std::string( testName ) + "\" argument " + std::to_string( argIdx ) + ": " + problem; + std::cerr << message << std::endl; + throw std::runtime_error( message ); + }; + + constexpr double floatMax = std::numeric_limits::max(); + + for( size_t argIdx = 0; argIdx < kMaxTestArgs; argIdx++ ) + { + const InputSpec& spec = test.inputs[argIdx]; + + if( argIdx >= test.argCount ) + { + if( !spec.IsDefault() ) + { + fail( argIdx, "test has fewer arguments than input specs" ); + } + continue; + } + + TestArgType type = test.argTypes[argIdx]; + if( type.mask && !spec.IsDefault() ) + { + fail( argIdx, "mask arguments are always generated from lane patterns, they take no input spec" ); + } + int maxBits = type.element == TestElement::Float ? FloatTraits::kSignificandBits : type.element == TestElement::Double ? FloatTraits::kSignificandBits : 0; + if( spec.significandBits < 0 || spec.significandBits > maxBits ) + { + fail( argIdx, maxBits ? "significand bits must be in [1, 24] for floats, [1, 53] for doubles" : "significand bits only apply to floats and doubles" ); + } + for( double special : spec.specials ) + { + if( std::isnan( special ) ) + { + fail( argIdx, "special values can't be NaN, the default spec includes NaNs" ); + } + } + if( ( spec.kind == InputSpec::Kind::NoNaN || spec.kind == InputSpec::Kind::Finite ) && type.element == TestElement::Int32 ) + { + fail( argIdx, "NoNaN and Finite only apply to floats and doubles" ); + } + if( spec.kind != InputSpec::Kind::Range && spec.kind != InputSpec::Kind::Magnitude ) + { + continue; + } + if( !std::isfinite( spec.lo ) || !std::isfinite( spec.hi ) || spec.lo > spec.hi ) + { + fail( argIdx, "range bounds must be finite with lo <= hi" ); + } + if( spec.kind == InputSpec::Kind::Magnitude && spec.lo <= 0 ) + { + fail( argIdx, "magnitude lo must be > 0" ); + } + if( type.element == TestElement::Float && ( std::abs( spec.lo ) > floatMax || std::abs( spec.hi ) > floatMax ) ) + { + fail( argIdx, "range bounds must be within float range" ); + } + if( type.element == TestElement::Int32 && + ( std::ceil( spec.lo ) > std::floor( spec.hi ) || std::ceil( spec.lo ) < -2147483648.0 || std::floor( spec.hi ) > 2147483647.0 ) ) + { + fail( argIdx, "int range must contain an integer and be within int32 range" ); + } + } +} + +std::string DescribeInputs( const TestData& test, const TestInputs& inputs, size_t inIdx ) +{ + if( !test.argCount ) + { + return "No Inputs"; + } + + std::ostringstream description; + description << std::setprecision( 17 ) << std::boolalpha; + + for( size_t argIdx = 0; argIdx < test.argCount; argIdx++ ) + { + TestArgType type = test.argTypes[argIdx]; + bool isRegister = type.lanes > 1; + + // Registers list their lanes, mask lanes are true or false + WithStoredType( type.element, [&]( auto stored ) + { + const auto* values = inputs.Get( argIdx ) + inIdx; + + description << "\n" << ( isRegister ? "[" : "" ); + for( size_t lane = 0; lane < type.lanes; lane++ ) + { + description << ( lane ? ", " : "" ); + if( type.mask ) + { + description << ( values[lane] != 0 ); + } + else + { + description << values[lane]; + } + } + description << ( isRegister ? "]" : "" ); + } ); + } + return description.str(); +} + +LaneArgs GetLaneArgs( const TestData& test, const TestInputs& inputs, size_t inIdx, size_t lane ) +{ + LaneArgs args = {}; + + for( size_t argIdx = 0; argIdx < test.argCount; argIdx++ ) + { + TestArgType type = test.argTypes[argIdx]; + size_t idx = inIdx + ( type.lanes > 1 ? lane : 0 ); + + WithStoredType( type.element, [&]( auto stored ) + { + auto value = inputs.Get( argIdx )[idx]; + if( type.mask ) + { + args.Set( argIdx, value != 0 ); + } + else + { + args.Set( argIdx, value ); + } + } ); + } + return args; +} + +template +static T RangeLo( const InputSpec& spec ) { return static_cast( spec.lo ); } +template +static T RangeHi( const InputSpec& spec ) { return static_cast( spec.hi ); } + +// Int bounds are the integers inside [lo, hi], ValidateInputs checks there is at least one +static int64_t IntLo( const InputSpec& spec ) { return static_cast( std::ceil( spec.lo ) ); } +static int64_t IntHi( const InputSpec& spec ) { return static_cast( std::floor( spec.hi ) ); } + +// Drops the low significand bits, finite values only +template +static T Truncate( T value, int significandBits ) +{ + constexpr int kBits = FloatTraits::kSignificandBits; + if( significandBits == 0 || significandBits >= kBits || !std::isfinite( value ) ) + { + return value; + } + using Bits = typename FloatTraits::Bits; + Bits lowBits = ( Bits( 1 ) << ( kBits - significandBits ) ) - 1; + return FromBits( ToBits( value ) & ~lowBits ); +} + +template +static bool InFloatDomain( const InputSpec& spec, T value ) +{ + switch( spec.kind ) + { + case InputSpec::Kind::Any: + return true; + case InputSpec::Kind::NoNaN: + return !std::isnan( value ); + case InputSpec::Kind::Finite: + return std::isfinite( value ); + case InputSpec::Kind::Range: + return value >= RangeLo( spec ) && value <= RangeHi( spec ); + case InputSpec::Kind::Magnitude: + return std::abs( value ) >= RangeLo( spec ) && std::abs( value ) <= RangeHi( spec ); + } + return false; +} + +// Values of every magnitude that commonly break SIMD code +static std::vector AnySpecials( const InputSpec& spec, float ) +{ + constexpr float inf = std::numeric_limits::infinity(); + constexpr float floatMin = std::numeric_limits::min(); + constexpr float denormMin = std::numeric_limits::denorm_min(); + const float denormMax = FromBits( 0x007FFFFF ); + + std::vector candidates = { + 0.0f, -0.0f, 1.0f, -1.0f, 0.5f, -0.5f, 1.5f, -1.5f, 2.5f, -2.5f, + 0.49999997f, -0.49999997f, 0.99999994f, -0.99999994f, 1.0000001f, -1.0000001f, + floatMin, -floatMin, denormMin, -denormMin, denormMax, -denormMax, + std::numeric_limits::max(), -std::numeric_limits::max(), + 8388607.5f, -8388607.5f, 8388608.0f, -8388608.0f, 16777216.0f, -16777216.0f, + 2147483520.0f, -2147483520.0f, 2147483648.0f, -2147483648.0f, + 3.14159274f, -3.14159274f, 1.e-10f, 1.e+10f, + }; + if( spec.kind != InputSpec::Kind::Finite ) + { + candidates.insert( candidates.end(), { inf, -inf } ); + } + if( spec.kind == InputSpec::Kind::Any ) + { + // Quiet, negative, signalling and payload NaNs + for( uint32_t bits : { 0x7FC00000u, 0xFFC00000u, 0x7F800001u, 0xFF800001u, 0x7FFFFFFFu, 0xFFC00001u } ) + { + candidates.push_back( FromBits( bits ) ); + } + } + return candidates; +} + +static std::vector AnySpecials( const InputSpec& spec, double ) +{ + constexpr double inf = std::numeric_limits::infinity(); + constexpr double doubleMin = std::numeric_limits::min(); + constexpr double denormMin = std::numeric_limits::denorm_min(); + const double denormMax = FromBits( 0x000FFFFFFFFFFFFFull ); + constexpr double floatMax = std::numeric_limits::max(); + constexpr double floatMin = std::numeric_limits::min(); + constexpr double floatDenormMin = std::numeric_limits::denorm_min(); + + std::vector candidates = { + 0.0, -0.0, 1.0, -1.0, 0.5, -0.5, 1.5, -1.5, 2.5, -2.5, + std::nextafter( 0.5, 0.0 ), -std::nextafter( 0.5, 0.0 ), std::nextafter( 1.0, 0.0 ), -std::nextafter( 1.0, 0.0 ), + std::nextafter( 1.0, 2.0 ), -std::nextafter( 1.0, 2.0 ), + doubleMin, -doubleMin, denormMin, -denormMin, denormMax, -denormMax, + std::numeric_limits::max(), -std::numeric_limits::max(), + // Integer precision limits, and the int32 convert range + 4503599627370495.5, -4503599627370495.5, 4503599627370496.0, -4503599627370496.0, 9007199254740992.0, -9007199254740992.0, + 2147483647.0, -2147483648.0, 2147483647.5, -2147483648.5, 2147483648.0, -2147483649.0, + // Narrowing to float: overflow, float denormals, and rounding ties + floatMax, -floatMax, 3.4028235677973366e+38, -3.4028235677973366e+38, floatMin, -floatMin, floatDenormMin, -floatDenormMin, + floatDenormMin / 2, 1.0 + 0x1p-24, -( 1.0 + 0x1p-24 ), 1.0 + 0x1.8p-23, 1.0 + 0x1p-23 + 0x1p-25, + 3.141592653589793, -3.141592653589793, 1.e-10, 1.e+10, 1.e-300, 1.e+300, + }; + if( spec.kind != InputSpec::Kind::Finite ) + { + candidates.insert( candidates.end(), { inf, -inf } ); + } + if( spec.kind == InputSpec::Kind::Any ) + { + // Quiet, negative, signalling and payload NaNs, including payloads narrowing to float drops + for( uint64_t bits : { 0x7FF8000000000000ull, 0xFFF8000000000000ull, 0x7FF0000000000001ull, 0xFFF0000000000001ull, + 0x7FFFFFFFFFFFFFFFull, 0xFFF8000000000001ull, 0x7FF4000000000000ull } ) + { + candidates.push_back( FromBits( bits ) ); + } + } + return candidates; +} + +template +static std::vector FloatSpecials( const InputSpec& spec ) +{ + constexpr T inf = std::numeric_limits::infinity(); + constexpr T typeMin = std::numeric_limits::min(); + constexpr T denormMin = std::numeric_limits::denorm_min(); + const T denormMax = std::nextafter( typeMin, T( 0 ) ); + const T belowHalf = std::nextafter( T( 0.5 ), T( 0 ) ); + const T belowOne = std::nextafter( T( 1 ), T( 0 ) ); + const T aboveOne = std::nextafter( T( 1 ), T( 2 ) ); + std::vector candidates; + + switch( spec.kind ) + { + case InputSpec::Kind::Any: + case InputSpec::Kind::NoNaN: + case InputSpec::Kind::Finite: + candidates = AnySpecials( spec, T() ); + break; + + case InputSpec::Kind::Range: + candidates = { RangeLo( spec ), RangeHi( spec ), std::nextafter( RangeLo( spec ), inf ), std::nextafter( RangeHi( spec ), -inf ) }; + for( T v : { T( 0 ), T( 0.5 ), T( 1 ), T( 1.5 ), T( 2.5 ), belowHalf, belowOne, aboveOne, typeMin, denormMin, denormMax, T( 1.e-10 ), T( 1.e-30 ) } ) + { + candidates.insert( candidates.end(), { v, -v } ); + } + break; + + case InputSpec::Kind::Magnitude: + for( T v : { RangeLo( spec ), RangeHi( spec ), std::nextafter( RangeLo( spec ), inf ), std::nextafter( RangeHi( spec ), -inf ), + T( 0.5 ), T( 1 ), T( 2 ), belowOne, aboveOne, typeMin, denormMin, denormMax } ) + { + candidates.insert( candidates.end(), { v, -v } ); + } + break; + } + + for( double special : spec.specials ) + { + T v = static_cast( special ); + candidates.insert( candidates.end(), { v, std::nextafter( v, inf ), std::nextafter( v, -inf ) } ); + } + + std::vector specials; + for( T candidate : candidates ) + { + T v = Truncate( candidate, spec.significandBits ); + + // Compare bits to keep both 0 and -0, and each NaN + bool duplicate = std::any_of( specials.begin(), specials.end(), [v]( T s ) { return ToBits( s ) == ToBits( v ); } ); + + if( InFloatDomain( spec, v ) && !duplicate ) + { + specials.push_back( v ); + } + } + if( specials.empty() ) + { + throw std::runtime_error( "Input spec has no special values in its domain" ); + } + return specials; +} + +static std::vector IntSpecials( const InputSpec& spec ) +{ + constexpr int32_t intMax = std::numeric_limits::max(); + constexpr int32_t intMin = std::numeric_limits::min(); + std::vector candidates; + + switch( spec.kind ) + { + case InputSpec::Kind::Any: + case InputSpec::Kind::NoNaN: + case InputSpec::Kind::Finite: + candidates = { + 0, 1, -1, 2, -2, 31, 32, 33, -31, -32, + intMax, intMin, intMax - 1, intMin + 1, + 0x7F800000, static_cast( 0xFF800000 ), 0x7FC00000, 0x7F800001, // inf, -inf, NaN, signalling NaN float bits + 0x3F800000, 0x00800000, 0x0000FFFF, 0x00010000, 0x55555555, static_cast( 0xAAAAAAAA ), + }; + break; + case InputSpec::Kind::Range: + candidates = { IntLo( spec ), IntHi( spec ), IntLo( spec ) + 1, IntHi( spec ) - 1, 0, 1, -1 }; + break; + case InputSpec::Kind::Magnitude: + for( int64_t v : { IntLo( spec ), IntHi( spec ), IntLo( spec ) + 1, IntHi( spec ) - 1, int64_t( 1 ) } ) + { + candidates.insert( candidates.end(), { v, -v } ); + } + break; + } + + for( double special : spec.specials ) + { + candidates.push_back( static_cast( std::round( std::min( std::max( special, -2147483648.0 ), 2147483647.0 ) ) ) ); + } + + std::vector specials; + for( int64_t v : candidates ) + { + bool inDomain = true; + switch( spec.kind ) + { + case InputSpec::Kind::Any: + case InputSpec::Kind::NoNaN: + case InputSpec::Kind::Finite: + inDomain = v >= intMin && v <= intMax; + break; + case InputSpec::Kind::Range: + inDomain = v >= IntLo( spec ) && v <= IntHi( spec ); + break; + case InputSpec::Kind::Magnitude: + inDomain = std::abs( v ) >= IntLo( spec ) && std::abs( v ) <= IntHi( spec ); + break; + } + + if( inDomain && std::find( specials.begin(), specials.end(), v ) == specials.end() ) + { + specials.push_back( static_cast( v ) ); + } + } + return specials; +} + +// Uniform [0, 1) +static double RandomUnit( std::mt19937_64& rng ) +{ + return static_cast( rng() >> 11 ) * ( 1.0 / 9007199254740992.0 ); +} + +// Log scaled magnitude within [lo, hi], lo > 0: a uniform power of 2 binade, then uniform within its part of [lo, hi] +template +static T RandomMagnitude( double lo, double hi, std::mt19937_64& rng ) +{ + int loExp, hiExp; + std::frexp( lo, &loExp ); + std::frexp( hi, &hiExp ); + + int exponent = loExp + static_cast( rng() % static_cast( hiExp - loExp + 1 ) ); + double binadeLo = std::max( lo, std::ldexp( 0.5, exponent ) ); + double binadeHi = std::min( hi, std::ldexp( 1.0, exponent ) ); + T value = static_cast( binadeLo + ( binadeHi - binadeLo ) * RandomUnit( rng ) ); + + return std::min( std::max( value, static_cast( lo ) ), static_cast( hi ) ); +} + +template +static T RandomFloat( const InputSpec& spec, std::mt19937_64& rng ) +{ + constexpr double denormMin = std::numeric_limits::denorm_min(); + + for( ;; ) + { + T value = 0; + + switch( spec.kind ) + { + case InputSpec::Kind::Any: + case InputSpec::Kind::NoNaN: + case InputSpec::Kind::Finite: + value = FromBits( static_cast::Bits>( rng() ) ); + break; + + case InputSpec::Kind::Range: + if( rng() & 1 ) + { + value = static_cast( spec.lo + ( spec.hi - spec.lo ) * RandomUnit( rng ) ); + } + else + { + // Log scaled half covers small magnitudes, including denormals, that uniform values almost never hit + bool positive = spec.lo >= 0 || ( spec.hi > 0 && ( rng() & 1 ) ); + double maxMag = positive ? spec.hi : -spec.lo; + double minMag = positive ? std::max( spec.lo, 0.0 ) : std::max( -spec.hi, 0.0 ); + value = RandomMagnitude( std::max( minMag, denormMin ), std::max( maxMag, denormMin ), rng ); + value = positive ? value : -value; + } + break; + + case InputSpec::Kind::Magnitude: + value = RandomMagnitude( spec.lo, spec.hi, rng ); + value = ( rng() & 1 ) ? -value : value; + break; + } + + value = Truncate( value, spec.significandBits ); + + if( InFloatDomain( spec, value ) ) + { + return value; + } + } +} + +static int32_t RandomInt( const InputSpec& spec, std::mt19937_64& rng ) +{ + switch( spec.kind ) + { + case InputSpec::Kind::Any: + case InputSpec::Kind::NoNaN: + case InputSpec::Kind::Finite: + return static_cast( rng() ); + + case InputSpec::Kind::Range: + return static_cast( IntLo( spec ) + static_cast( rng() % static_cast( IntHi( spec ) - IntLo( spec ) + 1 ) ) ); + + case InputSpec::Kind::Magnitude: + { + int64_t value = IntLo( spec ) + static_cast( rng() % static_cast( IntHi( spec ) - IntLo( spec ) + 1 ) ); + return static_cast( ( rng() & 1 ) ? -value : value ); + } + } + return 0; +} + +template +static std::vector Specials( const InputSpec& spec ) +{ + if constexpr( std::is_same_v ) + { + return IntSpecials( spec ); + } + else + { + return FloatSpecials( spec ); + } +} + +template +static T RandomValue( const InputSpec& spec, std::mt19937_64& rng ) +{ + if constexpr( std::is_same_v ) + { + return RandomInt( spec, rng ); + } + else + { + return RandomFloat( spec, rng ); + } +} + +InputGenerator::InputGenerator( uint64_t seed, std::string_view testName, const TestData& test ) : + mArgCount( test.argCount ), mLanes( test.lanes ), mSeed( seed ^ HashName( testName ) ), mCombinations( test.argCount ? 1 : 0 ) +{ + for( size_t argIdx = 0; argIdx < mArgCount; argIdx++ ) + { + Arg& arg = mArgs[argIdx]; + arg.type = test.argTypes[argIdx]; + arg.spec = test.inputs[argIdx]; + + WithStoredType( arg.type.element, [&]( auto stored ) { arg.values = MakeValues( arg ); } ); + + arg.specialCount = arg.type.mask ? MaskPatternCount() : std::visit( []( const auto& values ) { return values.specials.size(); }, arg.values ); + mCombinations *= arg.specialCount; + } +} + +size_t InputGenerator::SpecialCount() const +{ + return mCombinations * mLanes; +} + +const TestInputs& InputGenerator::Generate( size_t chunkStart ) +{ + for( size_t argIdx = 0; argIdx < mArgCount; argIdx++ ) + { + uint64_t state = mSeed + chunkStart * kMaxTestArgs + argIdx; + std::mt19937_64 rng( SplitMix64( state ) ); + + std::visit( [&]( auto& values ) + { + Fill( argIdx, values, chunkStart, rng ); + mInputs.values[argIdx] = values.chunk.data(); + }, mArgs[argIdx].values ); + } + return mInputs; +} + +template +InputGenerator::Values InputGenerator::MakeValues( const Arg& arg ) const +{ + Values values; + if( !arg.type.mask ) + { + values.specials = Specials( arg.spec ); + } + values.chunk.resize( kChunkSize ); + return values; +} + +template +void InputGenerator::Fill( size_t argIdx, Values& values, size_t chunkStart, std::mt19937_64& rng ) const +{ + const Arg& arg = mArgs[argIdx]; + + for( size_t i = 0; i < kChunkSize; i++ ) + { + size_t element = chunkStart + i; + bool special = element < SpecialCount(); + + if( arg.type.mask ) + { + int32_t lane = special ? MaskPatternLane( SpecialDigit( element, argIdx ), element % mLanes ) : static_cast( rng() & 1 ); + values.chunk[i] = static_cast( lane ); + } + else + { + values.chunk[i] = special ? values.specials[SpecialDigit( element, argIdx )] : RandomValue( arg.spec, rng ); + } + } +} + +// Value arguments use combination (call + lane), putting every combination in every lane and in lane 0 for scalar arguments +// Mask arguments use combination (call) so a whole mask pattern lands in one call +size_t InputGenerator::SpecialDigit( size_t element, size_t argIdx ) const +{ + size_t call = element / mLanes; + size_t combination = mArgs[argIdx].type.mask ? call : call + element % mLanes; + combination %= mCombinations; + + for( size_t i = 0; i < argIdx; i++ ) + { + combination /= mArgs[i].specialCount; + } + return combination % mArgs[argIdx].specialCount; +} + +// All clear, all set, each single lane set, each single lane clear, alternating, and each half set +size_t InputGenerator::MaskPatternCount() const +{ + return 2 + mLanes * 2 + 4; +} + +int32_t InputGenerator::MaskPatternLane( size_t pattern, size_t lane ) const +{ + if( pattern < 2 ) + { + return static_cast( pattern ); + } + pattern -= 2; + if( pattern < mLanes ) + { + return lane == pattern; + } + pattern -= mLanes; + if( pattern < mLanes ) + { + return lane != pattern; + } + pattern -= mLanes; + + switch( pattern ) + { + case 0: return ( lane & 1 ) == 0; + case 1: return ( lane & 1 ) == 1; + case 2: return lane < mLanes / 2; + default: return lane >= mLanes / 2; + } +} diff --git a/tests/test_inputs.h b/tests/test_inputs.h new file mode 100644 index 0000000..be3eb82 --- /dev/null +++ b/tests/test_inputs.h @@ -0,0 +1,69 @@ +#pragma once +#include "test.h" + +#include +#include +#include +#include +#include + +// Inputs are generated and tested in chunks to keep memory use flat +constexpr size_t kChunkSize = 1 << 14; + +static_assert( kChunkSize % kMaxTestLanes == 0, "Calls must not straddle input chunks" ); + +// Checks a test's input specs can be generated, mistakes here would silently test the wrong domain +void ValidateInputs( std::string_view testName, const TestData& test ); + +// A call's input values as text, one line per argument, for failure messages +std::string DescribeInputs( const TestData& test, const TestInputs& inputs, size_t inIdx ); + +// A lane's argument values as the oracle sees them (TestValue::Lane in test.inl), for known failure conditions +LaneArgs GetLaneArgs( const TestData& test, const TestInputs& inputs, size_t inIdx, size_t lane ); + +// Generates the input values for every argument of a test +// Only uses std::mt19937_64 and arithmetic without FMA contraction (see tests/CMakeLists.txt) so a seed produces the same values on any compiler +class InputGenerator +{ +public: + InputGenerator( uint64_t seed, std::string_view testName, const TestData& test ); + + // Inputs below this index are combinations of special values + size_t SpecialCount() const; + + // The kChunkSize input values per argument from chunkStart, valid until the next call + const TestInputs& Generate( size_t chunkStart ); + +private: + template + struct Values + { + std::vector specials; // Empty for masks, their special values are lane patterns + std::vector chunk; + }; + + struct Arg + { + TestArgType type = {}; + InputSpec spec; + size_t specialCount = 0; + std::variant, Values, Values> values; + }; + + template + Values MakeValues( const Arg& arg ) const; + + template + void Fill( size_t argIdx, Values& values, size_t chunkStart, std::mt19937_64& rng ) const; + + size_t SpecialDigit( size_t element, size_t argIdx ) const; + size_t MaskPatternCount() const; + int32_t MaskPatternLane( size_t pattern, size_t lane ) const; + + size_t mArgCount; + size_t mLanes; + uint64_t mSeed; + size_t mCombinations; + Arg mArgs[kMaxTestArgs]; + TestInputs mInputs = {}; +};