Fix library bugs found by the reworked tests

dispatch/cmake/ClassSIMD.cmake

@@ -114,6 +114,15 @@ function(fastsimd_create_dispatch_library simd_library_name)
     if(fastsimd_create_dispatch_library_RELAXED)
         target_compile_definitions(${simd_library_name} PUBLIC FASTSIMD_IS_RELAXED=1)
         set(relaxed_log_msg " (RELAXED)")
+    else()
+        # Non relaxed results must match across feature sets, stop the compiler fusing mul + add into FMA
+        # -mno-fma isn't enough, AVX512F has its own FMA instructions
+        # MSVC is also set for clang-cl, which ignores a bare -ffp-contract=off
+        if(MSVC AND CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
+            target_compile_options(${simd_library_name} PRIVATE /clang:-ffp-contract=off)
+        elseif(NOT MSVC)
+            target_compile_options(${simd_library_name} PRIVATE -ffp-contract=off)
+        endif()
     endif()
 
     set(feature_set_list "")

include/FastSIMD/ToolSet/ARM/128/f32x4.h

@@ -186,13 +186,29 @@ namespace FS
     template<FastSIMD::FeatureSet SIMD, typename = EnableIfNative<f32<4, SIMD>>>
     FS_FORCEINLINE f32<4, SIMD> Min( const f32<4, SIMD>& a, const f32<4, SIMD>& b )
     {
-        return vminq_f32( a.native, b.native );
+        if constexpr( FastSIMD::IsRelaxed() )
+        {
+            return vminq_f32( a.native, b.native );
+        }
+        else
+        {
+            // Same as std::min for +-0 and NaN, which vminq is not
+            return Select( b < a, b, a );
+        }
     }
 
     template<FastSIMD::FeatureSet SIMD, typename = EnableIfNative<f32<4, SIMD>>>
     FS_FORCEINLINE f32<4, SIMD> Max( const f32<4, SIMD>& a, const f32<4, SIMD>& b )
     {
-        return vmaxq_f32( a.native, b.native );
+        if constexpr( FastSIMD::IsRelaxed() )
+        {
+            return vmaxq_f32( a.native, b.native );
+        }
+        else
+        {
+            // Same as std::max for +-0 and NaN, which vmaxq is not
+            return Select( a < b, b, a );
+        }
     }
 
     template<FastSIMD::FeatureSet SIMD, typename = EnableIfNative<f32<4, SIMD>>>

include/FastSIMD/ToolSet/ARM/NEON.h

@@ -19,24 +19,50 @@ namespace FS
         return vcvtq_f32_s32( a.native );
     }
 
-    template<typename U, typename T, FastSIMD::FeatureSet SIMD, typename = EnableIfNative<Register<T, 4, SIMD>>>
-    FS_FORCEINLINE Register<U, 4, SIMD> Cast( const Register<T, 4, SIMD>& a, TypeDummy<U> )
+    namespace impl
     {
-        if constexpr( 
-            std::is_same_v<typename Register<T, 4, SIMD>::NativeType, float32x4_t> &&
-            std::is_same_v<typename Register<U, 4, SIMD>::NativeType, int32x4_t> )
+        // Native casts go through uint32x4_t and pick the reinterpret by element type, not native type
+        // With the MSVC ARM64 ABI every NEON vector type is the same __n128, so native types can't be told apart
+        template<typename T, typename V>
+        FS_FORCEINLINE uint32x4_t NeonCastToU32( V a )
         {
-            return vreinterpretq_s32_f32( a.GetNative() );
+            if constexpr( std::is_same_v<T, float> )
+            {
+                return vreinterpretq_u32_f32( a );
+            }
+            else if constexpr( std::is_same_v<T, std::int32_t> )
+            {
+                return vreinterpretq_u32_s32( a );
+            }
+            else
+            {
+                static_assert( std::is_same_v<T, Mask<32, true>> || std::is_same_v<T, Mask<32, false>>, "FastSIMD: FS::Cast not supported with provided types" );
+                return a;
+            }
         }
-        else if constexpr( 
-            std::is_same_v<typename Register<T, 4, SIMD>::NativeType, int32x4_t> &&
-            std::is_same_v<typename Register<U, 4, SIMD>::NativeType, float32x4_t> )
-        {
-            return vreinterpretq_f32_s32( a.GetNative() );
-        }
-        else
+
+        template<typename U>
+        FS_FORCEINLINE auto NeonCastFromU32( uint32x4_t a )
         {
-            return a.GetNative();
+            if constexpr( std::is_same_v<U, float> )
+            {
+                return vreinterpretq_f32_u32( a );
+            }
+            else if constexpr( std::is_same_v<U, std::int32_t> )
+            {
+                return vreinterpretq_s32_u32( a );
+            }
+            else
+            {
+                static_assert( std::is_same_v<U, Mask<32, true>> || std::is_same_v<U, Mask<32, false>>, "FastSIMD: FS::Cast not supported with provided types" );
+                return a;
+            }
         }
     }
+
+    template<typename U, typename T, FastSIMD::FeatureSet SIMD, typename = EnableIfNative<Register<T, 4, SIMD>>>
+    FS_FORCEINLINE Register<U, 4, SIMD> Cast( const Register<T, 4, SIMD>& a, TypeDummy<U> )
+    {
+        return impl::NeonCastFromU32<U>( impl::NeonCastToU32<T>( a.GetNative() ) );
+    }
 }

include/FastSIMD/ToolSet/Generic/Functions.h

@@ -634,7 +634,7 @@ namespace FS
 
             RegisterF xx = value * value;
             y *= xx;
-            y *= e * RegisterF( -2.12194440e-4f );
+            y += e * RegisterF( -2.12194440e-4f );
             y -= xx * RegisterF( 0.5f );
 
             value += y;

include/FastSIMD/ToolSet/Generic/Register.h

@@ -175,70 +175,72 @@ namespace FS
         DoubledType v0, v1;
     };
 
+    // Binary operators copy lhs into a temporary rather than taking it by value
+    // MSVC x86 doesn't copy over-aligned by value arguments, so modifying a by value lhs wrote into the caller's register
     template<typename T, std::size_t N, FastSIMD::FeatureSet SIMD>
-    FS_FORCEINLINE static Register<T, N, SIMD> operator +( Register<T, N, SIMD> lhs, const Register<T, N, SIMD>& rhs )
+    FS_FORCEINLINE static Register<T, N, SIMD> operator +( const Register<T, N, SIMD>& lhs, const Register<T, N, SIMD>& rhs )
     {
-        return lhs += rhs;
+        return Register<T, N, SIMD>( lhs ) += rhs;
     }
 
     template<typename T, std::size_t N, FastSIMD::FeatureSet SIMD>
-    FS_FORCEINLINE static Register<T, N, SIMD> operator -( Register<T, N, SIMD> lhs, const Register<T, N, SIMD>& rhs )
+    FS_FORCEINLINE static Register<T, N, SIMD> operator -( const Register<T, N, SIMD>& lhs, const Register<T, N, SIMD>& rhs )
     {
-        return lhs -= rhs;
+        return Register<T, N, SIMD>( lhs ) -= rhs;
     }
 
     template<typename T, std::size_t N, FastSIMD::FeatureSet SIMD>
-    FS_FORCEINLINE static Register<T, N, SIMD> operator *( Register<T, N, SIMD> lhs, const Register<T, N, SIMD>& rhs )
+    FS_FORCEINLINE static Register<T, N, SIMD> operator *( const Register<T, N, SIMD>& lhs, const Register<T, N, SIMD>& rhs )
     {
-        return lhs *= rhs;
+        return Register<T, N, SIMD>( lhs ) *= rhs;
     }
 
     template<typename T, std::size_t N, FastSIMD::FeatureSet SIMD>
-    FS_FORCEINLINE static Register<T, N, SIMD> operator /( Register<T, N, SIMD> lhs, const Register<T, N, SIMD>& rhs )
+    FS_FORCEINLINE static Register<T, N, SIMD> operator /( const Register<T, N, SIMD>& lhs, const Register<T, N, SIMD>& rhs )
     {
-        return lhs /= rhs;
+        return Register<T, N, SIMD>( lhs ) /= rhs;
     }
 
     template<typename T, std::size_t N, FastSIMD::FeatureSet SIMD>
-    FS_FORCEINLINE static Register<T, N, SIMD> operator &( Register<T, N, SIMD> lhs, const Register<T, N, SIMD>& rhs )
+    FS_FORCEINLINE static Register<T, N, SIMD> operator &( const Register<T, N, SIMD>& lhs, const Register<T, N, SIMD>& rhs )
     {
-        return lhs &= rhs;
+        return Register<T, N, SIMD>( lhs ) &= rhs;
     }
 
     template<typename T, std::size_t N, FastSIMD::FeatureSet SIMD>
-    FS_FORCEINLINE static Register<T, N, SIMD> operator |( Register<T, N, SIMD> lhs, const Register<T, N, SIMD>& rhs )
+    FS_FORCEINLINE static Register<T, N, SIMD> operator |( const Register<T, N, SIMD>& lhs, const Register<T, N, SIMD>& rhs )
     {
-        return lhs |= rhs;
+        return Register<T, N, SIMD>( lhs ) |= rhs;
     }
 
     template<typename T, std::size_t N, FastSIMD::FeatureSet SIMD>
-    FS_FORCEINLINE static Register<T, N, SIMD> operator ^( Register<T, N, SIMD> lhs, const Register<T, N, SIMD>& rhs )
+    FS_FORCEINLINE static Register<T, N, SIMD> operator ^( const Register<T, N, SIMD>& lhs, const Register<T, N, SIMD>& rhs )
     {
-        return lhs ^= rhs;
+        return Register<T, N, SIMD>( lhs ) ^= rhs;
     }
 
     template<typename T, std::size_t N, FastSIMD::FeatureSet SIMD>
-    FS_FORCEINLINE static Register<T, N, SIMD> operator <<( Register<T, N, SIMD> lhs, int rhs )
+    FS_FORCEINLINE static Register<T, N, SIMD> operator <<( const Register<T, N, SIMD>& lhs, int rhs )
     {
-        return lhs <<= rhs;
+        return Register<T, N, SIMD>( lhs ) <<= rhs;
     }
 
     template<typename T, std::size_t N, FastSIMD::FeatureSet SIMD>
-    FS_FORCEINLINE static Register<T, N, SIMD> operator >>( Register<T, N, SIMD> lhs, int rhs )
+    FS_FORCEINLINE static Register<T, N, SIMD> operator >>( const Register<T, N, SIMD>& lhs, int rhs )
     {
-        return lhs >>= rhs;
+        return Register<T, N, SIMD>( lhs ) >>= rhs;
     }
 
     template<typename T, typename U, std::size_t N, FastSIMD::FeatureSet SIMD>
-    FS_FORCEINLINE static Register<T, N, SIMD> operator<<( Register<T, N, SIMD> lhs, Register<U, N, SIMD> rhs )
+    FS_FORCEINLINE static Register<T, N, SIMD> operator<<( const Register<T, N, SIMD>& lhs, const Register<U, N, SIMD>& rhs )
     {
-        return lhs <<= rhs;
+        return Register<T, N, SIMD>( lhs ) <<= rhs;
     }
 
     template<typename T, typename U, std::size_t N, FastSIMD::FeatureSet SIMD>
-    FS_FORCEINLINE static Register<T, N, SIMD> operator>>( Register<T, N, SIMD> lhs, Register<U, N, SIMD> rhs )
+    FS_FORCEINLINE static Register<T, N, SIMD> operator>>( const Register<T, N, SIMD>& lhs, const Register<U, N, SIMD>& rhs )
     {
-        return lhs >>= rhs;
+        return Register<T, N, SIMD>( lhs ) >>= rhs;
     }
 
     template<typename T, std::size_t N = 0, FastSIMD::FeatureSet SIMD = FastSIMD::FeatureSetDefault()>

include/FastSIMD/ToolSet/WASM/128/f32x4.h

@@ -168,7 +168,8 @@ namespace FS
         }
         else
         {
-            return wasm_f32x4_min( a.native, b.native );
+            // pmin is b < a ? b : a, matching std::min for +-0 and NaN, which min does not
+            return wasm_f32x4_pmin( a.native, b.native );
         }
     }
 
@@ -181,7 +182,8 @@ namespace FS
         }
         else
         {
-            return wasm_f32x4_max( a.native, b.native );
+            // pmax is a < b ? b : a, matching std::max for +-0 and NaN, which max does not
+            return wasm_f32x4_pmax( a.native, b.native );
         }
     }
 

include/FastSIMD/ToolSet/x86/128/f32x4.h

@@ -204,13 +204,15 @@ namespace FS
     template<FastSIMD::FeatureSet SIMD, typename = EnableIfNative<f32<4, SIMD>>>
     FS_FORCEINLINE f32<4, SIMD> Min( const f32<4, SIMD>& a, const f32<4, SIMD>& b )
     {
-        return _mm_min_ps( a.native, b.native );
+        // minps returns its 2nd operand for equal (+-0) or NaN inputs, swapped to match std::min which returns a
+        return _mm_min_ps( b.native, a.native );
     }
 
     template<FastSIMD::FeatureSet SIMD, typename = EnableIfNative<f32<4, SIMD>>>
     FS_FORCEINLINE f32<4, SIMD> Max( const f32<4, SIMD>& a, const f32<4, SIMD>& b )
     {
-        return _mm_max_ps( a.native, b.native );
+        // maxps returns its 2nd operand for equal (+-0) or NaN inputs, swapped to match std::max which returns a
+        return _mm_max_ps( b.native, a.native );
     }
 
     template<FastSIMD::FeatureSet SIMD, typename = EnableIfNative<f32<4, SIMD>>>

include/FastSIMD/ToolSet/x86/256/f32x8.h

@@ -86,7 +86,7 @@ namespace FS
 
         FS_FORCEINLINE MaskType operator !=( const Register& rhs ) const
         {
-            return _mm256_cmp_ps( native, rhs.native, _CMP_NEQ_OQ );
+            return _mm256_cmp_ps( native, rhs.native, _CMP_NEQ_UQ );
         }
 
         FS_FORCEINLINE MaskType operator >=( const Register& rhs ) const
@@ -165,13 +165,15 @@ namespace FS
     template<FastSIMD::FeatureSet SIMD, typename = EnableIfNative<f32<8, SIMD>>>
     FS_FORCEINLINE f32<8, SIMD> Min( const f32<8, SIMD>& a, const f32<8, SIMD>& b )
     {
-        return _mm256_min_ps( a.native, b.native );
+        // minps returns its 2nd operand for equal (+-0) or NaN inputs, swapped to match std::min which returns a
+        return _mm256_min_ps( b.native, a.native );
     }
 
     template<FastSIMD::FeatureSet SIMD, typename = EnableIfNative<f32<8, SIMD>>>
     FS_FORCEINLINE f32<8, SIMD> Max( const f32<8, SIMD>& a, const f32<8, SIMD>& b )
     {
-        return _mm256_max_ps( a.native, b.native );
+        // maxps returns its 2nd operand for equal (+-0) or NaN inputs, swapped to match std::max which returns a
+        return _mm256_max_ps( b.native, a.native );
     }
 
     template<FastSIMD::FeatureSet SIMD, typename = EnableIfNative<f32<8, SIMD>>>

include/FastSIMD/ToolSet/x86/512/f32x16.h

@@ -86,7 +86,7 @@ namespace FS
 
         FS_FORCEINLINE MaskType operator !=( const Register& rhs ) const
         {
-            return _mm512_cmp_ps_mask( native, rhs.native, _CMP_NEQ_OQ );
+            return _mm512_cmp_ps_mask( native, rhs.native, _CMP_NEQ_UQ );
         }
 
         FS_FORCEINLINE MaskType operator >=( const Register& rhs ) const
@@ -164,13 +164,15 @@ namespace FS
     template<FastSIMD::FeatureSet SIMD, typename = EnableIfNative<f32<16, SIMD>>>
     FS_FORCEINLINE f32<16, SIMD> Min( const f32<16, SIMD>& a, const f32<16, SIMD>& b )
     {
-        return _mm512_min_ps( a.native, b.native );
+        // minps returns its 2nd operand for equal (+-0) or NaN inputs, swapped to match std::min which returns a
+        return _mm512_min_ps( b.native, a.native );
     }
 
     template<FastSIMD::FeatureSet SIMD, typename = EnableIfNative<f32<16, SIMD>>>
     FS_FORCEINLINE f32<16, SIMD> Max( const f32<16, SIMD>& a, const f32<16, SIMD>& b )
     {
-        return _mm512_max_ps( a.native, b.native );
+        // maxps returns its 2nd operand for equal (+-0) or NaN inputs, swapped to match std::max which returns a
+        return _mm512_max_ps( b.native, a.native );
     }
 
     template<FastSIMD::FeatureSet SIMD, typename = EnableIfNative<f32<16, SIMD>>>