diff --git a/docs/OperatorKernels.md b/docs/OperatorKernels.md index 51dfb1b0c8686..b80c79ba70e3c 100644 --- a/docs/OperatorKernels.md +++ b/docs/OperatorKernels.md @@ -703,6 +703,9 @@ The **OpSet Version** column uses the following notation: |||14|**T** = tensor(double), tensor(float), tensor(float16)
**U** = tensor(double), tensor(float), tensor(float16)| |||[9, 13]|**T** = tensor(double), tensor(float), tensor(float16)| |||[7, 8]|**T** = tensor(double), tensor(float), tensor(float16)| +|BitwiseAnd|*in* A:**T**
*in* B:**T**
*out* C:**T**|18+|**T** = tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|BitwiseNot|*in* X:**T**
*out* Y:**T**|18+|**T** = tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|BitwiseOr|*in* A:**T**
*in* B:**T**
*out* C:**T**|18+|**T** = tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| |Cast|*in* input:**T1**
*out* output:**T2**|28+|**T1** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(float4e2m1), tensor(float8e4m3fn), tensor(float8e5m2), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**T2** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(float4e2m1), tensor(float8e4m3fn), tensor(float8e5m2), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| |||[25, 27]|**T1** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(float4e2m1), tensor(float8e4m3fn), tensor(float8e5m2), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**T2** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(float4e2m1), tensor(float8e4m3fn), tensor(float8e5m2), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| |||[23, 24]|**T1** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(float4e2m1), tensor(float8e4m3fn), tensor(float8e5m2), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
**T2** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(float4e2m1), tensor(float8e4m3fn), tensor(float8e5m2), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| diff --git a/onnxruntime/core/providers/cuda/cuda_execution_provider.cc b/onnxruntime/core/providers/cuda/cuda_execution_provider.cc index 1c1ca9088b7a5..1646e2678b85b 100755 --- a/onnxruntime/core/providers/cuda/cuda_execution_provider.cc +++ b/onnxruntime/core/providers/cuda/cuda_execution_provider.cc @@ -1528,6 +1528,30 @@ class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 17, MLFloat16, LayerNormalization); // Opset 18 +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 18, uint8_t, BitwiseAnd); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 18, uint16_t, BitwiseAnd); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 18, uint32_t, BitwiseAnd); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 18, uint64_t, BitwiseAnd); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 18, int8_t, BitwiseAnd); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 18, int16_t, BitwiseAnd); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 18, int32_t, BitwiseAnd); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 18, int64_t, BitwiseAnd); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 18, uint8_t, BitwiseOr); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 18, uint16_t, BitwiseOr); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 18, uint32_t, BitwiseOr); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 18, uint64_t, BitwiseOr); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 18, int8_t, BitwiseOr); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 18, int16_t, BitwiseOr); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 18, int32_t, BitwiseOr); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 18, int64_t, BitwiseOr); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 18, uint8_t, BitwiseNot); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 18, uint16_t, BitwiseNot); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 18, uint32_t, BitwiseNot); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 18, uint64_t, BitwiseNot); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 18, int8_t, BitwiseNot); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 18, int16_t, BitwiseNot); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 18, int32_t, BitwiseNot); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 18, int64_t, BitwiseNot); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 18, Split); class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 18, 19, float, ReduceMax); @@ -2853,6 +2877,30 @@ static Status RegisterCudaKernels(KernelRegistry& kernel_registry) { BuildKernelCreateInfo, // Opset 18 + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, diff --git a/onnxruntime/core/providers/cuda/math/binary_elementwise_ops.cc b/onnxruntime/core/providers/cuda/math/binary_elementwise_ops.cc index 0340f3b4c7e8b..f22ae8bf238b6 100644 --- a/onnxruntime/core/providers/cuda/math/binary_elementwise_ops.cc +++ b/onnxruntime/core/providers/cuda/math/binary_elementwise_ops.cc @@ -292,6 +292,26 @@ BINARY_OP_REGISTER_VERSIONED_CLASS_HFD(Pow, Pow_7, 7, 11) BINARY_LOGICALOP_TYPED(And, 7, bool) BINARY_LOGICALOP_TYPED(Or, 7, bool) BINARY_LOGICALOP_TYPED(Xor, 7, bool) + +#define BINARY_BITWISE_OP_TYPED(name, class_name, T) \ + BINARY_ELEMENTWISE_REGISTER_KERNEL_TYPED_V(name, class_name, 18, T) \ + BINARY_ELEMENTWISE_COMPUTE(class_name, T) + +#define BINARY_BITWISE_OP(name, class_name) \ + BINARY_BITWISE_OP_TYPED(name, class_name, uint8_t) \ + BINARY_BITWISE_OP_TYPED(name, class_name, uint16_t) \ + BINARY_BITWISE_OP_TYPED(name, class_name, uint32_t) \ + BINARY_BITWISE_OP_TYPED(name, class_name, uint64_t) \ + BINARY_BITWISE_OP_TYPED(name, class_name, int8_t) \ + BINARY_BITWISE_OP_TYPED(name, class_name, int16_t) \ + BINARY_BITWISE_OP_TYPED(name, class_name, int32_t) \ + BINARY_BITWISE_OP_TYPED(name, class_name, int64_t) + +BINARY_BITWISE_OP(BitwiseAnd, And) +BINARY_BITWISE_OP(BitwiseOr, Or) + +#undef BINARY_BITWISE_OP +#undef BINARY_BITWISE_OP_TYPED BINARY_OP_VERSIONED_HFD(PRelu, 7, 8) BINARY_OP_VERSIONED_HFD(PRelu, 9, 15) // Opset-16 adds BFloat16 to allowed types for the PRelu operator diff --git a/onnxruntime/core/providers/cuda/math/binary_elementwise_ops_impl.cu b/onnxruntime/core/providers/cuda/math/binary_elementwise_ops_impl.cu index 2b08614b1e9c4..7b02297ec317f 100644 --- a/onnxruntime/core/providers/cuda/math/binary_elementwise_ops_impl.cu +++ b/onnxruntime/core/providers/cuda/math/binary_elementwise_ops_impl.cu @@ -158,6 +158,18 @@ SPECIALIZED_BINARY_ELEMENTWISE_IMPL_BWUZCSILHFD(Div) SPECIALIZED_BINARY_ELEMENTWISE_IMPL_HFD(Pow_7) SPECIALIZED_BINARY_ELEMENTWISE_IMPL(And, bool) SPECIALIZED_BINARY_ELEMENTWISE_IMPL(Or, bool) +#define SPECIALIZED_BINARY_BITWISE_IMPL(name) \ + SPECIALIZED_BINARY_ELEMENTWISE_IMPL(name, uint8_t) \ + SPECIALIZED_BINARY_ELEMENTWISE_IMPL(name, uint16_t) \ + SPECIALIZED_BINARY_ELEMENTWISE_IMPL_UZIL(name) \ + SPECIALIZED_BINARY_ELEMENTWISE_IMPL(name, int8_t) \ + SPECIALIZED_BINARY_ELEMENTWISE_IMPL(name, int16_t) + +SPECIALIZED_BINARY_BITWISE_IMPL(And) +SPECIALIZED_BINARY_BITWISE_IMPL(Or) + +#undef SPECIALIZED_BINARY_BITWISE_IMPL + SPECIALIZED_BINARY_ELEMENTWISE_IMPL(Xor, bool) SPECIALIZED_BINARY_ELEMENTWISE_IMPL_HFD(PRelu) SPECIALIZED_BINARY_ELEMENTWISE_IMPL_UZILHFD(Max) diff --git a/onnxruntime/core/providers/cuda/math/unary_elementwise_ops.cc b/onnxruntime/core/providers/cuda/math/unary_elementwise_ops.cc index ef31e2ecafae5..d2117867f7b37 100644 --- a/onnxruntime/core/providers/cuda/math/unary_elementwise_ops.cc +++ b/onnxruntime/core/providers/cuda/math/unary_elementwise_ops.cc @@ -249,6 +249,14 @@ UNARY_OP_HFDX(Erf, 13) UNARY_OP_BWUZCSILHFDX(Sign, 13) UNARY_LOGICALOP_NOT_TYPED(1, bool) +UNARY_OP_TYPED(BitwiseNot, 18, uint8_t) +UNARY_OP_TYPED(BitwiseNot, 18, uint16_t) +UNARY_OP_TYPED(BitwiseNot, 18, uint32_t) +UNARY_OP_TYPED(BitwiseNot, 18, uint64_t) +UNARY_OP_TYPED(BitwiseNot, 18, int8_t) +UNARY_OP_TYPED(BitwiseNot, 18, int16_t) +UNARY_OP_TYPED(BitwiseNot, 18, int32_t) +UNARY_OP_TYPED(BitwiseNot, 18, int64_t) UNARY_OP_VERSIONED_HFD(Round, 11, 21) UNARY_OP_HFD(Round, 22) UNARY_OP_VERSIONED_HFD(Cos, 7, 21) diff --git a/onnxruntime/core/providers/cuda/math/unary_elementwise_ops.h b/onnxruntime/core/providers/cuda/math/unary_elementwise_ops.h index 95d68b5e1d534..016e73a9310d4 100644 --- a/onnxruntime/core/providers/cuda/math/unary_elementwise_ops.h +++ b/onnxruntime/core/providers/cuda/math/unary_elementwise_ops.h @@ -92,6 +92,13 @@ class Not final : public UnaryElementwise { Status ComputeInternal(OpKernelContext* context) const override; }; +template +class BitwiseNot final : public UnaryElementwise { + public: + BitwiseNot(const OpKernelInfo& info) : UnaryElementwise(info) {} + Status ComputeInternal(OpKernelContext* context) const override; +}; + template class Round final : public UnaryElementwise { public: diff --git a/onnxruntime/core/providers/cuda/math/unary_elementwise_ops_impl.cu b/onnxruntime/core/providers/cuda/math/unary_elementwise_ops_impl.cu index b84cc17176ef2..b6e370af47376 100644 --- a/onnxruntime/core/providers/cuda/math/unary_elementwise_ops_impl.cu +++ b/onnxruntime/core/providers/cuda/math/unary_elementwise_ops_impl.cu @@ -92,6 +92,14 @@ SPECIALIZED_UNARY_ELEMENTWISE_IMPL_HFD(Round) SPECIALIZED_UNARY_ELEMENTWISE_IMPL_HFDX(Sin) SPECIALIZED_UNARY_ELEMENTWISE_IMPL_HFDX(Cos) SPECIALIZED_UNARY_ELEMENTWISE_IMPL(Not, bool) +SPECIALIZED_UNARY_ELEMENTWISE_IMPL(BitwiseNot, uint8_t) +SPECIALIZED_UNARY_ELEMENTWISE_IMPL(BitwiseNot, uint16_t) +SPECIALIZED_UNARY_ELEMENTWISE_IMPL(BitwiseNot, uint32_t) +SPECIALIZED_UNARY_ELEMENTWISE_IMPL(BitwiseNot, uint64_t) +SPECIALIZED_UNARY_ELEMENTWISE_IMPL(BitwiseNot, int8_t) +SPECIALIZED_UNARY_ELEMENTWISE_IMPL(BitwiseNot, int16_t) +SPECIALIZED_UNARY_ELEMENTWISE_IMPL(BitwiseNot, int32_t) +SPECIALIZED_UNARY_ELEMENTWISE_IMPL(BitwiseNot, int64_t) SPECIALIZED_UNARY_ELEMENTWISE_IMPL_BWUZCSILHFD(Sign) // When casting, half needs to be converted via float type from most other types diff --git a/onnxruntime/core/providers/cuda/math/unary_elementwise_ops_impl.h b/onnxruntime/core/providers/cuda/math/unary_elementwise_ops_impl.h index 20ef15c549506..b12cae23a56e0 100644 --- a/onnxruntime/core/providers/cuda/math/unary_elementwise_ops_impl.h +++ b/onnxruntime/core/providers/cuda/math/unary_elementwise_ops_impl.h @@ -29,6 +29,7 @@ namespace cuda { UNARY_OP_NAME_EXPR(Log, _Log(a)) \ UNARY_OP_NAME_EXPR(Erf, _Erf(a)) \ UNARY_OP_NAME_EXPR(Not, !a) \ + UNARY_OP_NAME_EXPR(BitwiseNot, ~a) \ UNARY_OP_NAME_EXPR(Round, _Round(a)) \ UNARY_OP_NAME_EXPR(Sin, _Sin(a)) \ UNARY_OP_NAME_EXPR(Cos, _Cos(a)) \ diff --git a/onnxruntime/test/providers/cuda/bitwise_ops_test.cc b/onnxruntime/test/providers/cuda/bitwise_ops_test.cc new file mode 100644 index 0000000000000..294ea40322d70 --- /dev/null +++ b/onnxruntime/test/providers/cuda/bitwise_ops_test.cc @@ -0,0 +1,136 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include +#include +#include +#include +#include + +#include "gtest/gtest.h" +#include "core/session/onnxruntime_session_options_config_keys.h" +#include "test/providers/provider_test_utils.h" +#include "test/util/include/default_providers.h" + +namespace onnxruntime { +namespace test { +namespace { + +template +void RunCudaBitwise(const char* op, int opset, + const std::vector& lhs_shape, const std::vector& lhs, + const std::vector& rhs_shape, const std::vector& rhs, + const std::vector& output_shape, const std::vector& expected) { + SCOPED_TRACE(std::string(op) + " opset=" + std::to_string(opset)); + OpTester test(op, opset); + const bool unary = std::string(op) == "BitwiseNot"; + test.AddInput(unary ? "X" : "A", lhs_shape, lhs); + if (!unary) { + test.AddInput("B", rhs_shape, rhs); + } + test.AddOutput(unary ? "Y" : "C", output_shape, expected); + SessionOptions options; + ASSERT_STATUS_OK(options.config_options.AddConfigEntry(kOrtSessionOptionsDisableCPUEPFallback, "1")); + test.Config(options).ConfigEp(DefaultCudaExecutionProvider()).RunWithConfig(); +} + +template +class CudaBitwiseOpTest : public ::testing::Test { + protected: + void SetUp() override { + if (!DefaultCudaExecutionProvider()) { + GTEST_SKIP() << "CUDA execution provider is unavailable"; + } + } +}; + +using IntegerTypes = ::testing::Types; +TYPED_TEST_SUITE(CudaBitwiseOpTest, IntegerTypes); + +TYPED_TEST(CudaBitwiseOpTest, IntegerExtremaBroadcastAndTails) { + using T = TypeParam; + const std::vector values{ + T{0}, T{1}, std::numeric_limits::max(), std::numeric_limits::lowest(), + static_cast(-1), static_cast(UINT64_C(0x5555555555555555) & std::numeric_limits::max()), + static_cast(UINT64_C(0x2aaaaaaaaaaaaaaa) & std::numeric_limits::max())}; + + for (int opset : {18, 24}) { + for (const char* op : {"BitwiseAnd", "BitwiseOr", "BitwiseNot"}) { + const auto apply = [op](T a, T b) -> T { + if (std::string(op) == "BitwiseAnd") { + return static_cast(a & b); + } + if (std::string(op) == "BitwiseOr") { + return static_cast(a | b); + } + return static_cast(~a); + }; + + for (int64_t size : {0, 1, 7, 255, 256, 257, 1025}) { + SCOPED_TRACE("size=" + std::to_string(size)); + std::vector lhs, rhs, expected; + lhs.reserve(static_cast(size)); + rhs.reserve(static_cast(size)); + expected.reserve(static_cast(size)); + for (int64_t i = 0; i < size; ++i) { + lhs.push_back(values[static_cast(i) % values.size()]); + rhs.push_back(values[(static_cast(i) / values.size() + 2) % values.size()]); + expected.push_back(apply(lhs.back(), rhs.back())); + } + RunCudaBitwise(op, opset, {size}, lhs, {size}, rhs, {size}, expected); + } + + const T scalar = std::numeric_limits::max(); + std::vector expected; + expected.reserve(values.size()); + for (T value : values) { + expected.push_back(apply(value, scalar)); + } + RunCudaBitwise(op, opset, {7}, values, {}, {scalar}, {7}, expected); + RunCudaBitwise(op, opset, {}, {scalar}, {}, {scalar}, {}, {apply(scalar, scalar)}); + + if (std::string(op) != "BitwiseNot") { + RunCudaBitwise(op, opset, {}, {scalar}, {7}, values, {7}, expected); + RunCudaBitwise(op, opset, {2, 0, 3}, {}, {1, 3}, {T{0}, T{1}, scalar}, {2, 0, 3}, {}); + const std::vector lhs(values.begin(), values.begin() + 6); + const std::vector rhs{values[2], values[3], values[4], values[5]}; + expected.clear(); + for (size_t batch = 0; batch < 2; ++batch) { + for (T b : rhs) { + for (size_t col = 0; col < 3; ++col) { + expected.push_back(apply(lhs[batch * 3 + col], b)); + } + } + } + RunCudaBitwise(op, opset, {2, 1, 3}, lhs, {1, 4, 1}, rhs, {2, 4, 3}, expected); + RunCudaBitwise(op, opset, {1, 4, 1}, rhs, {2, 1, 3}, lhs, {2, 4, 3}, expected); + } + } + } +} + +TEST(CudaBitwiseLogicalTest, BooleanOpsUnchanged) { + if (!DefaultCudaExecutionProvider()) { + GTEST_SKIP() << "CUDA execution provider is unavailable"; + } + const auto run = [](const char* op, int opset, std::initializer_list expected) { + SCOPED_TRACE(op); + const bool unary = std::string(op) == "Not"; + OpTester test(op, opset); + test.AddInput(unary ? "X" : "A", {4}, {false, false, true, true}); + if (!unary) { + test.AddInput("B", {4}, {false, true, false, true}); + } + test.AddOutput(unary ? "Y" : "C", {4}, expected); + SessionOptions options; + ASSERT_STATUS_OK(options.config_options.AddConfigEntry(kOrtSessionOptionsDisableCPUEPFallback, "1")); + test.Config(options).ConfigEp(DefaultCudaExecutionProvider()).RunWithConfig(); + }; + run("And", 7, {false, false, false, true}); + run("Or", 7, {false, true, true, true}); + run("Not", 1, {true, true, false, false}); +} + +} // namespace +} // namespace test +} // namespace onnxruntime