From 1f6f58ec99f1a74d9a2937b7ca4fc2846e74baa1 Mon Sep 17 00:00:00 2001 From: Akshay Sonawane Date: Mon, 19 Oct 2026 19:23:42 +0000 Subject: [PATCH 01/12] Optimize quantized MoE and block-scaled FP8 CUDA projections --- cmake/onnxruntime_unittests.cmake | 2 + docs/ContribOperators.md | 1 - .../contrib_ops/cuda/cuda_contrib_kernels.cc | 6 + .../cuda/llm/moe_gemm/moe_gemv_fp4.cu | 111 ++++++-- .../cuda/llm/moe_gemm/moe_gemv_fp4.h | 9 + .../cuda/llm/moe_gemm/moe_kernels.cu | 13 +- .../cuda/math/matmul_block_scaled_fp8.cc | 94 +++++-- .../cuda/math/matmul_block_scaled_fp8.cu | 121 +++++++-- .../cuda/math/matmul_block_scaled_fp8.h | 17 +- .../contrib_ops/cuda/moe/moe_quantization.cc | 15 +- .../contrib_ops/cuda/moe/qmoe_kernels.cu | 36 +++ .../contrib_ops/cuda/moe/qmoe_kernels.h | 4 + .../quantize_dequantize_linear.cc | 3 +- .../core/graph/contrib_ops/contrib_defs.cc | 39 ++- .../graph/contrib_ops/quantization_defs.cc | 61 +++-- .../cuda_kernels/qmoe_fp8_compaction_test.cc | 242 ++++++++++++++++++ .../contrib_ops/dequantize_fp8_bf16_test.cc | 34 +++ .../matmul_block_scaled_fp8_test.cc | 116 +++++++++ onnxruntime/test/contrib_ops/moe_test.cc | 21 +- 19 files changed, 846 insertions(+), 99 deletions(-) create mode 100644 onnxruntime/test/contrib_ops/dequantize_fp8_bf16_test.cc diff --git a/cmake/onnxruntime_unittests.cmake b/cmake/onnxruntime_unittests.cmake index 4e630b5bb45d2..b76a9dc55e417 100644 --- a/cmake/onnxruntime_unittests.cmake +++ b/cmake/onnxruntime_unittests.cmake @@ -1163,6 +1163,8 @@ if (onnxruntime_ENABLE_CUDA_EP_INTERNAL_TESTS AND onnxruntime_BUILD_CUDA_EP_AS_P if (onnxruntime_USE_FP4_QMOE) list(APPEND onnxruntime_providers_cuda_plugin_ut_impl_src "${ONNXRUNTIME_ROOT}/contrib_ops/cuda/llm/moe_gemm/moe_gemv_fp4.cu") + set_property(SOURCE "${TEST_SRC_DIR}/contrib_ops/cuda_kernels/qmoe_fp8_compaction_test.cc" + APPEND PROPERTY COMPILE_DEFINITIONS ORT_TEST_NVFP4_GEMV) endif() endif() diff --git a/docs/ContribOperators.md b/docs/ContribOperators.md index 2703de7cc1fae..3510b52792635 100644 --- a/docs/ContribOperators.md +++ b/docs/ContribOperators.md @@ -8771,4 +8771,3 @@ No versioning maintained for experimental ops.
T : tensor(float)
Constrain input and output types to float32 tensors.
- diff --git a/onnxruntime/contrib_ops/cuda/cuda_contrib_kernels.cc b/onnxruntime/contrib_ops/cuda/cuda_contrib_kernels.cc index 546b5b12e178f..b670435b4085f 100644 --- a/onnxruntime/contrib_ops/cuda/cuda_contrib_kernels.cc +++ b/onnxruntime/contrib_ops/cuda/cuda_contrib_kernels.cc @@ -170,6 +170,9 @@ class CUDA_MS_OP_TYPED_CLASS_NAME(1, BFloat16, LinearAttention); class CUDA_MS_OP_TYPED_CLASS_NAME(1, float, GatedDeltaNet); class CUDA_MS_OP_TYPED_CLASS_NAME(1, MLFloat16, GatedDeltaNet); class CUDA_MS_OP_TYPED_CLASS_NAME(1, BFloat16, GatedDeltaNet); +#if !defined(DISABLE_FLOAT8_TYPES) +class CUDA_MS_OP_TYPED_CLASS_NAME(1, Float8E4M3FN_BFloat16, DequantizeLinear); +#endif class CUDA_MS_OP_TYPED_CLASS_NAME(1, float, LinearAttentionGate); class CUDA_MS_OP_TYPED_CLASS_NAME(1, MLFloat16, LinearAttentionGate); class CUDA_MS_OP_TYPED_CLASS_NAME(1, BFloat16, LinearAttentionGate); @@ -534,6 +537,9 @@ Status RegisterCudaContribKernels(KernelRegistry& kernel_registry) { BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, +#if !defined(DISABLE_FLOAT8_TYPES) + BuildKernelCreateInfo, +#endif BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, diff --git a/onnxruntime/contrib_ops/cuda/llm/moe_gemm/moe_gemv_fp4.cu b/onnxruntime/contrib_ops/cuda/llm/moe_gemm/moe_gemv_fp4.cu index 6951bf9a5b458..dbe438cc16a7c 100644 --- a/onnxruntime/contrib_ops/cuda/llm/moe_gemm/moe_gemv_fp4.cu +++ b/onnxruntime/contrib_ops/cuda/llm/moe_gemm/moe_gemv_fp4.cu @@ -4,6 +4,7 @@ #include "contrib_ops/cuda/llm/moe_gemm/moe_gemv_fp4.h" #include +#include #include // Shared INT/FP4 profiled-shape thresholds. @@ -141,7 +142,7 @@ using Fp4KernelDetailsInterleaved = // (interleave_without_bias), i.e. exactly the layout the SM80 grouped GEMM consumes. Selecting // this variant lets the decode GEMV read the *same* pre-packed buffer the SM80 prefill uses, so // the MXFP4 expert weights are stored once instead of twice (~9 GiB saved for gpt-oss-20b). -// Each word is un-permuted with a few register ops before the shared packed decode. +// The un-permutation is a compile-time index remap, so it is free at runtime. template using Fp4KernelDetailsSm80Pair = fiv::KernelDetails::Type, fiv::Fp4DetailsW, fiv::ColumnMajorInterleaved, true, @@ -360,24 +361,33 @@ __global__ void MoeGemvFp4RawNPackedKernel( #endif } -template +template __global__ void MoeGemvFp4RawKPackedKernel( const T* act, const uint8_t* weight, const uint8_t* block_scales, const float* global_scales, const T* bias, T* out, const int64_t* expert_first_token_offset, const int* permuted_row_to_expert, int num_experts, int64_t weight_expert_stride, int64_t scale_expert_stride, int output_columns, int reduction_size, cutlass_kernels::ActivationParams activation_params, - const int* permuted_row_to_source_row, int num_rows) { + const int* permuted_row_to_source_row, int num_rows, + const int* unpermuted_row_to_permuted_row = nullptr, const float* routing_scales = nullptr, + int experts_per_token = 0) { #if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 800) constexpr int kKLanes = KLanes; constexpr int kBlockSize = 16; - constexpr int kColsPerBlock = 128 / kKLanes; + constexpr int kColsPerBlock = FusedFinalize ? 8 : 128 / kKLanes; + constexpr int kReductionUnroll = FusedSwiGlu ? 2 : 1; static_assert(kKLanes == 4 || kKLanes == 8); constexpr unsigned kFullMask = 0xffffffffu; using PackedT = std::conditional_t, half2, __nv_bfloat162>; - const int row = static_cast(blockIdx.x); - int expert = permuted_row_to_expert != nullptr ? permuted_row_to_expert[row] : 0; + const int row = FusedFinalize + ? unpermuted_row_to_permuted_row[threadIdx.y * num_rows + blockIdx.x] + : static_cast(blockIdx.x); + int expert = row >= 0 && permuted_row_to_expert != nullptr ? permuted_row_to_expert[row] : -1; + if constexpr (!FusedFinalize) { + if (permuted_row_to_expert == nullptr) expert = 0; + } #pragma unroll 1 for (int expert_index = 0; expert_index < num_experts && permuted_row_to_expert == nullptr; ++expert_index) { if (row >= static_cast(expert_first_token_offset[expert_index + 1])) { @@ -386,8 +396,10 @@ __global__ void MoeGemvFp4RawKPackedKernel( } break; } - if (expert < 0 || expert >= num_experts) { - return; + const bool valid_expert = expert >= 0 && expert < num_experts; + if (!valid_expert) { + if constexpr (!FusedFinalize) return; + expert = 0; } const int column = static_cast(blockIdx.y) * kColsPerBlock + threadIdx.x / kKLanes; @@ -403,7 +415,9 @@ __global__ void MoeGemvFp4RawKPackedKernel( } } const int k_lane = threadIdx.x % kKLanes; - const int source_row = permuted_row_to_source_row ? permuted_row_to_source_row[row] % num_rows : row; + const int source_row = valid_expert + ? (permuted_row_to_source_row ? permuted_row_to_source_row[row] % num_rows : row) + : 0; const T* row_act = act + static_cast(source_row) * reduction_size; const int second_source_row = second_row ? (permuted_row_to_source_row ? permuted_row_to_source_row[row + 1] % num_rows : row + 1) @@ -415,8 +429,9 @@ __global__ void MoeGemvFp4RawKPackedKernel( const T* expert_bias = EnableBias ? bias + static_cast(expert) * output_columns : nullptr; float accumulators[ReuseExpertRows ? 2 : 1][2] = {}; +#pragma unroll(kReductionUnroll) for (int reduction_column = k_lane * kBlockSize; - column < output_columns && reduction_column < reduction_size; + valid_expert && column < output_columns && reduction_column < reduction_size; reduction_column += kKLanes * kBlockSize) { uint2 packed = *reinterpret_cast( expert_weight + static_cast(column) * (reduction_size / 2) + reduction_column / 2); @@ -457,10 +472,36 @@ __global__ void MoeGemvFp4RawKPackedKernel( for (int offset = kKLanes / 2; offset > 0; offset /= 2) { accumulator += __shfl_xor_sync(kFullMask, accumulator, offset, kKLanes); } - const float paired_accumulator = __shfl_xor_sync(kFullMask, accumulator, kKLanes); - if (k_lane == 0 && column < output_columns && column % 2 == 0) { - StoreMoeGemvFp4Pair( - accumulator, paired_accumulator, expert_bias, out, row, column, output_columns, expert, activation_params); + if constexpr (FusedFinalize) { + __shared__ float expert_results[16][kColsPerBlock]; + __shared__ bool expert_valid[16]; + if (threadIdx.x == 0) expert_valid[threadIdx.y] = valid_expert; + if (k_lane == 0) { + if constexpr (EnableBias) { + if (valid_expert && column < output_columns) accumulator += static_cast(expert_bias[column]); + } + const T rounded = static_cast(accumulator); + expert_results[threadIdx.y][threadIdx.x / kKLanes] = static_cast(rounded); + } + __syncthreads(); + if (threadIdx.y == 0 && threadIdx.x < kColsPerBlock) { + const int output_column = static_cast(blockIdx.y) * kColsPerBlock + threadIdx.x; + if (output_column < output_columns) { + float sum = 0.0f; + for (int route = 0; route < experts_per_token; ++route) { + if (expert_valid[route]) { + sum += routing_scales[blockIdx.x * experts_per_token + route] * expert_results[route][threadIdx.x]; + } + } + out[static_cast(blockIdx.x) * output_columns + output_column] = static_cast(sum); + } + } + } else { + const float paired_accumulator = __shfl_xor_sync(kFullMask, accumulator, kKLanes); + if (k_lane == 0 && column < output_columns && column % 2 == 0) { + StoreMoeGemvFp4Pair( + accumulator, paired_accumulator, expert_bias, out, row, column, output_columns, expert, activation_params); + } } if constexpr (ReuseExpertRows) { if (second_row) { @@ -867,6 +908,43 @@ __global__ void __launch_bounds__(Warps * 32, MinBlocks) MoeGemvFp4DecodeKernel( #endif } +template +bool try_launch_moe_gemv_nvfp4_fused_finalize( + const T* act, const uint8_t* weight, const uint8_t* block_scales, const float* global_scales, + const T* bias, T* out, const int* unpermuted_row_to_permuted_row, + const int* permuted_row_to_expert, const float* routing_scales, int num_experts, + int num_rows, int experts_per_token, int n, int k, cudaStream_t stream) { + if (num_rows < 1 || num_rows > 2 || experts_per_token < 1 || experts_per_token > 16 || + n <= 0 || n % 2 != 0 || k <= 0 || k % 16 != 0) { + return false; + } + const dim3 grid(static_cast(num_rows), static_cast((n + 7) / 8)); + const dim3 block(64, static_cast(experts_per_token)); + const int64_t weight_stride = static_cast(n) * k / 2; + const int64_t scale_stride = static_cast(n) * (k / 16); + const cutlass_kernels::ActivationParams activation; + if (bias != nullptr) { + MoeGemvFp4RawKPackedKernel<<>>( + act, weight, block_scales, global_scales, bias, out, nullptr, permuted_row_to_expert, + num_experts, weight_stride, scale_stride, n, k, activation, nullptr, num_rows, + unpermuted_row_to_permuted_row, routing_scales, experts_per_token); + } else { + MoeGemvFp4RawKPackedKernel<<>>( + act, weight, block_scales, global_scales, bias, out, nullptr, permuted_row_to_expert, + num_experts, weight_stride, scale_stride, n, k, activation, nullptr, num_rows, + unpermuted_row_to_permuted_row, routing_scales, experts_per_token); + } + CUDA_CALL_THROW(cudaGetLastError()); + return true; +} + +template bool try_launch_moe_gemv_nvfp4_fused_finalize( + const half*, const uint8_t*, const uint8_t*, const float*, const half*, half*, + const int*, const int*, const float*, int, int, int, int, int, cudaStream_t); +template bool try_launch_moe_gemv_nvfp4_fused_finalize<__nv_bfloat16>( + const __nv_bfloat16*, const uint8_t*, const uint8_t*, const float*, const __nv_bfloat16*, __nv_bfloat16*, + const int*, const int*, const float*, int, int, int, int, int, cudaStream_t); + template void LaunchMoeGemvFp4RawNPacked( const T* act, const uint8_t* weight, const uint8_t* block_scales, const float* global_scales, @@ -879,10 +957,11 @@ void LaunchMoeGemvFp4RawNPacked( *decode_kernel_launched = false; } constexpr int kThreads = 128; - constexpr int kColsPerBlock = 16; + constexpr int kColsPerBlock = FusedSwiGlu ? 8 : 16; const int64_t weight_expert_stride = n * k / 2; const int64_t scale_expert_stride = n * (k / 16); - const dim3 grid(static_cast(expanded_num_rows), static_cast((n + kColsPerBlock - 1) / kColsPerBlock)); + const int columns_per_block = weights_row_major ? kColsPerBlock : 16; + const dim3 grid(static_cast(expanded_num_rows), static_cast((n + columns_per_block - 1) / columns_per_block)); if (weights_row_major) { static const bool enable_expert_reuse = onnxruntime::ParseEnvironmentVariableWithDefault("ORT_NVFP4_FC1_EXPERT_REUSE", 0) == 1; diff --git a/onnxruntime/contrib_ops/cuda/llm/moe_gemm/moe_gemv_fp4.h b/onnxruntime/contrib_ops/cuda/llm/moe_gemm/moe_gemv_fp4.h index 4081168fd07cc..19d9385520fb1 100644 --- a/onnxruntime/contrib_ops/cuda/llm/moe_gemm/moe_gemv_fp4.h +++ b/onnxruntime/contrib_ops/cuda/llm/moe_gemm/moe_gemv_fp4.h @@ -124,6 +124,15 @@ void launch_moe_gemv_fp4_symmetric( bool raw_n_packed, cudaStream_t stream, bool weights_row_major = false, bool decode_scale_eligible = false, bool* decode_kernel_launched = nullptr); +// Native K-packed NVFP4 FC2 with top-k reduction; false leaves unsupported shapes to the caller. +// The caller must use single-EP/single-TP routing. Each expert result is rounded to T before reduction. +template +bool try_launch_moe_gemv_nvfp4_fused_finalize( + const T* act, const uint8_t* weight, const uint8_t* block_scales, const float* global_scales, + const T* bias, T* out, const int* unpermuted_row_to_permuted_row, + const int* permuted_row_to_expert, const float* routing_scales, int num_experts, + int num_rows, int experts_per_token, int n, int k, cudaStream_t stream); + // Launches the MXFP4 MoE GEMV and fuses interleaved SwiGLU activation. // weight/scales/bias use raw FC1 output width n = 2 * inter_size // out is post-activation [expanded_num_rows, inter_size] diff --git a/onnxruntime/contrib_ops/cuda/llm/moe_gemm/moe_kernels.cu b/onnxruntime/contrib_ops/cuda/llm/moe_gemm/moe_kernels.cu index 43bd9170cc772..77e5c2f6e9783 100644 --- a/onnxruntime/contrib_ops/cuda/llm/moe_gemm/moe_kernels.cu +++ b/onnxruntime/contrib_ops/cuda/llm/moe_gemm/moe_kernels.cu @@ -1608,8 +1608,8 @@ __global__ void finalizeMoeRoutingKernel(const GemmOutputType* expanded_permuted constexpr int64_t FINALIZE_ELEM_PER_THREAD = 128 / std::min(sizeof_bits::value, sizeof_bits::value); assert(orig_cols % FINALIZE_ELEM_PER_THREAD == 0); - const int64_t start_offset = threadIdx.x; - const int64_t stride = FINALIZE_THREADS_PER_BLOCK; + const int64_t start_offset = static_cast(blockIdx.y) * blockDim.x + threadIdx.x; + const int64_t stride = static_cast(blockDim.x) * gridDim.y; const int64_t num_elems_in_col = orig_cols / FINALIZE_ELEM_PER_THREAD; using BiasElem = cutlass::Array; @@ -1628,7 +1628,7 @@ __global__ void finalizeMoeRoutingKernel(const GemmOutputType* expanded_permuted __shared__ int s_expert_id[STAGE_ROUTING ? FINALIZE_MAX_STAGED_TOPK : 1]; __shared__ float s_row_scale[STAGE_ROUTING ? FINALIZE_MAX_STAGED_TOPK : 1]; if constexpr (STAGE_ROUTING) { - for (int64_t k_idx = threadIdx.x; k_idx < experts_per_token; k_idx += FINALIZE_THREADS_PER_BLOCK) { + for (int64_t k_idx = threadIdx.x; k_idx < experts_per_token; k_idx += blockDim.x) { int64_t const k_offset = original_row * experts_per_token + k_idx; int const expert_id = token_selected_experts[k_offset] - start_expert_id; bool const is_local_expert = expert_id >= 0 && expert_id < num_experts_per_node; @@ -1915,6 +1915,13 @@ void finalizeMoeRoutingKernelLauncher(const GemmOutputType* expanded_permuted_ro } #undef LAUNCH_FINALIZE_ONE_ROW } else if (experts_per_token <= FINALIZE_MAX_STAGED_TOPK) { + if (num_rows <= 2) { + constexpr int kDecodeThreads = 32; + config.blockDim = dim3(kDecodeThreads); + config.gridDim = dim3(static_cast(num_rows), + static_cast((cols / kFinalizeElemPerThread + kDecodeThreads - 1) / + kDecodeThreads)); + } auto func = final_scales ? &finalizeMoeRoutingKernel : &finalizeMoeRoutingKernel; diff --git a/onnxruntime/contrib_ops/cuda/math/matmul_block_scaled_fp8.cc b/onnxruntime/contrib_ops/cuda/math/matmul_block_scaled_fp8.cc index 28ef396fc8cbf..f98f3a7232f6c 100644 --- a/onnxruntime/contrib_ops/cuda/math/matmul_block_scaled_fp8.cc +++ b/onnxruntime/contrib_ops/cuda/math/matmul_block_scaled_fp8.cc @@ -48,6 +48,16 @@ MatMulBlockQuantizedFp8Weight::MatMulBlockQuantizedFp8Weight(const OpKernelInfo& max_dequant_scratch_bytes_(DequantScratchLimitBytes()), enable_deep_gemm_(ParseEnvironmentVariableWithDefault("ORT_FP8_MATMUL_DEEPGEMM", false)) { ORT_ENFORCE(block_size_ > 0, "block_size must be positive."); + if (info.GetAttrs("split_sizes", split_sizes_).IsOK()) { + ORT_ENFORCE(!split_sizes_.empty() && split_sizes_.size() <= Fp8GemvOutputSplit::kMaxOutputs, + "split_sizes requires one to eight widths."); + int64_t total = 0; + for (const auto width : split_sizes_) { + ORT_ENFORCE(width > 0 && width <= INT64_MAX - total, + "split_sizes must contain positive widths with no sum overflow."); + total += width; + } + } } template @@ -94,27 +104,75 @@ Status MatMulBlockQuantizedFp8Weight::ComputeImpl(OpKernelContext* context) cons TensorShape b_logical_shape({n, k}); ORT_RETURN_IF_ERROR(helper.Compute(a_shape, b_logical_shape, transa, transb)); - Tensor* Y = context->Output(0, helper.OutputShape()); - if (Y->Shape().Size() == 0) { - return Status::OK(); - } - const int m_i = SafeInt(helper.M()); const int n_i = SafeInt(helper.N()); const int k_i = SafeInt(helper.K()); + const bool use_gemv = m_i > 0 && + m_i <= MatMulBlockScaledFp8GemvMaxM(k_i, SafeInt(block_size_), GetDeviceProp()) && + (k_i % 16 == 0) && (block_size_ % 16 == 0); + const bool direct_split = !split_sizes_.empty() && use_gemv && + MatMulBlockScaledFp8GemvSupportsSplitOutput( + m_i, k_i, SafeInt(block_size_), GetDeviceProp()); + Fp8GemvOutputSplit split; + void* y_data; + IAllocatorUniquePtr y_scratch; + const auto output_shape = helper.OutputShape(); + if (split_sizes_.empty()) { + ORT_ENFORCE(context->OutputCount() == 1, "Without split_sizes exactly one output is required."); + y_data = context->Output(0, output_shape)->MutableDataRaw(); + } else { + ORT_ENFORCE(context->OutputCount() == static_cast(split_sizes_.size()), + "Output count must match split_sizes."); + int64_t total = 0; + for (const auto width : split_sizes_) { + total += width; + } + ORT_ENFORCE(total == n, "split_sizes must sum to the weight's N dimension."); + split.count = static_cast(split_sizes_.size()); + for (int i = 0; i < split.count; ++i) { + TensorShapeVector shape(output_shape.GetDims().begin(), output_shape.GetDims().end()); + shape.back() = split_sizes_[i]; + split.outputs[i] = context->Output(i, TensorShape(shape))->MutableDataRaw(); + split.widths[i] = SafeInt(split_sizes_[i]); + } + if (direct_split) { + y_data = split.outputs[0]; + } else { + y_scratch = GetScratchBuffer(SafeInt(output_shape.Size()), GetComputeStream(context)); + y_data = y_scratch.get(); + } + } + if (output_shape.Size() == 0) { + return Status::OK(); + } + const auto finish_output = [&]() -> Status { + if (split.count != 0 && !direct_split) { + size_t offset = 0; + for (int i = 0; i < split.count; ++i) { + const size_t width_bytes = static_cast(split.widths[i]) * sizeof(CudaT); + CUDA_RETURN_IF_ERROR(cudaMemcpy2DAsync( + split.outputs[i], width_bytes, static_cast(y_data) + offset, + static_cast(n_i) * sizeof(CudaT), width_bytes, m_i, + cudaMemcpyDeviceToDevice, Stream(context))); + offset += width_bytes; + } + } + return Status::OK(); + }; + if (k_i == 0) { - CUDA_RETURN_IF_ERROR(cudaMemsetAsync(Y->MutableDataRaw(), 0, Y->SizeInBytes(), Stream(context))); + CUDA_RETURN_IF_ERROR(cudaMemsetAsync(y_data, 0, SafeInt(output_shape.Size()) * sizeof(CudaT), Stream(context))); if (bias != nullptr) { ORT_RETURN_IF_ERROR(LaunchAddBiasBlockScaledFp8( - Y->MutableDataRaw(), + y_data, bias->DataRaw(), m_i, n_i, std::is_same::value, Stream(context))); } - return Status::OK(); + return finish_output(); } // Optional W8A8 activation path: statically quantize A to FP8 E4M3 and dequantize back so the @@ -123,9 +181,6 @@ Status MatMulBlockQuantizedFp8Weight::ComputeImpl(OpKernelContext* context) cons // // The decode GEMV absorbs this round trip in-register, so the standalone kernel (and its [M, K] // scratch buffer) is only materialized for the cuBLAS path below. - const bool use_gemv = m_i > 0 && - m_i <= MatMulBlockScaledFp8GemvMaxM(k_i, SafeInt(block_size_), GetDeviceProp()) && - (k_i % 16 == 0) && (block_size_ % 16 == 0); const bool fuse_act_qdq = use_gemv && !FusedFp8ActivationQdqDisabled(); #if defined(USE_DEEP_GEMM) @@ -154,14 +209,14 @@ Status MatMulBlockQuantizedFp8Weight::ComputeImpl(OpKernelContext* context) cons const size_t weight_offset = SafeInt(offset) * k_i; const size_t scale_offset = SafeInt(offset) * (k_i / 128); ORT_RETURN_IF_ERROR(LaunchMatMulFp8DeepGemm( - Y->MutableData() + offset, a_quant.get(), a_scales.get(), + static_cast(y_data) + offset, a_quant.get(), a_scales.get(), static_cast(b->DataRaw()) + weight_offset, b_scale->Data() + scale_offset, bias != nullptr ? bias->Data() + offset : nullptr, packed_b_scales.get(), accum.get(), m_i, rows, k_i, aligned_m, n_i, std::is_same::value, GetDeviceProp().multiProcessorCount, Stream(context))); } - return Status::OK(); + return finish_output(); } } #endif @@ -186,8 +241,8 @@ Status MatMulBlockQuantizedFp8Weight::ComputeImpl(OpKernelContext* context) cons // A fused warp-per-column kernel reads the FP8 weight directly, avoiding both the [N, K] // dequant scratch buffer and the cuBLAS GEMM (which is underutilized at M == 1). if (use_gemv) { - return LaunchMatMulBlockScaledFp8Gemv( - Y->MutableDataRaw(), + ORT_RETURN_IF_ERROR(LaunchMatMulBlockScaledFp8Gemv( + y_data, a_ptr, b->DataRaw(), b_scale->Data(), @@ -199,7 +254,8 @@ Status MatMulBlockQuantizedFp8Weight::ComputeImpl(OpKernelContext* context) cons SafeInt(block_size_), std::is_same::value, GetDeviceProp(), - Stream(context)); + Stream(context), direct_split ? split : Fp8GemvOutputSplit{})); + return finish_output(); } // Dequantize the FP8 weight into a scratch buffer of the activation type, then GEMM. The scratch @@ -243,7 +299,7 @@ Status MatMulBlockQuantizedFp8Weight::ComputeImpl(OpKernelContext* context) cons reinterpret_cast(a_ptr), helper.Lda(transa), &zero, - reinterpret_cast(Y->MutableDataRaw()) + n_offset, + static_cast(y_data) + n_offset, helper.Ldc(), GetDeviceProp(), UseTF32())); @@ -251,7 +307,7 @@ Status MatMulBlockQuantizedFp8Weight::ComputeImpl(OpKernelContext* context) cons if (bias != nullptr) { ORT_RETURN_IF_ERROR(LaunchAddBiasBlockScaledFp8( - Y->MutableDataRaw(), + y_data, bias->DataRaw(), m_i, n_i, @@ -259,7 +315,7 @@ Status MatMulBlockQuantizedFp8Weight::ComputeImpl(OpKernelContext* context) cons Stream(context))); } - return Status::OK(); + return finish_output(); #endif } diff --git a/onnxruntime/contrib_ops/cuda/math/matmul_block_scaled_fp8.cu b/onnxruntime/contrib_ops/cuda/math/matmul_block_scaled_fp8.cu index d045519626b27..16ae3c0d5c949 100644 --- a/onnxruntime/contrib_ops/cuda/math/matmul_block_scaled_fp8.cu +++ b/onnxruntime/contrib_ops/cuda/math/matmul_block_scaled_fp8.cu @@ -514,7 +514,20 @@ struct Fp8GemvMma<__nv_bfloat16> { } }; -template +template +__device__ __forceinline__ T* Fp8OutputColumn(T* output, int col, int& stride, + const Fp8GemvOutputSplit& split) { + for (int i = 0; i < split.count; ++i) { + if (col < split.widths[i]) { + stride = split.widths[i]; + return static_cast(split.outputs[i]) + col; + } + col -= split.widths[i]; + } + return output + col; +} + +template __device__ __forceinline__ void Fp8MmaGemvBody(AType* __restrict__ output, const AType* __restrict__ input_a, const __nv_fp8_e4m3* __restrict__ input_b, @@ -525,7 +538,8 @@ __device__ __forceinline__ void Fp8MmaGemvBody(AType* __restrict__ output, int n, int k, int block_size, - int k_blocks) { + int k_blocks, + const Fp8GemvOutputSplit& split) { using Mma = Fp8GemvMma; const bool act_qdq = act_scale != nullptr; @@ -560,6 +574,7 @@ __device__ __forceinline__ void Fp8MmaGemvBody(AType* __restrict__ output, float accb[MTiles][4] = {}; // unscaled, within the current K block int cur_kb = -1; +#pragma unroll 2 for (int wi = warp; wi < windows; wi += KSplit) { const int k0 = wi << 6; const int kb = k0 / block_size; @@ -670,23 +685,31 @@ __device__ __forceinline__ void Fp8MmaGemvBody(AType* __restrict__ output, // M tile. const float bias_lo = (bias != nullptr && lo_ok) ? to_float(bias[col_lo]) : 0.f; const float bias_hi = (bias != nullptr && hi_ok) ? to_float(bias[col_hi]) : 0.f; + int stride_lo = n; + int stride_hi = n; + AType* output_lo = output + col_lo; + AType* output_hi = output + col_hi; + if constexpr (SplitOutput) { + output_lo = lo_ok ? Fp8OutputColumn(output, col_lo, stride_lo, split) : nullptr; + output_hi = hi_ok ? Fp8OutputColumn(output, col_hi, stride_hi, split) : nullptr; + } #pragma unroll for (int mt = 0; mt < MTiles; ++mt) { const int row = (t << 1) + (mt << 3); if (row < m) { if (lo_ok) { - output[static_cast(row) * n + col_lo] = from_float(acc[mt][0] + bias_lo); + output_lo[static_cast(row) * stride_lo] = from_float(acc[mt][0] + bias_lo); } if (hi_ok) { - output[static_cast(row) * n + col_hi] = from_float(acc[mt][2] + bias_hi); + output_hi[static_cast(row) * stride_hi] = from_float(acc[mt][2] + bias_hi); } } if (row + 1 < m) { if (lo_ok) { - output[static_cast(row + 1) * n + col_lo] = from_float(acc[mt][1] + bias_lo); + output_lo[static_cast(row + 1) * stride_lo] = from_float(acc[mt][1] + bias_lo); } if (hi_ok) { - output[static_cast(row + 1) * n + col_hi] = from_float(acc[mt][3] + bias_hi); + output_hi[static_cast(row + 1) * stride_hi] = from_float(acc[mt][3] + bias_hi); } } } @@ -695,6 +718,12 @@ __device__ __forceinline__ void Fp8MmaGemvBody(AType* __restrict__ output, // Two entry points over one body. The pinned one carries a residency hint; see // `Fp8MmaGemvPinsResidency` for when the launcher picks it and why the plain one has to stay. // clang-format off +#if CUDA_VERSION >= 12000 +// Avoid a per-thread local copy when the inlined store helper references this descriptor. +#define ORT_FP8_SPLIT_PARAM const __grid_constant__ Fp8GemvOutputSplit split +#else +#define ORT_FP8_SPLIT_PARAM const Fp8GemvOutputSplit split +#endif #define ORT_FP8_MMA_GEMV_PARAMS \ AType* __restrict__ output, \ const AType* __restrict__ input_a, \ @@ -702,24 +731,25 @@ __device__ __forceinline__ void Fp8MmaGemvBody(AType* __restrict__ output, const float* __restrict__ weight_scale, \ const AType* __restrict__ bias, \ const float* __restrict__ act_scale, \ - int m, int n, int k, int block_size, int k_blocks + int m, int n, int k, int block_size, int k_blocks, ORT_FP8_SPLIT_PARAM #define ORT_FP8_MMA_GEMV_ARGS \ - output, input_a, input_b, weight_scale, bias, act_scale, m, n, k, block_size, k_blocks + output, input_a, input_b, weight_scale, bias, act_scale, m, n, k, block_size, k_blocks, split // clang-format on -template +template __global__ void MatMulBlockScaledFp8MmaGemvKernel(ORT_FP8_MMA_GEMV_PARAMS) { - Fp8MmaGemvBody(ORT_FP8_MMA_GEMV_ARGS); + Fp8MmaGemvBody(ORT_FP8_MMA_GEMV_ARGS); } -template +template __global__ __launch_bounds__(32 * KSplit, 3) void MatMulBlockScaledFp8MmaGemvKernelPinned(ORT_FP8_MMA_GEMV_PARAMS) { - Fp8MmaGemvBody(ORT_FP8_MMA_GEMV_ARGS); + Fp8MmaGemvBody(ORT_FP8_MMA_GEMV_ARGS); } #undef ORT_FP8_MMA_GEMV_ARGS #undef ORT_FP8_MMA_GEMV_PARAMS +#undef ORT_FP8_SPLIT_PARAM // Kill switch for A/B testing the tensor-core path against the FMA path in the same binary. bool Fp8GemvMmaEnabled() { @@ -915,6 +945,20 @@ bool Fp8MmaGb10TuningEnabled() { return enabled; } +bool MatMulBlockScaledFp8GemvSupportsSplitOutput(int m, int k, int block_size, + const cudaDeviceProp& device_prop) { +#if !defined(DISABLE_FLOAT8_TYPES) && defined(CUDA_VERSION) && CUDA_VERSION >= 11080 + return device_prop.major >= 8 && m > 0 && m <= kFp8MmaGemvTileM && + k >= 256 && k % 64 == 0 && block_size % 64 == 0 && Fp8GemvMmaEnabled(); +#else + ORT_UNUSED_PARAMETER(m); + ORT_UNUSED_PARAMETER(k); + ORT_UNUSED_PARAMETER(block_size); + ORT_UNUSED_PARAMETER(device_prop); + return false; +#endif +} + static Status LaunchMatMulBlockScaledFp8GemvImpl(void* y, const void* a, const void* b_fp8, @@ -928,7 +972,8 @@ static Status LaunchMatMulBlockScaledFp8GemvImpl(void* y, bool is_bf16, const cudaDeviceProp& device_prop, cudaStream_t stream, - bool enable_gb10_ksplit32) { + bool enable_gb10_ksplit32, + const Fp8GemvOutputSplit& split) { #if !defined(DISABLE_FLOAT8_TYPES) && defined(CUDA_VERSION) && CUDA_VERSION >= 11080 if (m <= 0 || n <= 0 || k <= 0) { return Status::OK(); @@ -941,6 +986,19 @@ static Status LaunchMatMulBlockScaledFp8GemvImpl(void* y, "MatMulBlockQuantizedFp8Weight GEMV requires block_size divisible by 16, got ", block_size, "."); const int k_blocks = (k + block_size - 1) / block_size; const auto* b = reinterpret_cast(b_fp8); + ORT_RETURN_IF_NOT(split.count >= 0 && split.count <= Fp8GemvOutputSplit::kMaxOutputs, + "FP8 GEMV split output count must be between zero and eight."); + if (split.count != 0) { + int64_t total = 0; + for (int i = 0; i < split.count; ++i) { + ORT_RETURN_IF_NOT(split.widths[i] > 0 && split.outputs[i] != nullptr, + "FP8 GEMV split outputs require positive widths and non-null buffers."); + total += split.widths[i]; + } + ORT_RETURN_IF_NOT(total == n, "FP8 GEMV split widths must sum to N."); + } + ORT_RETURN_IF_NOT(split.count == 0 || MatMulBlockScaledFp8GemvSupportsSplitOutput(m, k, block_size, device_prop), + "Direct split output requires the tensor-core GEMV path with M <= 32."); if (m > kFp8MmaGemvTileM) { ORT_RETURN_IF_NOT(device_prop.major >= 8 && k % 64 == 0 && k >= 256 && @@ -950,12 +1008,12 @@ static Status LaunchMatMulBlockScaledFp8GemvImpl(void* y, const size_t element_size = is_bf16 ? sizeof(__nv_bfloat16) : sizeof(half); ORT_RETURN_IF_ERROR(LaunchMatMulBlockScaledFp8GemvImpl( y, a, b_fp8, weight_scale, bias, act_scale, kFp8MmaGemvTileM, n, k, block_size, - is_bf16, device_prop, stream, false)); + is_bf16, device_prop, stream, false, {})); return LaunchMatMulBlockScaledFp8GemvImpl( static_cast(y) + static_cast(kFp8MmaGemvTileM) * n * element_size, static_cast(a) + static_cast(kFp8MmaGemvTileM) * k * element_size, b_fp8, weight_scale, bias, act_scale, m - kFp8MmaGemvTileM, n, k, block_size, - is_bf16, device_prop, stream, false); + is_bf16, device_prop, stream, false, {}); } // Tensor-core path (SM80+). Beats the FMA kernel at every M on H200: 1.06-1.23x at M == 1 and @@ -982,28 +1040,35 @@ static Status LaunchMatMulBlockScaledFp8GemvImpl(void* y, n, k_split, mtiles, device_prop.multiProcessorCount, device_prop.major, device_prop.minor); const auto launch_mma = [&]() { const dim3 mma_threads{32, KSplit}; + const auto launch_variant = [&]() { #define ORT_FP8_LAUNCH_MMA(kernel_name) \ do { \ if (is_bf16) { \ - kernel_name<<>>( \ + kernel_name<<>>( \ reinterpret_cast<__nv_bfloat16*>(y), reinterpret_cast(a), b, \ weight_scale, reinterpret_cast(bias), act_scale, m, n, k, \ - block_size, k_blocks); \ + block_size, k_blocks, split); \ } else { \ - kernel_name<<>>( \ + kernel_name<<>>( \ reinterpret_cast(y), reinterpret_cast(a), b, \ weight_scale, reinterpret_cast(bias), act_scale, m, n, k, \ - block_size, k_blocks); \ + block_size, k_blocks, split); \ } \ } while (0) - if constexpr (KSplit == 16 && MTiles == 1) { - if (pin_residency) { - ORT_FP8_LAUNCH_MMA(MatMulBlockScaledFp8MmaGemvKernelPinned); - return; + if constexpr (KSplit == 16 && MTiles == 1) { + if (pin_residency) { + ORT_FP8_LAUNCH_MMA(MatMulBlockScaledFp8MmaGemvKernelPinned); + return; + } } - } - ORT_FP8_LAUNCH_MMA(MatMulBlockScaledFp8MmaGemvKernel); + ORT_FP8_LAUNCH_MMA(MatMulBlockScaledFp8MmaGemvKernel); #undef ORT_FP8_LAUNCH_MMA + }; + if (split.count) { + launch_variant.template operator()(); + } else { + launch_variant.template operator()(); + } }; // Only 1, 2 and 4 row tiles are instantiated; an M of 17..24 rounds up to 4 and masks the // remainder, which costs nothing next to the weight traffic it shares. @@ -1109,6 +1174,7 @@ static Status LaunchMatMulBlockScaledFp8GemvImpl(void* y, ORT_UNUSED_PARAMETER(device_prop); ORT_UNUSED_PARAMETER(stream); ORT_UNUSED_PARAMETER(enable_gb10_ksplit32); + ORT_UNUSED_PARAMETER(split); return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "MatMulBlockQuantizedFp8Weight requires CUDA 11.8 or later."); #endif } @@ -1125,10 +1191,11 @@ Status LaunchMatMulBlockScaledFp8Gemv(void* y, int block_size, bool is_bf16, const cudaDeviceProp& device_prop, - cudaStream_t stream) { + cudaStream_t stream, + const Fp8GemvOutputSplit& split) { return LaunchMatMulBlockScaledFp8GemvImpl( y, a, b_fp8, weight_scale, bias, act_scale, m, n, k, block_size, - is_bf16, device_prop, stream, true); + is_bf16, device_prop, stream, true, split); } } // namespace onnxruntime::contrib::cuda diff --git a/onnxruntime/contrib_ops/cuda/math/matmul_block_scaled_fp8.h b/onnxruntime/contrib_ops/cuda/math/matmul_block_scaled_fp8.h index 7128b86971dcc..b54a37da08a87 100644 --- a/onnxruntime/contrib_ops/cuda/math/matmul_block_scaled_fp8.h +++ b/onnxruntime/contrib_ops/cuda/math/matmul_block_scaled_fp8.h @@ -4,6 +4,7 @@ #pragma once #include +#include #include "core/providers/cuda/cuda_kernel.h" @@ -31,10 +32,21 @@ class MatMulBlockQuantizedFp8Weight final : public onnxruntime::cuda::CudaKernel Status ComputeImpl(OpKernelContext* context) const; int64_t block_size_; + std::vector split_sizes_; size_t max_dequant_scratch_bytes_; bool enable_deep_gemm_; }; +struct Fp8GemvOutputSplit { + static constexpr int kMaxOutputs = 8; + void* outputs[kMaxOutputs]{}; + int widths[kMaxOutputs]{}; + int count = 0; +}; + +bool MatMulBlockScaledFp8GemvSupportsSplitOutput(int m, int k, int block_size, + const cudaDeviceProp& device_prop); + #if defined(USE_DEEP_GEMM) Status LaunchPrepareMatMulFp8DeepGemm(void* a_quant, float* a_scales, const void* a, const float* a_scale, int m, int k, int aligned_m, @@ -83,6 +95,8 @@ Status LaunchQuantizeDequantizeActivationFp8(void* a_out, // directly (no [N, K] dequant buffer). a is [M, K] activation (FP16/BF16), b_fp8 is [N, K] // FP8 E4M3, weight_scale is [N, ceil(K/block_size)] fp32, bias is an optional [N] vector (may be // null). Output y is [M, N] in the activation type. Requires k % 16 == 0 and block_size % 16 == 0. +// A nonempty split descriptor routes tensor-core stores into consecutive column partitions; +// callers must check MatMulBlockScaledFp8GemvSupportsSplitOutput before using that mode. // device_prop selects the tensor-core (mma.m16n8k16) variant, which needs SM80+. // act_scale is an optional device fp32 scalar; when non-null the kernel applies the W8A8 // activation quantize/dequantize inline, bit-identically to LaunchQuantizeDequantizeActivationFp8, @@ -100,7 +114,8 @@ Status LaunchMatMulBlockScaledFp8Gemv(void* y, int block_size, bool is_bf16, const cudaDeviceProp& device_prop, - cudaStream_t stream); + cudaStream_t stream, + const Fp8GemvOutputSplit& split = {}); // Largest M selected for GEMV dispatch. The tensor-core default is conservative because the // crossover with dequantize + cuBLAS depends on the matrix shape; ORT_FP8_GEMV_MAX_M can raise the diff --git a/onnxruntime/contrib_ops/cuda/moe/moe_quantization.cc b/onnxruntime/contrib_ops/cuda/moe/moe_quantization.cc index a48b2b9ca8e3c..b0cea76b5514b 100644 --- a/onnxruntime/contrib_ops/cuda/moe/moe_quantization.cc +++ b/onnxruntime/contrib_ops/cuda/moe/moe_quantization.cc @@ -2280,7 +2280,10 @@ Status QMoE::ComputeInternal(OpKernelContext* context) const { IAllocatorUniquePtr blocked_expert_counts; IAllocatorUniquePtr blocked_expert_counts_cumsum; IAllocatorUniquePtr blocked_row_to_unpermuted_row; - if (!ck::fusedBuildExpertMapsSortFirstToken( + if (!TryLaunchSmallDecodeExpertMaps( + expert_indices, p_r2u, unpermuted_row_to_permuted_row, p_exp, p_efto, + static_cast(num_rows), num_experts, static_cast(k_), stream) && + !ck::fusedBuildExpertMapsSortFirstToken( expert_indices, p_r2u, unpermuted_row_to_permuted_row, p_exp, p_efto, num_rows, num_experts, static_cast(k_), 0, num_experts, stream)) { const int64_t num_tokens_per_block = ck::computeNumTokensPerBlock(num_rows, num_experts); @@ -2452,6 +2455,16 @@ Status QMoE::ComputeInternal(OpKernelContext* context) const { } launch_fc1(fc1_config); + if (use_raw_nvfp4_gemv && nvfp4_weights_row_major_ && + parallelism_config.tp_size == 1 && parallelism_config.ep_size == 1 && + gemv::try_launch_moe_gemv_nvfp4_fused_finalize( + static_cast(p_fc1_buf.get()), gemv_fc2_weight, gemv_fc2_raw_block_scales, + gemv_fc2_raw_global_scales, static_cast(fc2_bias), + static_cast(output->MutableDataRaw()), unpermuted_row_to_permuted_row, + p_exp, expert_scales, num_experts, static_cast(num_rows), static_cast(k_), + static_cast(hidden), static_cast(inter), stream)) { + return; + } launch_fc2(fc2_config); ck::finalizeMoeRoutingKernelLauncher( static_cast(p_fc2_buf.get()), static_cast(output->MutableDataRaw()), diff --git a/onnxruntime/contrib_ops/cuda/moe/qmoe_kernels.cu b/onnxruntime/contrib_ops/cuda/moe/qmoe_kernels.cu index 331351a6ac33b..7225714263b10 100644 --- a/onnxruntime/contrib_ops/cuda/moe/qmoe_kernels.cu +++ b/onnxruntime/contrib_ops/cuda/moe/qmoe_kernels.cu @@ -223,6 +223,42 @@ __global__ void SoftmaxTopKMergeKernel(const T* logits, float* topk_scales, int* } } +__global__ void SmallDecodeExpertMapsKernel( + const int* indices, int* row_to_source, int* source_to_row, int* sorted_experts, + int64_t* expert_offsets, int rows, int experts, int k) { + __shared__ int selected[32]; + const int expanded = rows * k; + if (threadIdx.x < expanded) selected[threadIdx.x] = indices[threadIdx.x]; + __syncthreads(); + if (threadIdx.x < expanded) { + const int original = threadIdx.x; + const int expert = selected[original]; + int rank = 0; + for (int i = 0; i < expanded; ++i) { + rank += selected[i] < expert || (selected[i] == expert && i < original); + } + const int source = (original % k) * rows + original / k; + row_to_source[rank] = source; + source_to_row[source] = rank; + sorted_experts[rank] = expert; + } + for (int expert = threadIdx.x; expert <= experts; expert += blockDim.x) { + int count = 0; + for (int i = 0; i < expanded; ++i) count += selected[i] < expert; + expert_offsets[expert] = count; + } +} + +bool TryLaunchSmallDecodeExpertMaps( + const int* indices, int* row_to_source, int* source_to_row, int* sorted_experts, + int64_t* expert_offsets, int rows, int experts, int k, cudaStream_t stream) { + if (rows < 1 || rows > 2 || experts < 1 || experts > 512 || k < 1 || k > 16 || k > experts) return false; + SmallDecodeExpertMapsKernel<<<1, 128, 0, stream>>>( + indices, row_to_source, source_to_row, sorted_experts, expert_offsets, rows, experts, k); + CUDA_CALL_THROW(cudaGetLastError()); + return true; +} + // Warp-bitonic softmax + top-k for num_experts <= 32. Each warp handles one // row, with lane `l` owning expert `l`. The whole softmax reduction and the // sort are done with warp shuffles (no shared memory). This is the fastest path diff --git a/onnxruntime/contrib_ops/cuda/moe/qmoe_kernels.h b/onnxruntime/contrib_ops/cuda/moe/qmoe_kernels.h index 00a49bddbe72d..c28cc343f4ccf 100644 --- a/onnxruntime/contrib_ops/cuda/moe/qmoe_kernels.h +++ b/onnxruntime/contrib_ops/cuda/moe/qmoe_kernels.h @@ -12,6 +12,10 @@ namespace onnxruntime { namespace contrib { namespace cuda { +bool TryLaunchSmallDecodeExpertMaps( + const int* indices, int* row_to_source, int* source_to_row, int* sorted_experts, + int64_t* expert_offsets, int rows, int experts, int k, cudaStream_t stream); + void LaunchSoftmaxTopK( const float* logits, float* topk_scales, diff --git a/onnxruntime/contrib_ops/cuda/quantization/quantize_dequantize_linear.cc b/onnxruntime/contrib_ops/cuda/quantization/quantize_dequantize_linear.cc index f612a9b6a6f54..40f54aba301cf 100644 --- a/onnxruntime/contrib_ops/cuda/quantization/quantize_dequantize_linear.cc +++ b/onnxruntime/contrib_ops/cuda/quantization/quantize_dequantize_linear.cc @@ -34,7 +34,8 @@ REGISTER_Q_KERNEL_TYPED(uint8_t, MLFloat16) kCudaExecutionProvider, \ (*KernelDefBuilder::Create()) \ .TypeConstraint("T1", DataTypeImpl::GetTensorType()) \ - .TypeConstraint("T2", DataTypeImpl::GetTensorType()), \ + .TypeConstraint("T2", DataTypeImpl::GetTensorType()) \ + .TypeConstraint("TS", DataTypeImpl::GetTensorType()), \ DequantizeLinear); REGISTER_DQ_KERNEL_TYPED(int8_t, MLFloat16) diff --git a/onnxruntime/core/graph/contrib_ops/contrib_defs.cc b/onnxruntime/core/graph/contrib_ops/contrib_defs.cc index c38c4d1e1936e..477b8b2b6dd69 100644 --- a/onnxruntime/core/graph/contrib_ops/contrib_defs.cc +++ b/onnxruntime/core/graph/contrib_ops/contrib_defs.cc @@ -3191,9 +3191,13 @@ are `A_scaled = fp8_e4m3(A / a_scale) * a_scale` (W8A8). Otherwise, A retains it precision (weight-only W8A16). The operator multiplies the activation by the transpose of B_scaled and adds the optional bias. -The output has shape [..., N] and the same element type as A.)DOC") +The output has shape [..., N] and the same element type as A. When split_sizes is provided, +return up to eight consecutive last-dimension partitions instead, equivalent to Split(Y, split_sizes). +The partition widths must be positive and sum to N.)DOC") .Attr("block_size", "Number of consecutive K values that share one weight scale. Default 128.", AttributeProto::INT, static_cast(128)) + .Attr("split_sizes", "Optional last-dimension output widths (one to eight positive sizes).", + AttributeProto::INTS, OPTIONAL_VALUE) .Input(0, "A", "Row-major FP16/BF16 activation of shape [..., K].", "T") .Input(1, "B", "Row-major FP8 E4M3 weight of shape [N, K].", "T1") .Input(2, "b_scale", "Per-block FP32 weight scales of shape [N, ceil(K / block_size)].", "T2") @@ -3203,13 +3207,31 @@ The output has shape [..., N] and the same element type as A.)DOC") "its FP16/BF16 precision.", "T2", OpSchema::Optional) .Input(4, "bias", "Optional bias of shape [N].", "T", OpSchema::Optional) - .Output(0, "Y", "Output of shape [..., N] in the activation type.", "T") + .Output(0, "Y", "Output(s) of shape [..., N] or [..., split_sizes[i]] in the activation type.", + "T", OpSchema::Variadic) .TypeConstraint("T", {"tensor(float16)", "tensor(bfloat16)"}, "Constrain activation, bias and output to FP16 or BF16.") .TypeConstraint("T1", {"tensor(float8e4m3fn)"}, "Constrain weight to FP8 E4M3.") .TypeConstraint("T2", {"tensor(float)"}, "Constrain scales to FP32.") .TypeAndShapeInferenceFunction([](ONNX_NAMESPACE::InferenceContext& ctx) { - propagateElemTypeFromInputToOutput(ctx, 0, 0); + const auto* split = ctx.getAttribute("split_sizes"); + if ((split == nullptr && ctx.getNumOutputs() != 1) || + (split != nullptr && (split->ints_size() < 1 || split->ints_size() > 8 || + static_cast(split->ints_size()) != ctx.getNumOutputs()))) { + fail_shape_inference("Output count must match split_sizes, or be one when split_sizes is absent."); + } + int64_t split_total = 0; + if (split != nullptr) { + for (const auto width : split->ints()) { + if (width <= 0 || width > INT64_MAX - split_total) { + fail_shape_inference("split_sizes must contain positive widths with no sum overflow."); + } + split_total += width; + } + } + for (size_t i = 0; i < ctx.getNumOutputs(); ++i) { + propagateElemTypeFromInputToOutput(ctx, 0, i); + } if (!hasNInputShapes(ctx, 2)) { return; } @@ -3227,7 +3249,16 @@ The output has shape [..., N] and the same element type as A.)DOC") *output_shape.add_dim() = a_shape.dim(i); } *output_shape.add_dim() = b_shape.dim(0); - updateOutputShape(ctx, 0, output_shape); + if (split != nullptr && b_shape.dim(0).has_dim_value() && + split_total != b_shape.dim(0).dim_value()) { + fail_shape_inference("split_sizes must sum to the weight's N dimension."); + } + for (size_t i = 0; i < ctx.getNumOutputs(); ++i) { + if (split != nullptr) { + output_shape.mutable_dim(output_shape.dim_size() - 1)->set_dim_value(split->ints(static_cast(i))); + } + updateOutputShape(ctx, i, output_shape); + } })); static void MatmulWithQuantWeightShapeInference(ONNX_NAMESPACE::InferenceContext& ctx, diff --git a/onnxruntime/core/graph/contrib_ops/quantization_defs.cc b/onnxruntime/core/graph/contrib_ops/quantization_defs.cc index 70cd688832b02..46bfebf9ec693 100644 --- a/onnxruntime/core/graph/contrib_ops/quantization_defs.cc +++ b/onnxruntime/core/graph/contrib_ops/quantization_defs.cc @@ -182,12 +182,16 @@ ONNX_MS_OPERATOR_SET_SCHEMA( })); static const char* DequantizeLinear_ver1_doc = R"DOC( -The linear dequantization operator. It consumes a quantized data, a scale, a zero point and computes the full precision data. -The dequantization formula is y = (x - x_zero_point) * x_scale. -Scale and zero point must have same shape. They must be either scalar (per tensor) or 1-D tensor (per 'axis').)DOC"; +The linear dequantization operator. It consumes quantized data, a scale, and an optional zero point. +The dequantization formula is y = (x - x_zero_point) * x_scale. Scale and zero point are scalar +or per-axis vectors. FP8 E4M3FN inputs can be dequantized directly to BF16 with FP32 scales.)DOC"; ONNX_MS_OPERATOR_SET_SCHEMA(DequantizeLinear, 1, OpSchema() + .Attr("output_dtype", + "Optional output element type. When omitted, the scale " + "element type determines the output type.", + AttributeProto::INT, false) .Attr("axis", "The axis along which same quantization parameters are applied. It's optional." "If it's not specified, it means per-tensor quantization and input 'x_scale' and " @@ -199,25 +203,36 @@ ONNX_MS_OPERATOR_SET_SCHEMA(DequantizeLinear, 1, .Input(1, "x_scale", "Scale for input 'x'. It can be a scalar, which means a per-tensor/layer " "dequantization, or a 1-D tensor for per-axis dequantization.", - "T2") + "TS") .Input(2, "x_zero_point", "Zero point for input 'x'. Shape must match x_scale. It's optional. " "Zero point is 0 when it's not specified.", "T1", OpSchema::Optional) .Output(0, "y", "N-D full precision output tensor. It has same shape as input 'x'.", "T2") - .TypeConstraint("T1", {"tensor(int8)", "tensor(uint8)", "tensor(int16)", - "tensor(uint16)", "tensor(int32)", "tensor(int4)", - "tensor(uint4)"}, + .TypeConstraint("T1", {"tensor(int8)", "tensor(uint8)", "tensor(int16)", "tensor(uint16)", "tensor(int32)", "tensor(int4)", "tensor(uint4)", "tensor(float8e4m3fn)"}, "Constrain 'x' and 'x_zero_point' to 8-bit integer tensors, " - "16-bit integer tensors, or 32-bit signed integer tensors.") - .TypeConstraint("T2", {"tensor(float16)", "tensor(float)"}, - "Constrain 'y', 'x_scale' to float tensors.") + "16-bit integer tensors, 32-bit signed integer tensors, or " + "FP8 E4M3FN tensors.") + .TypeConstraint("T2", {"tensor(float16)", "tensor(float)", "tensor(bfloat16)"}, + "Constrain the output element type.") + .TypeConstraint("TS", {"tensor(float16)", "tensor(float)", "tensor(bfloat16)"}, + "Constrain the scale element type.") .SetDoc(DequantizeLinear_ver1_doc) .TypeAndShapeInferenceFunction([](ONNX_NAMESPACE::InferenceContext& ctx) { - auto y_type = ctx.getOutputType(0); - // only float is supported - y_type->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto::FLOAT); + const int64_t output_dtype = + getAttribute(ctx, "output_dtype", static_cast(0)); + if (output_dtype != 0) { + if (output_dtype != ONNX_NAMESPACE::TensorProto::FLOAT && + output_dtype != ONNX_NAMESPACE::TensorProto::FLOAT16 && + output_dtype != ONNX_NAMESPACE::TensorProto::BFLOAT16) { + fail_shape_inference( + "DequantizeLinear: output_dtype must be float, float16, or bfloat16"); + } + updateOutputElemType(ctx, 0, static_cast(output_dtype)); + } else if (ctx.getInputType(1) != nullptr) { + propagateElemTypeFromInputToOutput(ctx, 1, 0); + } if (!hasInputShape(ctx, 0)) return; @@ -852,16 +867,16 @@ ONNX_MS_OPERATOR_SET_SCHEMA( {"tensor(uint8)", "tensor(int8)"}, "Constrain input and output types to 8 bit signed and unsigned tensors.") .TypeAndShapeInferenceFunction([](InferenceContext& ctx) { - propagateElemTypeFromInputToOutput(ctx, 1, 0); - if (hasNInputShapes(ctx, 9)) { - std::vector shapes; - shapes.push_back(&ctx.getInputType(0)->tensor_type().shape()); - shapes.push_back(&ctx.getInputType(1)->tensor_type().shape()); - shapes.push_back(&ctx.getInputType(4)->tensor_type().shape()); - multidirectionalBroadcastShapeInference( - shapes, *ctx.getOutputType(0)->mutable_tensor_type()->mutable_shape()); - } - })); + propagateElemTypeFromInputToOutput(ctx, 1, 0); + if (hasNInputShapes(ctx, 9)) { + std::vector shapes; + shapes.push_back(&ctx.getInputType(0)->tensor_type().shape()); + shapes.push_back(&ctx.getInputType(1)->tensor_type().shape()); + shapes.push_back(&ctx.getInputType(4)->tensor_type().shape()); + multidirectionalBroadcastShapeInference( + shapes, *ctx.getOutputType(0)->mutable_tensor_type()->mutable_shape()); + } +})); ONNX_MS_OPERATOR_SET_SCHEMA( QGemm, 1, diff --git a/onnxruntime/test/contrib_ops/cuda_kernels/qmoe_fp8_compaction_test.cc b/onnxruntime/test/contrib_ops/cuda_kernels/qmoe_fp8_compaction_test.cc index c47c029a58168..3888cd665e71e 100644 --- a/onnxruntime/test/contrib_ops/cuda_kernels/qmoe_fp8_compaction_test.cc +++ b/onnxruntime/test/contrib_ops/cuda_kernels/qmoe_fp8_compaction_test.cc @@ -11,6 +11,9 @@ #include #include "contrib_ops/cuda/moe/qmoe_kernels.h" +#ifdef ORT_TEST_NVFP4_GEMV +#include "contrib_ops/cuda/llm/moe_gemm/moe_gemv_fp4.h" +#endif #include "core/providers/cuda/cuda_common.h" #include "core/platform/env_var_utils.h" @@ -23,6 +26,9 @@ namespace test { namespace { namespace qmoe = contrib::cuda; +#ifdef ORT_TEST_NVFP4_GEMV +namespace gemv = llm::kernels::moe_gemv; +#endif struct CudaBuffer { ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(CudaBuffer); @@ -53,6 +59,242 @@ struct CudaBuffer { size_t bytes; }; +#ifdef ORT_TEST_NVFP4_GEMV +template +void TestRoutingMaps(int rows, int experts, int k, bool normalize, bool tied) { + std::vector logits(rows * experts); + for (int i = 0; i < rows * experts; ++i) { + logits[i] = T(tied ? 0.0f : float((i * 73 + i / experts * 11) % 101 - 50) / 16.0f); + } + const int expanded = rows * k; + CudaBuffer input(logits.size() * sizeof(T)); + CudaBuffer scales(expanded * sizeof(float)), indices(expanded * sizeof(int)); + CudaBuffer r2u(expanded * sizeof(int)), u2r(expanded * sizeof(int)), sorted(expanded * sizeof(int)); + CudaBuffer offsets((experts + 1) * sizeof(int64_t)); + input.Upload(logits.data()); + qmoe::LaunchSoftmaxTopK(input.As(), scales.As(), indices.As(), + rows, experts, k, normalize, nullptr); + ASSERT_TRUE(qmoe::TryLaunchSmallDecodeExpertMaps( + indices.As(), r2u.As(), u2r.As(), sorted.As(), + offsets.As(), rows, experts, k, nullptr)); + std::vector expected_indices(expanded), actual_r2u(expanded), + actual_u2r(expanded), actual_sorted(expanded); + std::vector actual_offsets(experts + 1); + indices.Download(expected_indices.data()); + r2u.Download(actual_r2u.data()); + u2r.Download(actual_u2r.data()); + sorted.Download(actual_sorted.data()); + offsets.Download(actual_offsets.data()); + std::vector order(expanded); + for (int i = 0; i < expanded; ++i) order[i] = i; + std::stable_sort(order.begin(), order.end(), [&](int a, int b) { + return expected_indices[a] < expected_indices[b]; + }); + for (int rank = 0; rank < expanded; ++rank) { + const int original = order[rank]; + const int source = (original % k) * rows + original / k; + EXPECT_EQ(actual_r2u[rank], source); + EXPECT_EQ(actual_u2r[source], rank); + EXPECT_EQ(actual_sorted[rank], expected_indices[original]); + } + for (int expert = 0; expert <= experts; ++expert) { + EXPECT_EQ(actual_offsets[expert], + std::count_if(expected_indices.begin(), expected_indices.end(), + [&](int selected) { return selected < expert; })); + } +} + +TEST(CUDA_EP_Unittest, QMoENvfp4SmallDecodeMapsPreserveStableOrder) { + for (int rows : {1, 2}) { + for (int experts : {16, 257, 512}) { + for (int k : {1, 10, 16}) { + for (bool normalize : {false, true}) { + for (bool tied : {false, true}) { + TestRoutingMaps(rows, experts, k, normalize, tied); + TestRoutingMaps<__nv_bfloat16>(rows, experts, k, normalize, tied); + } + } + } + } + } + EXPECT_FALSE(qmoe::TryLaunchSmallDecodeExpertMaps( + nullptr, nullptr, nullptr, nullptr, nullptr, 3, 512, 10, nullptr)); + EXPECT_FALSE(qmoe::TryLaunchSmallDecodeExpertMaps( + nullptr, nullptr, nullptr, nullptr, nullptr, 2, 513, 10, nullptr)); + EXPECT_FALSE(qmoe::TryLaunchSmallDecodeExpertMaps( + nullptr, nullptr, nullptr, nullptr, nullptr, 0, 512, 10, nullptr)); + EXPECT_FALSE(qmoe::TryLaunchSmallDecodeExpertMaps( + nullptr, nullptr, nullptr, nullptr, nullptr, 2, 512, 17, nullptr)); + EXPECT_FALSE(qmoe::TryLaunchSmallDecodeExpertMaps( + nullptr, nullptr, nullptr, nullptr, nullptr, 2, 8, 10, nullptr)); +} + +template +void TestNvfp4KPackedGateUp(int k, bool use_bias, bool varied_scales = false, int n = 48) { + constexpr int experts = 3; + constexpr int rows = 4; + constexpr std::array routes{2, 0, 2, 1}; + constexpr std::array source_rows{1, 0, 0, 1}; + constexpr std::array globals{0.25f, 0.125f, 0.5f}; + constexpr std::array magnitudes{0, 0.5f, 1, 1.5f, 2, 3, 4, 6}; + std::vector activations(2 * k), bias(experts * n); + std::vector weights(experts * n * k / 2), scales(experts * n * k / 16); + for (size_t i = 0; i < activations.size(); ++i) { + activations[i] = T(float(int(i % 11) - 5) / 64.0f); + } + for (size_t i = 0; i < weights.size(); ++i) weights[i] = static_cast(i * 37 + 19); + for (size_t i = 0; i < scales.size(); ++i) { + scales[i] = varied_scales ? static_cast(i % 127) : (i % 2 ? 0x38 : 0x30); + } + for (size_t i = 0; i < bias.size(); ++i) bias[i] = T(float(int(i % 7) - 3) / 16.0f); + CudaBuffer a(activations.size() * sizeof(T)), w(weights.size()), s(scales.size()); + CudaBuffer g(sizeof(globals)), b(bias.size() * sizeof(T)), r(sizeof(routes)), sources(sizeof(source_rows)); + CudaBuffer output(rows * n / 2 * sizeof(T)); + a.Upload(activations.data()); + w.Upload(weights.data()); + s.Upload(scales.data()); + g.Upload(globals.data()); + b.Upload(bias.data()); + r.Upload(routes.data()); + sources.Upload(source_rows.data()); + llm::kernels::cutlass_kernels::ActivationParams activation; + activation.alpha = 1.0f; + activation.beta = 0.25f; + activation.limit = 7.0f; + gemv::launch_moe_gemv_fp4_symmetric_interleaved_swiglu( + a.As(), w.As(), nullptr, s.As(), g.As(), + use_bias ? b.As() : nullptr, output.As(), nullptr, r.As(), + experts, rows, n / 2, k, 16, 90, activation, gemv::MoeGemvConfig::kDefault, + false, true, sources.As(), 2, nullptr, true); + CUDA_CALL_THROW(cudaGetLastError()); + std::vector actual(rows * n / 2); + output.Download(actual.data()); + for (int row = 0; row < rows; ++row) { + for (int col = 0; col < n; col += 2) { + float sums[2]{}; + for (int pair = 0; pair < 2; ++pair) { + const int column = col + pair; + for (int d = 0; d < k; ++d) { + const uint8_t packed = weights[(routes[row] * n + column) * (k / 2) + d / 2]; + const int code = (d % 2 ? packed >> 4 : packed) & 15; + const uint8_t scale_code = scales[(routes[row] * n + column) * (k / 16) + d / 16]; + const int exponent = scale_code >> 3; + const int mantissa = scale_code & 7; + const float block_scale = exponent == 0 ? std::ldexp(float(mantissa), -9) + : std::ldexp(1.0f + mantissa * 0.125f, exponent - 7); + const float rounded_scale = static_cast(T(block_scale * globals[routes[row]])); + const float weight = static_cast(T(magnitudes[code & 7] * (code & 8 ? -1 : 1) * rounded_scale)); + sums[pair] += weight * static_cast(activations[source_rows[row] * k + d]); + } + if (use_bias) sums[pair] += static_cast(bias[routes[row] * n + column]); + } + const float gate = std::min(sums[0], activation.limit); + const float up = std::clamp(sums[1], -activation.limit, activation.limit) + activation.beta; + const float expected = static_cast(T(gate / (1.0f + std::exp(-gate)) * up)); + EXPECT_NEAR(static_cast(actual[row * n / 2 + col / 2]), expected, + 0.001f + std::abs(expected) * 0.01f) + << "row=" << row << ", column=" << col << ", K=" << k << ", bias=" << use_bias; + } + } +} + +TEST(CUDA_EP_Unittest, QMoENvfp4KPackedGateUpTiling) { + for (int k : {144, 2560}) { + for (bool use_bias : {false, true}) { + TestNvfp4KPackedGateUp(k, use_bias); + TestNvfp4KPackedGateUp<__nv_bfloat16>(k, use_bias); + } + } +} + +TEST(CUDA_EP_Unittest, QMoENvfp4KPackedGateUpScaleRange) { + TestNvfp4KPackedGateUp(2560, true, true); + TestNvfp4KPackedGateUp<__nv_bfloat16>(2560, true, true); +} + +TEST(CUDA_EP_Unittest, QMoENvfp4KPackedGateUpPartialTiles) { + for (int n : {2, 10}) { + for (int k : {16, 272}) { + for (bool use_bias : {false, true}) { + TestNvfp4KPackedGateUp(k, use_bias, false, n); + TestNvfp4KPackedGateUp<__nv_bfloat16>(k, use_bias, false, n); + } + } + } +} + +template +void TestNvfp4FusedFinalize(int num_rows, int topk, int k, bool use_bias) { + constexpr int n = 50, experts = 3; + const int expanded = num_rows * topk; + std::vector activations(expanded * k), biases(experts * n); + std::vector weights(experts * n * k / 2), scales(experts * n * k / 16, 0x30); + std::vector globals{0.3f, 0.17f, 0.51f}, routing(expanded); + std::vector mapping(expanded), expert_ids(expanded); + for (size_t i = 0; i < activations.size(); ++i) activations[i] = T(float(int(i % 11) - 5) / 64.0f); + for (size_t i = 0; i < weights.size(); ++i) weights[i] = static_cast(i * 37 + 19); + for (size_t i = 0; i < biases.size(); ++i) biases[i] = T(float(int(i % 7) - 3) / 16.0f); + for (int i = 0; i < expanded; ++i) { + mapping[i] = (i * 7) % expanded; + expert_ids[i] = i % experts; + routing[i] = float(i % topk + 1) / float(topk * (topk + 1) / 2); + } + if (topk > 1) mapping[expanded - 1] = -1; + if (expanded > 1) expert_ids[expanded / 2] = experts; + CudaBuffer a(activations.size() * sizeof(T)), w(weights.size()), s(scales.size()); + CudaBuffer g(globals.size() * sizeof(float)), b(biases.size() * sizeof(T)); + CudaBuffer m(mapping.size() * sizeof(int)), e(expert_ids.size() * sizeof(int)), r(routing.size() * sizeof(float)); + CudaBuffer reference(expanded * n * sizeof(T)), output(num_rows * n * sizeof(T)); + a.Upload(activations.data()); + w.Upload(weights.data()); + s.Upload(scales.data()); + g.Upload(globals.data()); + b.Upload(biases.data()); + m.Upload(mapping.data()); + e.Upload(expert_ids.data()); + r.Upload(routing.data()); + gemv::launch_moe_gemv_fp4_symmetric( + a.As(), w.As(), nullptr, s.As(), g.As(), + use_bias ? b.As() : nullptr, reference.As(), nullptr, e.As(), + experts, expanded, n, k, 16, 90, gemv::MoeGemvConfig::kDefault, false, true, nullptr, true); + ASSERT_TRUE(gemv::try_launch_moe_gemv_nvfp4_fused_finalize( + a.As(), w.As(), s.As(), g.As(), + use_bias ? b.As() : nullptr, output.As(), m.As(), e.As(), r.As(), + experts, num_rows, topk, n, k, nullptr)); + CUDA_CALL_THROW(cudaGetLastError()); + std::vector reference_values(expanded * n), actual(num_rows * n); + reference.Download(reference_values.data()); + output.Download(actual.data()); + for (int row = 0; row < num_rows; ++row) { + for (int col = 0; col < n; ++col) { + float sum = 0.0f; + for (int rank = 0; rank < topk; ++rank) { + const int permuted = mapping[rank * num_rows + row]; + if (permuted >= 0 && expert_ids[permuted] < experts) { + sum += routing[row * topk + rank] * static_cast(reference_values[permuted * n + col]); + } + } + const float expected = static_cast(T(sum)); + EXPECT_NEAR(static_cast(actual[row * n + col]), expected, 1e-5f + std::abs(expected) * 0.01f) + << "row=" << row << ", col=" << col << ", topk=" << topk << ", bias=" << use_bias; + } + } +} + +TEST(CUDA_EP_Unittest, QMoENvfp4FusedFinalizePreservesRoutingAndRounding) { + for (int rows : {1, 2}) { + for (int topk : {1, 4, 10, 16}) { + for (int k : {144, 640}) { + for (bool use_bias : {false, true}) { + TestNvfp4FusedFinalize(rows, topk, k, use_bias); + TestNvfp4FusedFinalize<__nv_bfloat16>(rows, topk, k, use_bias); + } + } + } + } +} +#endif + TEST(CUDA_EP_Unittest, QMoEFp8CompactExpertsPreservesRoutesAndClearsUnusedSlots) { constexpr int num_experts = 512; constexpr int capacity = 6; diff --git a/onnxruntime/test/contrib_ops/dequantize_fp8_bf16_test.cc b/onnxruntime/test/contrib_ops/dequantize_fp8_bf16_test.cc new file mode 100644 index 0000000000000..7e932c799fd0a --- /dev/null +++ b/onnxruntime/test/contrib_ops/dequantize_fp8_bf16_test.cc @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "core/common/float8.h" +#include "test/common/tensor_op_test_utils.h" +#include "test/providers/provider_test_utils.h" +#include "test/util/include/default_providers.h" + +namespace onnxruntime { +namespace test { + +#if defined(USE_CUDA) && !defined(DISABLE_FLOAT8_TYPES) +TEST(DequantizeLinearTest, Fp8ToBf16) { + auto provider = DefaultCudaExecutionProvider(); + if (provider == nullptr) { + GTEST_SKIP() << "CUDA execution provider is not available"; + } + std::vector> providers; + providers.push_back(std::move(provider)); + + OpTester test("DequantizeLinear", 1, kMSDomain); + test.AddAttribute("axis", 1); + test.AddAttribute("output_dtype", ONNX_NAMESPACE::TensorProto_DataType_BFLOAT16); + test.AddInput("weight", {1, 8}, std::vector(8, Float8E4M3FN(1.0f))); + test.AddInput("scale", {8}, std::vector(8, 2.0f)); + test.AddOptionalInputEdge(); + test.AddOutput("weight_bf16", {1, 8}, ToBFloat16(std::vector(8, 2.0f))); + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &providers); +} +#endif + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/contrib_ops/matmul_block_scaled_fp8_test.cc b/onnxruntime/test/contrib_ops/matmul_block_scaled_fp8_test.cc index 9dd9b0e722a29..0695e717ea6eb 100644 --- a/onnxruntime/test/contrib_ops/matmul_block_scaled_fp8_test.cc +++ b/onnxruntime/test/contrib_ops/matmul_block_scaled_fp8_test.cc @@ -2,6 +2,7 @@ // Licensed under the MIT License. #include +#include #include "gtest/gtest.h" #include "test/common/cuda_op_test_utils.h" @@ -69,6 +70,121 @@ std::vector MakeConstRowWeight(const std::vector& row_value } } // namespace +template +void TestFp8SplitOutputs() { + if (!HasCudaEnvironment(800)) { + GTEST_SKIP() << "CUDA device is required for split-output FP8 matmul."; + } + for (const auto& [m, k] : {std::pair{0, 256}, {1, 256}, {2, 256}, {2, 320}, {2, 1088}, {9, 256}, {32, 256}, {33, 256}, {65, 256}, {2, 16}, {2, 30}, {2, 0}}) { + for (const bool activation_scale : {false, true}) { + SCOPED_TRACE("M=" + std::to_string(m) + " K=" + std::to_string(k) + + " W8A8=" + std::to_string(activation_scale)); + constexpr int64_t n = 17; + constexpr int64_t block_size = 64; + const int64_t k_blocks = (k + block_size - 1) / block_size; + const std::vector widths = m == 2 && k == 256 + ? std::vector{1, 1, 1, 1, 1, 1, 1, 10} + : (m == 33 ? std::vector{17} + : std::vector{1, 7, 9}); + std::vector a; + std::vector b; + std::vector scales; + std::vector bias; + for (int64_t row = 0; row < m; ++row) { + for (int64_t i = 0; i < k; ++i) { + a.emplace_back(static_cast((row + i) % 5 - 2) / 4.0f); + } + } + for (int64_t col = 0; col < n; ++col) { + bias.emplace_back(static_cast(col - 8) / 8.0f); + for (int64_t i = 0; i < k; ++i) { + b.emplace_back(static_cast((col + i) % 3 - 1)); + } + for (int64_t kb = 0; kb < k_blocks; ++kb) { + scales.push_back(static_cast(1 + (col + kb) % 2) / 4.0f); + } + } + TensorShapeVector a_shape = m == 1 ? TensorShapeVector{k} : TensorShapeVector{m, k}; + if (m == 9) { + a_shape = {3, 3, k}; + } + OpTester test("MatMulBlockQuantizedFp8Weight", 1, kMSDomain); + test.AddAttribute("block_size", block_size); + test.AddAttribute>("split_sizes", widths); + test.AddInput("A", a_shape, a); + test.AddInput("B", {n, k}, b); + test.AddInput("b_scale", {n, k_blocks}, scales); + if (activation_scale) { + test.AddInput("a_scale", {}, {0.5f}); + } else { + test.AddOptionalInputEdge(); + } + if (activation_scale) { + test.AddInput("bias", {n}, bias); + } + int64_t start = 0; + for (size_t output = 0; output < widths.size(); ++output) { + std::vector expected; + for (int64_t row = 0; row < m; ++row) { + for (int64_t col = start; col < start + widths[output]; ++col) { + float sum = activation_scale ? static_cast(bias[col]) : 0.0f; + for (int64_t i = 0; i < k; ++i) { + sum += static_cast(a[row * k + i]) * b[col * k + i].ToFloat() * + scales[col * k_blocks + i / block_size]; + } + expected.emplace_back(sum); + } + } + auto shape = a_shape; + shape.back() = widths[output]; + test.AddOutput(("Y" + std::to_string(output)).c_str(), shape, expected); + start += widths[output]; + } + test.SetOutputTolerance(0.0f, 0.0f); + std::vector> providers; + providers.push_back(DefaultCudaExecutionProvider()); + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &providers); + } + } +} + +TEST(MatMulBlockQuantizedFp8WeightOpTest, SplitOutputsFp16) { + TestFp8SplitOutputs(); +} + +TEST(MatMulBlockQuantizedFp8WeightOpTest, SplitOutputsBFloat16) { + TestFp8SplitOutputs(); +} + +TEST(MatMulBlockQuantizedFp8WeightOpTest, RejectsInvalidSplitSizes) { + if (!HasCudaEnvironment(800)) { + GTEST_SKIP() << "CUDA device is required for split-output FP8 matmul."; + } + for (const std::vector& widths : {std::vector{}, {0, 17}, {-1, 18}, {8, 8}, {1, 1, 1, 1, 1, 1, 1, 1, 9}}) { + OpTester test("MatMulBlockQuantizedFp8Weight", 1, kMSDomain); + test.AddAttribute>("split_sizes", widths); + test.AddInput("A", {1, 16}, std::vector(16)); + test.AddInput("B", {17, 16}, std::vector(17 * 16)); + test.AddInput("b_scale", {17, 1}, std::vector(17, 1.0f)); + for (size_t i = 0; i < std::max(1, widths.size()); ++i) { + const int64_t width = i < widths.size() ? std::max(0, widths[i]) : 17; + test.AddOutput(("Y" + std::to_string(i)).c_str(), {1, width}, std::vector(width)); + } + std::vector> providers; + providers.push_back(DefaultCudaExecutionProvider()); + test.Run(OpTester::ExpectResult::kExpectFailure, "split_sizes", {}, nullptr, &providers); + } + OpTester count_test("MatMulBlockQuantizedFp8Weight", 1, kMSDomain); + count_test.AddAttribute>("split_sizes", {8, 9}); + count_test.AddInput("A", {1, 16}, std::vector(16)); + count_test.AddInput("B", {17, 16}, std::vector(17 * 16)); + count_test.AddInput("b_scale", {17, 1}, std::vector(17, 1.0f)); + count_test.AddOutput("Y", {1, 17}, std::vector(17)); + std::vector> providers; + providers.push_back(DefaultCudaExecutionProvider()); + count_test.Run(OpTester::ExpectResult::kExpectFailure, "split_sizes", {}, nullptr, &providers); +} + TEST(MatMulBlockQuantizedFp8WeightOpTest, GemvTensorCoreKSplitSelection) { struct Case { int n; diff --git a/onnxruntime/test/contrib_ops/moe_test.cc b/onnxruntime/test/contrib_ops/moe_test.cc index 84fd88c061bb2..8864069727f84 100644 --- a/onnxruntime/test/contrib_ops/moe_test.cc +++ b/onnxruntime/test/contrib_ops/moe_test.cc @@ -2620,8 +2620,8 @@ template static void RunQMoEPackedPrefillRoutingTest(int64_t num_rows, int64_t hidden_size, int64_t inter_size, int64_t top_k, bool skewed, bool scale_initializers = true, int64_t row_tile_size = 0, int64_t block_size = 64, - int64_t fc1_bits = 2, int64_t fc2_bits = 4) { - constexpr int64_t num_experts = 8; + int64_t fc1_bits = 2, int64_t fc2_bits = 4, + int64_t num_experts = 8) { constexpr float alpha = 1.5f; constexpr float beta = 0.125f; constexpr float limit = 0.5f; @@ -2634,7 +2634,7 @@ static void RunQMoEPackedPrefillRoutingTest(int64_t num_rows, int64_t hidden_siz return 0.125f * static_cast(expert + 1) + 0.015625f * static_cast(column % 5) + 0.03125f * static_cast(block); }; - const auto make_weights = [](int64_t rows, int64_t columns, int64_t bits, bool interleaved) { + const auto make_weights = [num_experts](int64_t rows, int64_t columns, int64_t bits, bool interleaved) { const int64_t pack_size = 8 / bits; const uint8_t zero = static_cast(1u << (bits - 1)); std::vector weights(static_cast(num_experts * rows * columns / pack_size), 0); @@ -2762,6 +2762,7 @@ TEST(MoETest, QMoETest_MixedWidthCudaPackedPrefillRouting) { if (!HasCudaEnvironment(800)) { GTEST_SKIP() << "SM80 or later is required for packed INT2 prefill."; } + ScopedEnvironmentVariables scoped_env_vars{ {{"ORT_DISABLE_MOE_GEMV", "1"}, {"ORT_ENABLE_QMOE_INT2_PREFILL", "1"}}}; for (int64_t top_k : {1, 2, 4}) { @@ -2775,6 +2776,20 @@ TEST(MoETest, QMoETest_MixedWidthCudaPackedPrefillRouting) { } } +TEST(MoETest, QMoETest_CudaColumnTiledFinalize) { + if (!HasCudaEnvironment(800)) { + GTEST_SKIP() << "SM80 or later is required for packed INT2 decode."; + } + for (int64_t rows : {1, 2}) { + for (int64_t hidden : {2560, 2624}) { + RunQMoEPackedPrefillRoutingTest(rows, hidden, 128, 10, false, true, 0, 64, 2, 4, 16); +#if defined(ENABLE_BF16) + RunQMoEPackedPrefillRoutingTest(rows, hidden, 128, 10, false, true, 0, 64, 2, 4, 16); +#endif + } + } +} + TEST(MoETest, QMoETest_MixedWidthCudaPackedPrefillRuntimeScales) { if (!HasCudaEnvironment(800)) { GTEST_SKIP() << "SM80 or later is required for packed INT2 prefill."; From bf470aa2dd158bb35b0aedc8660447f901bdeff6 Mon Sep 17 00:00:00 2001 From: Akshay Sonawane Date: Mon, 19 Oct 2026 21:24:56 +0000 Subject: [PATCH 02/12] Fix quantized MoE review coverage and FP8 dequantization CI --- cmake/onnxruntime_unittests.cmake | 8 +- docs/ContribOperators.md | 33 +- docs/OperatorKernels.md | 6 +- .../cuda/quantization/dequantize_fp8_bf16.cu | 34 + .../cuda/quantization/dequantize_fp8_bf16.h | 18 + .../quantize_dequantize_linear.cc | 60 ++ .../kernel_type_str_resolver_utils.cc | 719 +++++++++--------- .../contrib_ops/dequantize_fp8_bf16_test.cc | 65 ++ 8 files changed, 567 insertions(+), 376 deletions(-) create mode 100644 onnxruntime/contrib_ops/cuda/quantization/dequantize_fp8_bf16.cu create mode 100644 onnxruntime/contrib_ops/cuda/quantization/dequantize_fp8_bf16.h diff --git a/cmake/onnxruntime_unittests.cmake b/cmake/onnxruntime_unittests.cmake index b76a9dc55e417..34fd56b75907b 100644 --- a/cmake/onnxruntime_unittests.cmake +++ b/cmake/onnxruntime_unittests.cmake @@ -1028,6 +1028,12 @@ set(all_tests ) set(onnxruntime_test_providers_runtime_dependencies) +if (onnxruntime_ENABLE_CUDA_EP_INTERNAL_TESTS AND onnxruntime_USE_FP4_QMOE AND + NOT onnxruntime_DISABLE_CONTRIB_OPS) + set_property(SOURCE "${TEST_SRC_DIR}/contrib_ops/cuda_kernels/qmoe_fp8_compaction_test.cc" + APPEND PROPERTY COMPILE_DEFINITIONS ORT_TEST_NVFP4_GEMV) +endif() + if (onnxruntime_ENABLE_CUDA_EP_INTERNAL_TESTS AND NOT onnxruntime_BUILD_CUDA_EP_AS_PLUGIN) if (NOT onnxruntime_MINIMAL_BUILD AND NOT onnxruntime_REDUCED_OPS_BUILD) set(onnxruntime_test_cuda_kernels_src_patterns "${TEST_SRC_DIR}/contrib_ops/cuda_kernels/*.cc") @@ -1163,8 +1169,6 @@ if (onnxruntime_ENABLE_CUDA_EP_INTERNAL_TESTS AND onnxruntime_BUILD_CUDA_EP_AS_P if (onnxruntime_USE_FP4_QMOE) list(APPEND onnxruntime_providers_cuda_plugin_ut_impl_src "${ONNXRUNTIME_ROOT}/contrib_ops/cuda/llm/moe_gemm/moe_gemv_fp4.cu") - set_property(SOURCE "${TEST_SRC_DIR}/contrib_ops/cuda_kernels/qmoe_fp8_compaction_test.cc" - APPEND PROPERTY COMPILE_DEFINITIONS ORT_TEST_NVFP4_GEMV) endif() endif() diff --git a/docs/ContribOperators.md b/docs/ContribOperators.md index 3510b52792635..1f42a009eb5b4 100644 --- a/docs/ContribOperators.md +++ b/docs/ContribOperators.md @@ -1495,9 +1495,9 @@ This version of the operator has been available since version 1 of the 'com.micr ### **com.microsoft.DequantizeLinear** - The linear dequantization operator. It consumes a quantized data, a scale, a zero point and computes the full precision data. - The dequantization formula is y = (x - x_zero_point) * x_scale. - Scale and zero point must have same shape. They must be either scalar (per tensor) or 1-D tensor (per 'axis'). + The linear dequantization operator. It consumes quantized data, a scale, and an optional zero point. + The dequantization formula is y = (x - x_zero_point) * x_scale. Scale and zero point are scalar + or per-axis vectors. FP8 E4M3FN inputs can be dequantized directly to BF16 with FP32 scales. #### Version @@ -1508,6 +1508,8 @@ This version of the operator has been available since version 1 of the 'com.micr
axis : int
The axis along which same quantization parameters are applied. It's optional.If it's not specified, it means per-tensor quantization and input 'x_scale' and 'x_zero_point' must be scalars.If it's specified, it means per 'axis' quantization and input 'x_scale' and 'x_zero_point' must be 1-D tensors.
+
output_dtype : int
+
Optional output element type. When omitted, the scale element type determines the output type.
#### Inputs (2 - 3) @@ -1515,7 +1517,7 @@ This version of the operator has been available since version 1 of the 'com.micr
x : T1
N-D quantized Input tensor to be de-quantized.
-
x_scale : T2
+
x_scale : TS
Scale for input 'x'. It can be a scalar, which means a per-tensor/layer dequantization, or a 1-D tensor for per-axis dequantization.
x_zero_point (optional) : T1
Zero point for input 'x'. Shape must match x_scale. It's optional. Zero point is 0 when it's not specified.
@@ -1531,10 +1533,12 @@ This version of the operator has been available since version 1 of the 'com.micr #### Type Constraints
-
T1 : tensor(int8), tensor(uint8), tensor(int16), tensor(uint16), tensor(int32), tensor(int4), tensor(uint4)
-
Constrain 'x' and 'x_zero_point' to 8-bit integer tensors, 16-bit integer tensors, or 32-bit signed integer tensors.
-
T2 : tensor(float16), tensor(float)
-
Constrain 'y', 'x_scale' to float tensors.
+
T1 : tensor(int8), tensor(uint8), tensor(int16), tensor(uint16), tensor(int32), tensor(int4), tensor(uint4), tensor(float8e4m3fn)
+
Constrain 'x' and 'x_zero_point' to 8-bit integer tensors, 16-bit integer tensors, 32-bit signed integer tensors, or FP8 E4M3FN tensors.
+
T2 : tensor(float16), tensor(float), tensor(bfloat16)
+
Constrain the output element type.
+
TS : tensor(float16), tensor(float), tensor(bfloat16)
+
Constrain the scale element type.
@@ -3840,7 +3844,9 @@ This version of the operator has been available since version 1 of the 'com.micr precision (weight-only W8A16). The operator multiplies the activation by the transpose of B_scaled and adds the optional bias. - The output has shape [..., N] and the same element type as A. + The output has shape [..., N] and the same element type as A. When split_sizes is provided, + return up to eight consecutive last-dimension partitions instead, equivalent to Split(Y, split_sizes). + The partition widths must be positive and sum to N. #### Version @@ -3851,6 +3857,8 @@ This version of the operator has been available since version 1 of the 'com.micr
block_size : int
Number of consecutive K values that share one weight scale. Default 128.
+
split_sizes : list of ints
+
Optional last-dimension output widths (one to eight positive sizes).
#### Inputs (3 - 5) @@ -3868,11 +3876,11 @@ This version of the operator has been available since version 1 of the 'com.micr
Optional bias of shape [N].
-#### Outputs +#### Outputs (1 - ∞)
-
Y : T
-
Output of shape [..., N] in the activation type.
+
Y (variadic) : T
+
Output(s) of shape [..., N] or [..., split_sizes[i]] in the activation type.
#### Type Constraints @@ -8771,3 +8779,4 @@ No versioning maintained for experimental ops.
T : tensor(float)
Constrain input and output types to float32 tensors.
+ diff --git a/docs/OperatorKernels.md b/docs/OperatorKernels.md index 669e3083f6b15..9bc7548677773 100644 --- a/docs/OperatorKernels.md +++ b/docs/OperatorKernels.md @@ -581,7 +581,7 @@ The **OpSet Version** column uses the following notation: |ConvTransposeWithDynamicPads|*in* X:**T**
*in* W:**T**
*in* Pads:**tensor(int64)**
*in* B:**T**
*out* Y:**T**|1+|**T** = tensor(float)| |CropAndResize|*in* X:**T1**
*in* rois:**T1**
*in* batch_indices:**T2**
*in* crop_size:**T2**
*out* Y:**T1**|1+|**T1** = tensor(float)
**T2** = tensor(int32)| |DecoderMaskedMultiHeadAttention|*in* query:**T**
*in* key:**T**
*in* value:**T**
*in* mask_index:**M**
*in* attention_bias:**T**
*in* past_key:**T**
*in* past_value:**T**
*in* past_sequence_length:**M**
*in* beam_width:**M**
*in* cache_indirection:**M**
*in* bias:**T**
*out* output:**T**
*out* present_key:**T**
*out* present_value:**T**
*out* qk:**QK**|1+|**T** = tensor(float)| -|DequantizeLinear|*in* x:**T1**
*in* x_scale:**T2**
*in* x_zero_point:**T1**
*out* y:**T2**|1+|**T1** = tensor(int16), tensor(int32), tensor(int4), tensor(int8), tensor(uint16), tensor(uint4), tensor(uint8)
**T2** = tensor(float)| +|DequantizeLinear|*in* x:**T1**
*in* x_scale:**TS**
*in* x_zero_point:**T1**
*out* y:**T2**|1+|**T1** = tensor(int16), tensor(int32), tensor(int4), tensor(int8), tensor(uint16), tensor(uint4), tensor(uint8)
**T2** = tensor(float)| |DynamicQuantizeLSTM|*in* X:**T**
*in* W:**T2**
*in* R:**T2**
*in* B:**T**
*in* sequence_lens:**T1**
*in* initial_h:**T**
*in* initial_c:**T**
*in* P:**T**
*in* W_scale:**T**
*in* W_zero_point:**T2**
*in* R_scale:**T**
*in* R_zero_point:**T2**
*out* Y:**T**
*out* Y_h:**T**
*out* Y_c:**T**|1+|**T** = tensor(float)
**T1** = tensor(int32)
**T2** = tensor(int8), tensor(uint8)| |DynamicQuantizeMatMul|*in* A:**T1**
*in* B:**T2**
*in* b_scale:**T1**
*in* b_zero_point:**T2**
*in* bias:**T1**
*out* Y:**T1**|1+|**T1** = tensor(float)
**T2** = tensor(int8), tensor(uint8)| |DynamicTimeWarping|*in* input:**F**
*out* output:**I**|1+|**F** = tensor(float)
**I** = tensor(int32)| @@ -1096,7 +1096,7 @@ The **OpSet Version** column uses the following notation: |DecoderAttention|*in* query:**T**
*in* key:**T**
*in* q_weight:**T**
*in* kv_weight:**T**
*in* bias:**T**
*in* key_padding_mask:**B**
*in* key_cache:**T**
*in* value_cache:**T**
*in* static_kv:**B**
*in* use_past:**B**
*in* has_layer_state:**B**
*in* has_key_padding_mask:**B**
*out* output:**T**
*out* new_key_cache:**T**
*out* new_value_cache:**T**|1+|**T** = tensor(float), tensor(float16)| |DecoderMaskedMultiHeadAttention|*in* query:**T**
*in* key:**T**
*in* value:**T**
*in* mask_index:**M**
*in* attention_bias:**T**
*in* past_key:**T**
*in* past_value:**T**
*in* past_sequence_length:**M**
*in* beam_width:**M**
*in* cache_indirection:**M**
*in* bias:**T**
*out* output:**T**
*out* present_key:**T**
*out* present_value:**T**
*out* qk:**QK**|1+|**QK** = tensor(float), tensor(float16)
**T** = tensor(float), tensor(float16)| |DecoderMaskedSelfAttention|*in* input:**T**
*in* weights:**T**
*in* bias:**T**
*in* mask_index:**M**
*in* past:**T**
*in* attention_bias:**T**
*in* past_sequence_length:**M**
*in* beam_width:**M**
*in* cache_indirection:**M**
*out* output:**T**
*out* present:**T**|1+|**T** = tensor(float), tensor(float16)| -|DequantizeLinear|*in* x:**T1**
*in* x_scale:**T2**
*in* x_zero_point:**T1**
*out* y:**T2**|1+|**T1** = tensor(int8), tensor(uint8)
**T2** = tensor(float16)| +|DequantizeLinear|*in* x:**T1**
*in* x_scale:**TS**
*in* x_zero_point:**T1**
*out* y:**T2**|1+|**T1** = tensor(float8e4m3fn), tensor(int8), tensor(uint8)
**T2** = tensor(bfloat16), tensor(float16)
**TS** = tensor(float), tensor(float16)| |DequantizeWithOrder|*in* input:**Q**
*in* scale_input:**S**
*out* output:**F**|1+|**F** = tensor(float), tensor(float16)
**Q** = tensor(int8)
**S** = tensor(float)| |DynamicSparseAttention|*in* query:**T**
*in* key:**T**
*in* value:**T**
*in* past_key:**T**
*in* past_value:**T**
*in* auxiliary_key:**T**
*in* auxiliary_value:**T**
*in* selected_indices:**M**
*in* selected_counts:**M**
*in* seqlens_k:**M**
*in* total_sequence_length:**M**
*in* cos_cache:**T**
*in* sin_cache:**T**
*in* position_ids:**tensor(int64)**
*in* q_norm_weight:**T**
*in* k_norm_weight:**T**
*in* head_sink:**T**
*out* output:**T**
*out* present_key:**T**
*out* present_value:**T**|1+|**M** = tensor(int32)
**T** = tensor(bfloat16), tensor(float), tensor(float16)| |DynamicTimeWarping|*in* input:**F**
*out* output:**I**|1+|**F** = tensor(float)
**I** = tensor(int32)| @@ -1611,7 +1611,7 @@ The **OpSet Version** column uses the following notation: |BiasGelu|*in* A:**T**
*in* B:**T**
*out* C:**T**|1+|**T** = tensor(float), tensor(float16)| |BiasSplitGelu|*in* X:**T**
*in* bias:**T**
*out* Y:**T**|1+|**T** = tensor(float), tensor(float16)| |ConvTransposeWithDynamicPads|*in* X:**T**
*in* W:**T**
*in* Pads:**tensor(int64)**
*in* B:**T**
*out* Y:**T**|1+|**T** = tensor(float), tensor(float16)| -|DequantizeLinear|*in* x:**T1**
*in* x_scale:**T2**
*in* x_zero_point:**T1**
*out* y:**T2**|1+|**T1** = tensor(int32), tensor(int8), tensor(uint8)
**T2** = tensor(float), tensor(float16)| +|DequantizeLinear|*in* x:**T1**
*in* x_scale:**TS**
*in* x_zero_point:**T1**
*out* y:**T2**|1+|**T1** = tensor(int32), tensor(int8), tensor(uint8)
**T2** = tensor(float), tensor(float16)| |DynamicQuantizeMatMul|*in* A:**T1**
*in* B:**T2**
*in* b_scale:**T1**
*in* b_zero_point:**T2**
*in* bias:**T1**
*out* Y:**T1**|1+|**T1** = tensor(float)
**T2** = tensor(int8), tensor(uint8)| |EmbedLayerNormalization|*in* input_ids:**T1**
*in* segment_ids:**T1**
*in* word_embedding:**T**
*in* position_embedding:**T**
*in* segment_embedding:**T**
*in* gamma:**T**
*in* beta:**T**
*in* mask:**T1**
*in* position_ids:**T1**
*out* output:**T**
*out* mask_index:**T1**
*out* embedding_sum:**T**|1+|**T** = tensor(float), tensor(float16)| |FastGelu|*in* X:**T**
*in* bias:**T**
*out* Y:**T**|1+|**T** = tensor(float), tensor(float16)| diff --git a/onnxruntime/contrib_ops/cuda/quantization/dequantize_fp8_bf16.cu b/onnxruntime/contrib_ops/cuda/quantization/dequantize_fp8_bf16.cu new file mode 100644 index 0000000000000..8e5ebb516ed0d --- /dev/null +++ b/onnxruntime/contrib_ops/cuda/quantization/dequantize_fp8_bf16.cu @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "dequantize_fp8_bf16.h" + +#if !defined(DISABLE_FLOAT8_TYPES) +namespace onnxruntime::contrib::cuda { +namespace { + +__global__ void DequantizeFp8Bf16Kernel(const Float8E4M3FN* input, const float* scale, + const Float8E4M3FN* zero_point, BFloat16* output, + size_t count, size_t axis_stride, size_t scale_count) { + const size_t index = static_cast(blockIdx.x) * blockDim.x + threadIdx.x; + if (index < count) { + const size_t scale_index = (index / axis_stride) % scale_count; + const float zero = zero_point == nullptr ? 0.0f : zero_point[scale_index].ToFloat(); + output[index] = BFloat16((input[index].ToFloat() - zero) * scale[scale_index]); + } +} + +} // namespace + +void DequantizeFp8Bf16(cudaStream_t stream, const Float8E4M3FN* input, const float* scale, + const Float8E4M3FN* zero_point, BFloat16* output, size_t count, + size_t axis_stride, size_t scale_count) { + constexpr int threads = 256; + const int blocks = static_cast((count + threads - 1) / threads); + DequantizeFp8Bf16Kernel<<>>( + input, scale, zero_point, output, + count, axis_stride, scale_count); +} + +} // namespace onnxruntime::contrib::cuda +#endif diff --git a/onnxruntime/contrib_ops/cuda/quantization/dequantize_fp8_bf16.h b/onnxruntime/contrib_ops/cuda/quantization/dequantize_fp8_bf16.h new file mode 100644 index 0000000000000..be21387b35228 --- /dev/null +++ b/onnxruntime/contrib_ops/cuda/quantization/dequantize_fp8_bf16.h @@ -0,0 +1,18 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/providers/cuda/cuda_common.h" + +#if !defined(DISABLE_FLOAT8_TYPES) +#include "core/common/float8.h" + +namespace onnxruntime::contrib::cuda { + +void DequantizeFp8Bf16(cudaStream_t stream, const Float8E4M3FN* input, const float* scale, + const Float8E4M3FN* zero_point, BFloat16* output, size_t count, + size_t axis_stride, size_t scale_count); + +} // namespace onnxruntime::contrib::cuda +#endif diff --git a/onnxruntime/contrib_ops/cuda/quantization/quantize_dequantize_linear.cc b/onnxruntime/contrib_ops/cuda/quantization/quantize_dequantize_linear.cc index 40f54aba301cf..af94287eb9f50 100644 --- a/onnxruntime/contrib_ops/cuda/quantization/quantize_dequantize_linear.cc +++ b/onnxruntime/contrib_ops/cuda/quantization/quantize_dequantize_linear.cc @@ -3,6 +3,8 @@ #include "core/providers/cuda/cuda_common.h" #include "core/providers/cuda/tensor/quantize_linear.h" +#include "core/providers/cuda/tensor/quantize_linear_common.h" +#include "dequantize_fp8_bf16.h" namespace onnxruntime { namespace contrib { @@ -41,6 +43,64 @@ REGISTER_Q_KERNEL_TYPED(uint8_t, MLFloat16) REGISTER_DQ_KERNEL_TYPED(int8_t, MLFloat16) REGISTER_DQ_KERNEL_TYPED(uint8_t, MLFloat16) +#if !defined(DISABLE_FLOAT8_TYPES) +class DequantizeLinearFp8Bf16 final : public CudaKernel { + public: + ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(DequantizeLinearFp8Bf16); + explicit DequantizeLinearFp8Bf16(const OpKernelInfo& info) : CudaKernel(info) { + axis_ = info.GetAttrOrDefault("axis", 1); + } + + Status ComputeInternal(OpKernelContext* ctx) const override { + const auto& input = *ctx->Input(0); + const auto& scale = *ctx->Input(1); + const auto* zero_point = ctx->Input(2); + const auto& shape = input.Shape(); + const int64_t count = shape.Size(); + ORT_RETURN_IF_NOT(IsQDQElementCountSupported(count), QDQElementCountErrorMessage()); + ORT_RETURN_IF_NOT(scale.Shape().NumDimensions() == 0 || scale.Shape().NumDimensions() == 1, + "x_scale must be a scalar or 1D tensor."); + ORT_RETURN_IF_NOT(zero_point == nullptr || zero_point->Shape() == scale.Shape(), + "x_zero_point and x_scale must have the same shape."); + + const int64_t scale_count = scale.Shape().Size(); + ORT_RETURN_IF_NOT(scale_count > 0, "x_scale must not be empty."); + int64_t axis_stride = 1; + if (scale_count != 1) { + const int64_t axis = HandleNegativeAxis(axis_, shape.NumDimensions()); + ORT_RETURN_IF_NOT(scale_count == shape[axis], + "x_scale must match the input dimension along axis."); + axis_stride = shape.SizeFromDimension(axis + 1); + } + + auto* output = ctx->Output(0, shape); + if (count == 0) { + return Status::OK(); + } + DequantizeFp8Bf16(Stream(ctx), input.Data(), scale.Data(), + zero_point == nullptr ? nullptr : zero_point->Data(), + output->MutableData(), static_cast(count), + static_cast(axis_stride), static_cast(scale_count)); + return Status::OK(); + } + + private: + int64_t axis_; +}; + +ONNX_OPERATOR_TYPED_KERNEL_EX( + DequantizeLinear, + kMSDomain, + 1, + Float8E4M3FN_BFloat16, + kCudaExecutionProvider, + (*KernelDefBuilder::Create()) + .TypeConstraint("T1", DataTypeImpl::GetTensorType()) + .TypeConstraint("T2", DataTypeImpl::GetTensorType()) + .TypeConstraint("TS", DataTypeImpl::GetTensorType()), + DequantizeLinearFp8Bf16); +#endif + } // namespace cuda } // namespace contrib } // namespace onnxruntime diff --git a/onnxruntime/core/framework/kernel_type_str_resolver_utils.cc b/onnxruntime/core/framework/kernel_type_str_resolver_utils.cc index 27312a2f0ed37..9f829ac38c61c 100644 --- a/onnxruntime/core/framework/kernel_type_str_resolver_utils.cc +++ b/onnxruntime/core/framework/kernel_type_str_resolver_utils.cc @@ -48,377 +48,378 @@ Status AddLayoutTransformationRequiredOpsToKernelTypeStrResolver(KernelTypeStrRe // clang-format off constexpr uint8_t kLayoutTransformationRequiredOpsKernelTypeStrResolverBytes[] = { 0x10, 0x00, 0x00, 0x00, 0x6b, 0x74, 0x73, 0x72, 0x00, 0x00, 0x06, 0x00, 0x08, 0x00, 0x04, 0x00, - 0x06, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x33, 0x00, 0x00, 0x00, 0x3c, 0x02, 0x00, 0x00, - 0x88, 0x0a, 0x00, 0x00, 0x60, 0x15, 0x00, 0x00, 0x6c, 0x13, 0x00, 0x00, 0x64, 0x11, 0x00, 0x00, - 0x7c, 0x04, 0x00, 0x00, 0xe4, 0x0e, 0x00, 0x00, 0xdc, 0x13, 0x00, 0x00, 0xb0, 0x02, 0x00, 0x00, - 0x10, 0x0b, 0x00, 0x00, 0xf8, 0x14, 0x00, 0x00, 0x40, 0x06, 0x00, 0x00, 0xa0, 0x08, 0x00, 0x00, - 0xa0, 0x10, 0x00, 0x00, 0x08, 0x0a, 0x00, 0x00, 0xb4, 0x01, 0x00, 0x00, 0xe4, 0x04, 0x00, 0x00, - 0xb0, 0x0b, 0x00, 0x00, 0x40, 0x10, 0x00, 0x00, 0x14, 0x0e, 0x00, 0x00, 0xb0, 0x03, 0x00, 0x00, - 0xe4, 0x02, 0x00, 0x00, 0x34, 0x0c, 0x00, 0x00, 0x18, 0x12, 0x00, 0x00, 0xb4, 0x00, 0x00, 0x00, - 0x2c, 0x0f, 0x00, 0x00, 0x08, 0x05, 0x00, 0x00, 0x60, 0x00, 0x00, 0x00, 0x48, 0x14, 0x00, 0x00, - 0x44, 0x05, 0x00, 0x00, 0x70, 0x15, 0x00, 0x00, 0xa4, 0x0f, 0x00, 0x00, 0xe8, 0x07, 0x00, 0x00, - 0x70, 0x09, 0x00, 0x00, 0x1c, 0x01, 0x00, 0x00, 0x9c, 0x10, 0x00, 0x00, 0xb0, 0x0b, 0x00, 0x00, - 0xd8, 0x13, 0x00, 0x00, 0x1c, 0x03, 0x00, 0x00, 0x84, 0x05, 0x00, 0x00, 0x08, 0x09, 0x00, 0x00, - 0x50, 0x0d, 0x00, 0x00, 0x10, 0x06, 0x00, 0x00, 0x60, 0x12, 0x00, 0x00, 0x58, 0x11, 0x00, 0x00, - 0xd4, 0x0c, 0x00, 0x00, 0x64, 0x08, 0x00, 0x00, 0x4c, 0x0c, 0x00, 0x00, 0xdc, 0x0a, 0x00, 0x00, - 0x60, 0x06, 0x00, 0x00, 0x9c, 0x15, 0x00, 0x00, 0xfc, 0xe9, 0xff, 0xff, 0x10, 0x00, 0x00, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x0b, 0x00, 0x00, 0x00, - 0x3a, 0x53, 0x71, 0x75, 0x65, 0x65, 0x7a, 0x65, 0x3a, 0x31, 0x31, 0x00, 0x20, 0xea, 0xff, 0xff, - 0x34, 0x15, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x0e, 0xea, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, 0x48, 0xea, 0xff, 0xff, - 0x44, 0xea, 0xff, 0xff, 0x18, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, - 0x40, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x50, 0x00, 0x00, 0x00, 0x12, 0x00, 0x00, 0x00, - 0x3a, 0x51, 0x75, 0x61, 0x6e, 0x74, 0x69, 0x7a, 0x65, 0x4c, 0x69, 0x6e, 0x65, 0x61, 0x72, 0x3a, - 0x32, 0x34, 0x00, 0x00, 0x78, 0xea, 0xff, 0xff, 0xa4, 0x15, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x54, 0xea, 0xff, 0xff, 0x01, 0x00, 0x00, 0x00, - 0x94, 0xea, 0xff, 0xff, 0x50, 0x15, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, - 0x04, 0x00, 0x00, 0x00, 0xb0, 0xea, 0xff, 0xff, 0xac, 0xea, 0xff, 0xff, 0x40, 0x11, 0x00, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x9a, 0xea, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, 0x94, 0xea, 0xff, 0xff, 0x02, 0x00, 0x00, 0x00, - 0xd4, 0xea, 0xff, 0xff, 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, - 0x18, 0x00, 0x00, 0x00, 0x0d, 0x00, 0x00, 0x00, 0x3a, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x70, 0x6f, - 0x73, 0x65, 0x3a, 0x31, 0x33, 0x00, 0x00, 0x00, 0xfc, 0xea, 0xff, 0xff, 0x58, 0x14, 0x00, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0xea, 0xea, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, 0x24, 0xeb, 0xff, 0xff, 0x20, 0xeb, 0xff, 0xff, - 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, - 0x0c, 0x00, 0x00, 0x00, 0x3a, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x3a, 0x32, 0x31, - 0x00, 0x00, 0x00, 0x00, 0x48, 0xeb, 0xff, 0xff, 0xf8, 0x0e, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x36, 0xeb, 0xff, 0xff, - 0x00, 0x00, 0x00, 0x01, 0x70, 0xeb, 0xff, 0xff, 0x6c, 0xeb, 0xff, 0xff, 0x18, 0x00, 0x00, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x60, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, - 0x20, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x3a, 0x44, 0x65, 0x71, 0x75, 0x61, 0x6e, 0x74, - 0x69, 0x7a, 0x65, 0x4c, 0x69, 0x6e, 0x65, 0x61, 0x72, 0x3a, 0x31, 0x30, 0x00, 0x00, 0x00, 0x00, - 0xa4, 0xeb, 0xff, 0xff, 0x60, 0x08, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x8e, 0xeb, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, 0xc0, 0xeb, 0xff, 0xff, - 0x8c, 0x08, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x9c, 0xeb, 0xff, 0xff, 0x01, 0x00, 0x00, 0x00, 0xdc, 0xeb, 0xff, 0xff, 0x78, 0x13, 0x00, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0xbc, 0xeb, 0xff, 0xff, 0x02, 0x00, 0x00, 0x00, 0x04, 0xec, 0xff, 0xff, 0x00, 0xec, 0xff, 0xff, - 0x14, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x34, 0x00, 0x00, 0x00, - 0x14, 0x00, 0x00, 0x00, 0x0a, 0x00, 0x00, 0x00, 0x3a, 0x47, 0x61, 0x74, 0x68, 0x65, 0x72, 0x3a, - 0x31, 0x31, 0x00, 0x00, 0x28, 0xec, 0xff, 0xff, 0x38, 0x11, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x04, 0xec, 0xff, 0xff, 0x01, 0x00, 0x00, 0x00, - 0x44, 0xec, 0xff, 0xff, 0x10, 0x13, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, - 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x32, 0xec, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, - 0x6c, 0xec, 0xff, 0xff, 0x68, 0xec, 0xff, 0xff, 0x14, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x02, 0x00, 0x00, 0x00, 0x48, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, 0x12, 0x00, 0x00, 0x00, - 0x3a, 0x51, 0x75, 0x61, 0x6e, 0x74, 0x69, 0x7a, 0x65, 0x4c, 0x69, 0x6e, 0x65, 0x61, 0x72, 0x3a, - 0x31, 0x39, 0x00, 0x00, 0x98, 0xec, 0xff, 0xff, 0x84, 0x13, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x86, 0xec, 0xff, 0xff, - 0x00, 0x00, 0x00, 0x01, 0x80, 0xec, 0xff, 0xff, 0x02, 0x00, 0x00, 0x00, 0xc0, 0xec, 0xff, 0xff, - 0x24, 0x13, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, - 0x04, 0x00, 0x00, 0x00, 0xa0, 0xec, 0xff, 0xff, 0x01, 0x00, 0x00, 0x00, 0xe8, 0xec, 0xff, 0xff, - 0xe4, 0xec, 0xff, 0xff, 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, - 0x18, 0x00, 0x00, 0x00, 0x0d, 0x00, 0x00, 0x00, 0x3a, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x70, 0x6f, - 0x73, 0x65, 0x3a, 0x32, 0x35, 0x00, 0x00, 0x00, 0x0c, 0xed, 0xff, 0xff, 0x48, 0x12, 0x00, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0xfa, 0xec, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, 0x34, 0xed, 0xff, 0xff, 0x30, 0xed, 0xff, 0xff, - 0x18, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x68, 0x00, 0x00, 0x00, - 0x3c, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, 0x12, 0x00, 0x00, 0x00, 0x3a, 0x51, 0x75, 0x61, - 0x6e, 0x74, 0x69, 0x7a, 0x65, 0x4c, 0x69, 0x6e, 0x65, 0x61, 0x72, 0x3a, 0x31, 0x33, 0x00, 0x00, - 0x64, 0xed, 0xff, 0xff, 0xb0, 0x0a, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x40, 0xed, 0xff, 0xff, 0x01, 0x00, 0x00, 0x00, 0x80, 0xed, 0xff, 0xff, - 0x9c, 0x12, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x6e, 0xed, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, 0x68, 0xed, 0xff, 0xff, - 0x02, 0x00, 0x00, 0x00, 0xa8, 0xed, 0xff, 0xff, 0x3c, 0x12, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0xc4, 0xed, 0xff, 0xff, 0xc0, 0xed, 0xff, 0xff, - 0x18, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x44, 0x00, 0x00, 0x00, - 0x64, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x3a, 0x44, 0x65, 0x71, - 0x75, 0x61, 0x6e, 0x74, 0x69, 0x7a, 0x65, 0x4c, 0x69, 0x6e, 0x65, 0x61, 0x72, 0x3a, 0x32, 0x34, - 0x00, 0x00, 0x00, 0x00, 0xf8, 0xed, 0xff, 0xff, 0xf4, 0x0d, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0xe2, 0xed, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, - 0x14, 0xee, 0xff, 0xff, 0xd0, 0x11, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, - 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0xf4, 0xed, 0xff, 0xff, 0x02, 0x00, 0x00, 0x00, - 0x3c, 0xee, 0xff, 0xff, 0x38, 0xee, 0xff, 0xff, 0xe4, 0x11, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x14, 0xee, 0xff, 0xff, 0x01, 0x00, 0x00, 0x00, - 0x54, 0xee, 0xff, 0xff, 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, - 0x18, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x3a, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, - 0x79, 0x3a, 0x32, 0x33, 0x00, 0x00, 0x00, 0x00, 0x7c, 0xee, 0xff, 0xff, 0xc4, 0x0b, 0x00, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x6a, 0xee, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, 0xa4, 0xee, 0xff, 0xff, 0xa0, 0xee, 0xff, 0xff, - 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, - 0x0a, 0x00, 0x00, 0x00, 0x3a, 0x53, 0x71, 0x75, 0x65, 0x65, 0x7a, 0x65, 0x3a, 0x31, 0x00, 0x00, - 0xc4, 0xee, 0xff, 0xff, 0x90, 0x10, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, - 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0xb2, 0xee, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, - 0xec, 0xee, 0xff, 0xff, 0xe8, 0xee, 0xff, 0xff, 0x14, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x02, 0x00, 0x00, 0x00, 0x34, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x0b, 0x00, 0x00, 0x00, - 0x3a, 0x53, 0x71, 0x75, 0x65, 0x65, 0x7a, 0x65, 0x3a, 0x32, 0x31, 0x00, 0x10, 0xef, 0xff, 0xff, - 0x6c, 0x10, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0xec, 0xee, 0xff, 0xff, 0x01, 0x00, 0x00, 0x00, 0x2c, 0xef, 0xff, 0xff, 0x28, 0x10, 0x00, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x1a, 0xef, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, 0x54, 0xef, 0xff, 0xff, 0x50, 0xef, 0xff, 0xff, - 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, - 0x0c, 0x00, 0x00, 0x00, 0x3a, 0x55, 0x6e, 0x73, 0x71, 0x75, 0x65, 0x65, 0x7a, 0x65, 0x3a, 0x31, - 0x00, 0x00, 0x00, 0x00, 0x78, 0xef, 0xff, 0xff, 0xdc, 0x0f, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x66, 0xef, 0xff, 0xff, - 0x00, 0x00, 0x00, 0x01, 0xa0, 0xef, 0xff, 0xff, 0x9c, 0xef, 0xff, 0xff, 0x10, 0x00, 0x00, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, - 0x3a, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x3a, 0x31, 0x33, 0x00, 0x00, 0x00, 0x00, - 0xc4, 0xef, 0xff, 0xff, 0x90, 0x0f, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, - 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0xb2, 0xef, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, - 0xec, 0xef, 0xff, 0xff, 0xe8, 0xef, 0xff, 0xff, 0x14, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x02, 0x00, 0x00, 0x00, 0x38, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x0d, 0x00, 0x00, 0x00, - 0x3a, 0x55, 0x6e, 0x73, 0x71, 0x75, 0x65, 0x65, 0x7a, 0x65, 0x3a, 0x32, 0x31, 0x00, 0x00, 0x00, - 0x14, 0xf0, 0xff, 0xff, 0x68, 0x0f, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, - 0x04, 0x00, 0x00, 0x00, 0xf0, 0xef, 0xff, 0xff, 0x01, 0x00, 0x00, 0x00, 0x30, 0xf0, 0xff, 0xff, - 0x24, 0x0f, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x1e, 0xf0, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, 0x58, 0xf0, 0xff, 0xff, - 0x54, 0xf0, 0xff, 0xff, 0x28, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, - 0x98, 0x00, 0x00, 0x00, 0xb8, 0x00, 0x00, 0x00, 0x04, 0x01, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00, - 0xd4, 0x00, 0x00, 0x00, 0x68, 0x00, 0x00, 0x00, 0x48, 0x00, 0x00, 0x00, 0x1b, 0x00, 0x00, 0x00, + 0x06, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x33, 0x00, 0x00, 0x00, 0x68, 0x16, 0x00, 0x00, + 0xd0, 0x15, 0x00, 0x00, 0x3c, 0x15, 0x00, 0x00, 0xb8, 0x14, 0x00, 0x00, 0x18, 0x14, 0x00, 0x00, + 0x80, 0x13, 0x00, 0x00, 0xe8, 0x12, 0x00, 0x00, 0x70, 0x12, 0x00, 0x00, 0x04, 0x12, 0x00, 0x00, + 0x98, 0x11, 0x00, 0x00, 0x4c, 0x11, 0x00, 0x00, 0xfc, 0x10, 0x00, 0x00, 0xa4, 0x10, 0x00, 0x00, + 0x54, 0x10, 0x00, 0x00, 0x04, 0x10, 0x00, 0x00, 0xb4, 0x0f, 0x00, 0x00, 0x64, 0x0f, 0x00, 0x00, + 0x14, 0x0f, 0x00, 0x00, 0xc4, 0x0e, 0x00, 0x00, 0x24, 0x0e, 0x00, 0x00, 0x90, 0x0d, 0x00, 0x00, + 0x10, 0x0d, 0x00, 0x00, 0x90, 0x0c, 0x00, 0x00, 0xfc, 0x0b, 0x00, 0x00, 0x68, 0x0b, 0x00, 0x00, + 0xd4, 0x0a, 0x00, 0x00, 0x88, 0x0a, 0x00, 0x00, 0x3c, 0x0a, 0x00, 0x00, 0xc4, 0x09, 0x00, 0x00, + 0x58, 0x09, 0x00, 0x00, 0xec, 0x08, 0x00, 0x00, 0x80, 0x08, 0x00, 0x00, 0x14, 0x08, 0x00, 0x00, + 0xc4, 0x07, 0x00, 0x00, 0x74, 0x07, 0x00, 0x00, 0x24, 0x07, 0x00, 0x00, 0xd4, 0x06, 0x00, 0x00, + 0x84, 0x06, 0x00, 0x00, 0x34, 0x06, 0x00, 0x00, 0xe4, 0x05, 0x00, 0x00, 0x94, 0x05, 0x00, 0x00, + 0x24, 0x05, 0x00, 0x00, 0xb4, 0x04, 0x00, 0x00, 0x44, 0x04, 0x00, 0x00, 0xd4, 0x03, 0x00, 0x00, + 0x64, 0x03, 0x00, 0x00, 0xb8, 0x02, 0x00, 0x00, 0x34, 0x02, 0x00, 0x00, 0xd8, 0x01, 0x00, 0x00, + 0x90, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0xdc, 0xe9, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, + 0x10, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x54, 0x00, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, + 0x1e, 0x00, 0x00, 0x00, 0x63, 0x6f, 0x6d, 0x2e, 0x6d, 0x69, 0x63, 0x72, 0x6f, 0x73, 0x6f, 0x66, + 0x74, 0x3a, 0x51, 0x75, 0x61, 0x6e, 0x74, 0x69, 0x7a, 0x65, 0x4c, 0x69, 0x6e, 0x65, 0x61, 0x72, + 0x3a, 0x31, 0x00, 0x00, 0x18, 0xea, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0x80, 0x14, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x66, 0xea, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x01, 0x10, 0xea, 0xff, 0xff, 0x02, 0x00, 0x00, 0x00, 0x40, 0xea, 0xff, 0xff, + 0x08, 0x00, 0x00, 0x00, 0x88, 0x14, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x30, 0xea, 0xff, 0xff, 0x01, 0x00, 0x00, 0x00, 0x28, 0xea, 0xff, 0xff, + 0x64, 0xea, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0x24, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, + 0x10, 0x01, 0x00, 0x00, 0xe4, 0x00, 0x00, 0x00, 0xb8, 0x00, 0x00, 0x00, 0x90, 0x00, 0x00, 0x00, + 0x64, 0x00, 0x00, 0x00, 0x44, 0x00, 0x00, 0x00, 0x24, 0x00, 0x00, 0x00, 0x1b, 0x00, 0x00, 0x00, 0x63, 0x6f, 0x6d, 0x2e, 0x6d, 0x69, 0x63, 0x72, 0x6f, 0x73, 0x6f, 0x66, 0x74, 0x3a, 0x51, 0x4c, - 0x69, 0x6e, 0x65, 0x61, 0x72, 0x43, 0x6f, 0x6e, 0x76, 0x3a, 0x31, 0x00, 0xa0, 0xf0, 0xff, 0xff, - 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, - 0x02, 0x00, 0x00, 0x00, 0x54, 0x34, 0x00, 0x00, 0x84, 0xf0, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, - 0xc4, 0xf0, 0xff, 0xff, 0x50, 0x07, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, - 0x04, 0x00, 0x00, 0x00, 0xa0, 0xf0, 0xff, 0xff, 0x06, 0x00, 0x00, 0x00, 0xe0, 0xf0, 0xff, 0xff, - 0x6c, 0x03, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0xbc, 0xf0, 0xff, 0xff, 0x01, 0x00, 0x00, 0x00, 0xfc, 0xf0, 0xff, 0xff, 0xe8, 0x0e, 0x00, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0xdc, 0xf0, 0xff, 0xff, 0x02, 0x00, 0x00, 0x00, 0x24, 0xf1, 0xff, 0xff, 0x20, 0xf1, 0xff, 0xff, - 0xfc, 0x0e, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x00, 0xf1, 0xff, 0xff, 0x05, 0x00, 0x00, 0x00, 0x08, 0xf1, 0xff, 0xff, - 0x03, 0x00, 0x00, 0x00, 0x48, 0xf1, 0xff, 0xff, 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x69, 0x6e, 0x65, 0x61, 0x72, 0x43, 0x6f, 0x6e, 0x76, 0x3a, 0x31, 0x00, 0xb0, 0xea, 0xff, 0xff, + 0x08, 0x00, 0x00, 0x00, 0x10, 0x0d, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x9c, 0xea, 0xff, 0xff, 0x06, 0x00, 0x00, 0x00, 0xcc, 0xea, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, + 0x18, 0x15, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0xb8, 0xea, 0xff, 0xff, + 0x01, 0x00, 0x00, 0x00, 0xe8, 0xea, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x77, 0x5f, 0x73, 0x63, - 0x61, 0x6c, 0x65, 0x00, 0x30, 0xf1, 0xff, 0xff, 0x04, 0x00, 0x00, 0x00, 0x70, 0xf1, 0xff, 0xff, - 0x7c, 0x0a, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x5e, 0xf1, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, 0x58, 0xf1, 0xff, 0xff, - 0x07, 0x00, 0x00, 0x00, 0x98, 0xf1, 0xff, 0xff, 0x14, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x02, 0x00, 0x00, 0x00, 0x34, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x0b, 0x00, 0x00, 0x00, - 0x3a, 0x53, 0x71, 0x75, 0x65, 0x65, 0x7a, 0x65, 0x3a, 0x32, 0x35, 0x00, 0xc0, 0xf1, 0xff, 0xff, - 0xbc, 0x0d, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x9c, 0xf1, 0xff, 0xff, 0x01, 0x00, 0x00, 0x00, 0xdc, 0xf1, 0xff, 0xff, 0x78, 0x0d, 0x00, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0xca, 0xf1, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, 0x04, 0xf2, 0xff, 0xff, 0x00, 0xf2, 0xff, 0xff, - 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, - 0x0c, 0x00, 0x00, 0x00, 0x3a, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x3a, 0x31, 0x34, - 0x00, 0x00, 0x00, 0x00, 0x28, 0xf2, 0xff, 0xff, 0x18, 0x08, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x16, 0xf2, 0xff, 0xff, - 0x00, 0x00, 0x00, 0x01, 0x50, 0xf2, 0xff, 0xff, 0x4c, 0xf2, 0xff, 0xff, 0x14, 0x00, 0x00, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x58, 0x00, 0x00, 0x00, 0x2c, 0x00, 0x00, 0x00, - 0x20, 0x00, 0x00, 0x00, 0x63, 0x6f, 0x6d, 0x2e, 0x6d, 0x69, 0x63, 0x72, 0x6f, 0x73, 0x6f, 0x66, - 0x74, 0x3a, 0x44, 0x65, 0x71, 0x75, 0x61, 0x6e, 0x74, 0x69, 0x7a, 0x65, 0x4c, 0x69, 0x6e, 0x65, - 0x61, 0x72, 0x3a, 0x31, 0x00, 0x00, 0x00, 0x00, 0x8c, 0xf2, 0xff, 0xff, 0x90, 0x0d, 0x00, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x7a, 0xf2, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, 0x74, 0xf2, 0xff, 0xff, 0x01, 0x00, 0x00, 0x00, - 0xb4, 0xf2, 0xff, 0xff, 0x30, 0x0d, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, - 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x94, 0xf2, 0xff, 0xff, 0x02, 0x00, 0x00, 0x00, - 0xdc, 0xf2, 0xff, 0xff, 0xd8, 0xf2, 0xff, 0xff, 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x01, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x0d, 0x00, 0x00, 0x00, 0x3a, 0x55, 0x6e, 0x73, - 0x71, 0x75, 0x65, 0x65, 0x7a, 0x65, 0x3a, 0x31, 0x31, 0x00, 0x00, 0x00, 0x00, 0xf3, 0xff, 0xff, - 0x54, 0x0c, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, - 0x04, 0x00, 0x00, 0x00, 0xee, 0xf2, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, 0x28, 0xf3, 0xff, 0xff, - 0x24, 0xf3, 0xff, 0xff, 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, - 0x18, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x3a, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x70, 0x6f, - 0x73, 0x65, 0x3a, 0x31, 0x00, 0x00, 0x00, 0x00, 0x4c, 0xf3, 0xff, 0xff, 0x08, 0x0c, 0x00, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x3a, 0xf3, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, 0x74, 0xf3, 0xff, 0xff, 0x70, 0xf3, 0xff, 0xff, - 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, - 0x0c, 0x00, 0x00, 0x00, 0x3a, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x3a, 0x31, 0x39, - 0x00, 0x00, 0x00, 0x00, 0x98, 0xf3, 0xff, 0xff, 0xa8, 0x06, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x86, 0xf3, 0xff, 0xff, - 0x00, 0x00, 0x00, 0x01, 0xc0, 0xf3, 0xff, 0xff, 0xbc, 0xf3, 0xff, 0xff, 0x18, 0x00, 0x00, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x4c, 0x00, 0x00, 0x00, 0x6c, 0x00, 0x00, 0x00, - 0x20, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x3a, 0x44, 0x65, 0x71, 0x75, 0x61, 0x6e, 0x74, - 0x69, 0x7a, 0x65, 0x4c, 0x69, 0x6e, 0x65, 0x61, 0x72, 0x3a, 0x31, 0x33, 0x00, 0x00, 0x00, 0x00, - 0xf4, 0xf3, 0xff, 0xff, 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, - 0x0c, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x79, 0x00, 0x00, 0x00, 0xe6, 0xf3, 0xff, 0xff, - 0x00, 0x00, 0x00, 0x01, 0x18, 0xf4, 0xff, 0xff, 0x3c, 0x0b, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0xf8, 0xf3, 0xff, 0xff, - 0x02, 0x00, 0x00, 0x00, 0x40, 0xf4, 0xff, 0xff, 0x3c, 0xf4, 0xff, 0xff, 0x10, 0x00, 0x00, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, - 0x78, 0x5f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x00, 0x24, 0xf4, 0xff, 0xff, 0x01, 0x00, 0x00, 0x00, - 0x64, 0xf4, 0xff, 0xff, 0x14, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, - 0x34, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x0a, 0x00, 0x00, 0x00, 0x3a, 0x47, 0x61, 0x74, - 0x68, 0x65, 0x72, 0x3a, 0x31, 0x33, 0x00, 0x00, 0x8c, 0xf4, 0xff, 0xff, 0xd4, 0x08, 0x00, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x68, 0xf4, 0xff, 0xff, - 0x01, 0x00, 0x00, 0x00, 0xa8, 0xf4, 0xff, 0xff, 0xac, 0x0a, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x96, 0xf4, 0xff, 0xff, - 0x00, 0x00, 0x00, 0x01, 0xd0, 0xf4, 0xff, 0xff, 0xcc, 0xf4, 0xff, 0xff, 0x10, 0x00, 0x00, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x24, 0x00, 0x00, 0x00, 0x1b, 0x00, 0x00, 0x00, - 0x63, 0x6f, 0x6d, 0x2e, 0x6d, 0x69, 0x63, 0x72, 0x6f, 0x73, 0x6f, 0x66, 0x74, 0x3a, 0x4e, 0x68, - 0x77, 0x63, 0x4d, 0x61, 0x78, 0x50, 0x6f, 0x6f, 0x6c, 0x3a, 0x31, 0x00, 0x00, 0xf5, 0xff, 0xff, - 0x54, 0x0a, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, - 0x04, 0x00, 0x00, 0x00, 0xee, 0xf4, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, 0x28, 0xf5, 0xff, 0xff, - 0x24, 0xf5, 0xff, 0xff, 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, - 0x18, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x3a, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, - 0x79, 0x3a, 0x32, 0x34, 0x00, 0x00, 0x00, 0x00, 0x4c, 0xf5, 0xff, 0xff, 0xf4, 0x04, 0x00, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x3a, 0xf5, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, 0x74, 0xf5, 0xff, 0xff, 0x70, 0xf5, 0xff, 0xff, - 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, - 0x0d, 0x00, 0x00, 0x00, 0x3a, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x70, 0x6f, 0x73, 0x65, 0x3a, 0x32, - 0x33, 0x00, 0x00, 0x00, 0x98, 0xf5, 0xff, 0xff, 0xbc, 0x09, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x86, 0xf5, 0xff, 0xff, - 0x00, 0x00, 0x00, 0x01, 0xc0, 0xf5, 0xff, 0xff, 0xbc, 0xf5, 0xff, 0xff, 0x14, 0x00, 0x00, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, - 0x12, 0x00, 0x00, 0x00, 0x3a, 0x51, 0x75, 0x61, 0x6e, 0x74, 0x69, 0x7a, 0x65, 0x4c, 0x69, 0x6e, - 0x65, 0x61, 0x72, 0x3a, 0x32, 0x31, 0x00, 0x00, 0xec, 0xf5, 0xff, 0xff, 0xf8, 0x09, 0x00, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0xcc, 0xf5, 0xff, 0xff, 0x01, 0x00, 0x00, 0x00, 0x14, 0xf6, 0xff, 0xff, 0x10, 0xf6, 0xff, 0xff, - 0x0c, 0x0a, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, - 0x04, 0x00, 0x00, 0x00, 0xfe, 0xf5, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, 0xf8, 0xf5, 0xff, 0xff, - 0x02, 0x00, 0x00, 0x00, 0x38, 0xf6, 0xff, 0xff, 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x01, 0x00, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, 0x1d, 0x00, 0x00, 0x00, 0x63, 0x6f, 0x6d, 0x2e, - 0x6d, 0x69, 0x63, 0x72, 0x6f, 0x73, 0x6f, 0x66, 0x74, 0x3a, 0x4e, 0x68, 0x77, 0x63, 0x46, 0x75, - 0x73, 0x65, 0x64, 0x43, 0x6f, 0x6e, 0x76, 0x3a, 0x31, 0x00, 0x00, 0x00, 0x70, 0xf6, 0xff, 0xff, - 0xe4, 0x08, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x34, 0x00, 0x00, 0x00, - 0x28, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x6a, 0xf6, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, 0x64, 0xf6, 0xff, 0xff, 0x03, 0x00, 0x00, 0x00, - 0x6c, 0xf6, 0xff, 0xff, 0x02, 0x00, 0x00, 0x00, 0x74, 0xf6, 0xff, 0xff, 0x01, 0x00, 0x00, 0x00, - 0xbc, 0xf6, 0xff, 0xff, 0xb8, 0xf6, 0xff, 0xff, 0x14, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x61, 0x6c, 0x65, 0x00, 0xe0, 0xea, 0xff, 0xff, 0x04, 0x00, 0x00, 0x00, 0x10, 0xeb, 0xff, 0xff, + 0x08, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x54, 0x34, 0x00, 0x00, 0x04, 0xeb, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, + 0x34, 0xeb, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0x48, 0x12, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x82, 0xeb, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, + 0x2c, 0xeb, 0xff, 0xff, 0x07, 0x00, 0x00, 0x00, 0x5c, 0xeb, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, + 0x3c, 0x13, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x4c, 0xeb, 0xff, 0xff, 0x05, 0x00, 0x00, 0x00, 0x54, 0xeb, 0xff, 0xff, 0x03, 0x00, 0x00, 0x00, + 0x84, 0xeb, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0x44, 0x13, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x74, 0xeb, 0xff, 0xff, 0x02, 0x00, 0x00, 0x00, + 0x6c, 0xeb, 0xff, 0xff, 0xa8, 0xeb, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x24, 0x00, 0x00, 0x00, 0x1b, 0x00, 0x00, 0x00, 0x63, 0x6f, 0x6d, 0x2e, + 0x6d, 0x69, 0x63, 0x72, 0x6f, 0x73, 0x6f, 0x66, 0x74, 0x3a, 0x4e, 0x68, 0x77, 0x63, 0x4d, 0x61, + 0x78, 0x50, 0x6f, 0x6f, 0x6c, 0x3a, 0x31, 0x00, 0xdc, 0xeb, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, + 0x3c, 0x14, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x2a, 0xec, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, 0xc4, 0xeb, 0xff, 0xff, 0x00, 0xec, 0xff, 0xff, + 0x08, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, + 0x1d, 0x00, 0x00, 0x00, 0x63, 0x6f, 0x6d, 0x2e, 0x6d, 0x69, 0x63, 0x72, 0x6f, 0x73, 0x6f, 0x66, + 0x74, 0x3a, 0x4e, 0x68, 0x77, 0x63, 0x46, 0x75, 0x73, 0x65, 0x64, 0x43, 0x6f, 0x6e, 0x76, 0x3a, + 0x31, 0x00, 0x00, 0x00, 0x38, 0xec, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0xe0, 0x13, 0x00, 0x00, + 0x05, 0x00, 0x00, 0x00, 0x34, 0x00, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, + 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x92, 0xec, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, + 0x3c, 0xec, 0xff, 0xff, 0x03, 0x00, 0x00, 0x00, 0x44, 0xec, 0xff, 0xff, 0x02, 0x00, 0x00, 0x00, + 0x4c, 0xec, 0xff, 0xff, 0x01, 0x00, 0x00, 0x00, 0x44, 0xec, 0xff, 0xff, 0x80, 0xec, 0xff, 0xff, + 0x08, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x74, 0x00, 0x00, 0x00, + 0x54, 0x00, 0x00, 0x00, 0x2c, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x63, 0x6f, 0x6d, 0x2e, + 0x6d, 0x69, 0x63, 0x72, 0x6f, 0x73, 0x6f, 0x66, 0x74, 0x3a, 0x44, 0x65, 0x71, 0x75, 0x61, 0x6e, + 0x74, 0x69, 0x7a, 0x65, 0x4c, 0x69, 0x6e, 0x65, 0x61, 0x72, 0x3a, 0x31, 0x00, 0x00, 0x00, 0x00, + 0xc4, 0xec, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x0c, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x54, 0x53, 0x00, 0x00, 0xb8, 0xec, 0xff, 0xff, + 0x01, 0x00, 0x00, 0x00, 0xe8, 0xec, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0xb0, 0x11, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x32, 0xed, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, + 0x04, 0xed, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0xc4, 0x11, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0xf4, 0xec, 0xff, 0xff, 0x02, 0x00, 0x00, 0x00, + 0xec, 0xec, 0xff, 0xff, 0x28, 0xed, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x38, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x0d, 0x00, 0x00, 0x00, 0x3a, 0x55, 0x6e, 0x73, 0x71, 0x75, 0x65, 0x65, 0x7a, 0x65, 0x3a, 0x32, 0x35, 0x00, 0x00, 0x00, - 0xe4, 0xf6, 0xff, 0xff, 0x98, 0x08, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, - 0x04, 0x00, 0x00, 0x00, 0xc0, 0xf6, 0xff, 0xff, 0x01, 0x00, 0x00, 0x00, 0x00, 0xf7, 0xff, 0xff, - 0x54, 0x08, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, - 0x04, 0x00, 0x00, 0x00, 0xee, 0xf6, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, 0x28, 0xf7, 0xff, 0xff, - 0x24, 0xf7, 0xff, 0xff, 0x14, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x54, 0xed, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0x24, 0x06, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x40, 0xed, 0xff, 0xff, 0x01, 0x00, 0x00, 0x00, 0x70, 0xed, 0xff, 0xff, + 0x08, 0x00, 0x00, 0x00, 0xa8, 0x12, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0xbe, 0xed, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, 0x58, 0xed, 0xff, 0xff, + 0x94, 0xed, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x38, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x0d, 0x00, 0x00, 0x00, 0x3a, 0x55, 0x6e, 0x73, - 0x71, 0x75, 0x65, 0x65, 0x7a, 0x65, 0x3a, 0x31, 0x33, 0x00, 0x00, 0x00, 0x50, 0xf7, 0xff, 0xff, - 0x2c, 0x08, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x2c, 0xf7, 0xff, 0xff, 0x01, 0x00, 0x00, 0x00, 0x6c, 0xf7, 0xff, 0xff, 0xe8, 0x07, 0x00, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x5a, 0xf7, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, 0x94, 0xf7, 0xff, 0xff, 0x90, 0xf7, 0xff, 0xff, - 0x18, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x4c, 0x00, 0x00, 0x00, - 0x20, 0x00, 0x00, 0x00, 0x5c, 0x00, 0x00, 0x00, 0x12, 0x00, 0x00, 0x00, 0x3a, 0x51, 0x75, 0x61, + 0x71, 0x75, 0x65, 0x65, 0x7a, 0x65, 0x3a, 0x32, 0x34, 0x00, 0x00, 0x00, 0xc0, 0xed, 0xff, 0xff, + 0x08, 0x00, 0x00, 0x00, 0xb8, 0x05, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0xac, 0xed, 0xff, 0xff, 0x01, 0x00, 0x00, 0x00, 0xdc, 0xed, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, + 0x3c, 0x12, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x2a, 0xee, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, 0xc4, 0xed, 0xff, 0xff, 0x00, 0xee, 0xff, 0xff, + 0x08, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x38, 0x00, 0x00, 0x00, + 0x18, 0x00, 0x00, 0x00, 0x0d, 0x00, 0x00, 0x00, 0x3a, 0x55, 0x6e, 0x73, 0x71, 0x75, 0x65, 0x65, + 0x7a, 0x65, 0x3a, 0x32, 0x33, 0x00, 0x00, 0x00, 0x2c, 0xee, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, + 0x4c, 0x05, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x18, 0xee, 0xff, 0xff, + 0x01, 0x00, 0x00, 0x00, 0x48, 0xee, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0xd0, 0x11, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x96, 0xee, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x01, 0x30, 0xee, 0xff, 0xff, 0x6c, 0xee, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, + 0x10, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x38, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, + 0x0d, 0x00, 0x00, 0x00, 0x3a, 0x55, 0x6e, 0x73, 0x71, 0x75, 0x65, 0x65, 0x7a, 0x65, 0x3a, 0x32, + 0x31, 0x00, 0x00, 0x00, 0x98, 0xee, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0xe0, 0x04, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x84, 0xee, 0xff, 0xff, 0x01, 0x00, 0x00, 0x00, + 0xb4, 0xee, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0x64, 0x11, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x02, 0xef, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, + 0x9c, 0xee, 0xff, 0xff, 0xd8, 0xee, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x38, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x0d, 0x00, 0x00, 0x00, + 0x3a, 0x55, 0x6e, 0x73, 0x71, 0x75, 0x65, 0x65, 0x7a, 0x65, 0x3a, 0x31, 0x33, 0x00, 0x00, 0x00, + 0x04, 0xef, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0x74, 0x04, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0xf0, 0xee, 0xff, 0xff, 0x01, 0x00, 0x00, 0x00, 0x20, 0xef, 0xff, 0xff, + 0x08, 0x00, 0x00, 0x00, 0xf8, 0x10, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x6e, 0xef, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, 0x08, 0xef, 0xff, 0xff, + 0x44, 0xef, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x18, 0x00, 0x00, 0x00, 0x0d, 0x00, 0x00, 0x00, 0x3a, 0x55, 0x6e, 0x73, 0x71, 0x75, 0x65, 0x65, + 0x7a, 0x65, 0x3a, 0x31, 0x31, 0x00, 0x00, 0x00, 0x6c, 0xef, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, + 0xac, 0x10, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0xba, 0xef, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, 0x54, 0xef, 0xff, 0xff, 0x90, 0xef, 0xff, 0xff, + 0x08, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, + 0x0c, 0x00, 0x00, 0x00, 0x3a, 0x55, 0x6e, 0x73, 0x71, 0x75, 0x65, 0x65, 0x7a, 0x65, 0x3a, 0x31, + 0x00, 0x00, 0x00, 0x00, 0xb8, 0xef, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0x60, 0x10, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x06, 0xf0, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x01, 0xa0, 0xef, 0xff, 0xff, 0xdc, 0xef, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, + 0x0c, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x0d, 0x00, 0x00, 0x00, + 0x3a, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x70, 0x6f, 0x73, 0x65, 0x3a, 0x32, 0x35, 0x00, 0x00, 0x00, + 0x04, 0xf0, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0x14, 0x10, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x52, 0xf0, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, + 0xec, 0xef, 0xff, 0xff, 0x28, 0xf0, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x0d, 0x00, 0x00, 0x00, 0x3a, 0x54, 0x72, 0x61, + 0x6e, 0x73, 0x70, 0x6f, 0x73, 0x65, 0x3a, 0x32, 0x34, 0x00, 0x00, 0x00, 0x50, 0xf0, 0xff, 0xff, + 0x08, 0x00, 0x00, 0x00, 0xc8, 0x0f, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x9e, 0xf0, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, 0x38, 0xf0, 0xff, 0xff, + 0x74, 0xf0, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x18, 0x00, 0x00, 0x00, 0x0d, 0x00, 0x00, 0x00, 0x3a, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x70, 0x6f, + 0x73, 0x65, 0x3a, 0x32, 0x33, 0x00, 0x00, 0x00, 0x9c, 0xf0, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, + 0x7c, 0x0f, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0xea, 0xf0, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, 0x84, 0xf0, 0xff, 0xff, 0xc0, 0xf0, 0xff, 0xff, + 0x08, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, + 0x0d, 0x00, 0x00, 0x00, 0x3a, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x70, 0x6f, 0x73, 0x65, 0x3a, 0x32, + 0x31, 0x00, 0x00, 0x00, 0xe8, 0xf0, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0x30, 0x0f, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x36, 0xf1, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x01, 0xd0, 0xf0, 0xff, 0xff, 0x0c, 0xf1, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, + 0x0c, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x0d, 0x00, 0x00, 0x00, + 0x3a, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x70, 0x6f, 0x73, 0x65, 0x3a, 0x31, 0x33, 0x00, 0x00, 0x00, + 0x34, 0xf1, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0xe4, 0x0e, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x82, 0xf1, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, + 0x1c, 0xf1, 0xff, 0xff, 0x58, 0xf1, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x3a, 0x54, 0x72, 0x61, + 0x6e, 0x73, 0x70, 0x6f, 0x73, 0x65, 0x3a, 0x31, 0x00, 0x00, 0x00, 0x00, 0x80, 0xf1, 0xff, 0xff, + 0x08, 0x00, 0x00, 0x00, 0x98, 0x0e, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0xce, 0xf1, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, 0x68, 0xf1, 0xff, 0xff, + 0xa4, 0xf1, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x34, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x0b, 0x00, 0x00, 0x00, 0x3a, 0x53, 0x71, 0x75, + 0x65, 0x65, 0x7a, 0x65, 0x3a, 0x32, 0x35, 0x00, 0xcc, 0xf1, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, + 0xac, 0x01, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0xb8, 0xf1, 0xff, 0xff, + 0x01, 0x00, 0x00, 0x00, 0xe8, 0xf1, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0x30, 0x0e, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x36, 0xf2, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x01, 0xd0, 0xf1, 0xff, 0xff, 0x0c, 0xf2, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, + 0x10, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x34, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, + 0x0b, 0x00, 0x00, 0x00, 0x3a, 0x53, 0x71, 0x75, 0x65, 0x65, 0x7a, 0x65, 0x3a, 0x32, 0x34, 0x00, + 0x34, 0xf2, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0x44, 0x01, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x20, 0xf2, 0xff, 0xff, 0x01, 0x00, 0x00, 0x00, 0x50, 0xf2, 0xff, 0xff, + 0x08, 0x00, 0x00, 0x00, 0xc8, 0x0d, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x9e, 0xf2, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, 0x38, 0xf2, 0xff, 0xff, + 0x74, 0xf2, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x34, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x0b, 0x00, 0x00, 0x00, 0x3a, 0x53, 0x71, 0x75, + 0x65, 0x65, 0x7a, 0x65, 0x3a, 0x32, 0x33, 0x00, 0x9c, 0xf2, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, + 0xdc, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x88, 0xf2, 0xff, 0xff, + 0x01, 0x00, 0x00, 0x00, 0xb8, 0xf2, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0x60, 0x0d, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x06, 0xf3, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x01, 0xa0, 0xf2, 0xff, 0xff, 0xdc, 0xf2, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, + 0x10, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x34, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, + 0x0b, 0x00, 0x00, 0x00, 0x3a, 0x53, 0x71, 0x75, 0x65, 0x65, 0x7a, 0x65, 0x3a, 0x32, 0x31, 0x00, + 0x04, 0xf3, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0x74, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0xf0, 0xf2, 0xff, 0xff, 0x01, 0x00, 0x00, 0x00, 0x20, 0xf3, 0xff, 0xff, + 0x08, 0x00, 0x00, 0x00, 0xf8, 0x0c, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x6e, 0xf3, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, 0x08, 0xf3, 0xff, 0xff, + 0x44, 0xf3, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x40, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x0b, 0x00, 0x00, 0x00, 0x3a, 0x53, 0x71, 0x75, + 0x65, 0x65, 0x7a, 0x65, 0x3a, 0x31, 0x33, 0x00, 0x6c, 0xf3, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, + 0x0c, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x61, 0x78, 0x65, 0x73, 0x00, 0x00, 0x00, 0x00, 0x64, 0xf3, 0xff, 0xff, 0x01, 0x00, 0x00, 0x00, + 0x94, 0xf3, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0x84, 0x0c, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0xe2, 0xf3, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, + 0x7c, 0xf3, 0xff, 0xff, 0xb8, 0xf3, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x0b, 0x00, 0x00, 0x00, 0x3a, 0x53, 0x71, 0x75, + 0x65, 0x65, 0x7a, 0x65, 0x3a, 0x31, 0x31, 0x00, 0xdc, 0xf3, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, + 0x3c, 0x0c, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x2a, 0xf4, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, 0xc4, 0xf3, 0xff, 0xff, 0x00, 0xf4, 0xff, 0xff, + 0x08, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, + 0x0a, 0x00, 0x00, 0x00, 0x3a, 0x53, 0x71, 0x75, 0x65, 0x65, 0x7a, 0x65, 0x3a, 0x31, 0x00, 0x00, + 0x24, 0xf4, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0xf4, 0x0b, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x72, 0xf4, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, + 0x0c, 0xf4, 0xff, 0xff, 0x48, 0xf4, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, + 0x03, 0x00, 0x00, 0x00, 0x68, 0x00, 0x00, 0x00, 0x48, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, + 0x12, 0x00, 0x00, 0x00, 0x3a, 0x51, 0x75, 0x61, 0x6e, 0x74, 0x69, 0x7a, 0x65, 0x4c, 0x69, 0x6e, + 0x65, 0x61, 0x72, 0x3a, 0x32, 0x35, 0x00, 0x00, 0x7c, 0xf4, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, + 0x00, 0x09, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0xca, 0xf4, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, 0x74, 0xf4, 0xff, 0xff, 0x02, 0x00, 0x00, 0x00, + 0xa4, 0xf4, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0xf4, 0x09, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x90, 0xf4, 0xff, 0xff, 0x01, 0x00, 0x00, 0x00, 0xc0, 0xf4, 0xff, 0xff, + 0x08, 0x00, 0x00, 0x00, 0x08, 0x0a, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x9c, 0xf4, 0xff, 0xff, 0xd8, 0xf4, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, + 0x03, 0x00, 0x00, 0x00, 0x68, 0x00, 0x00, 0x00, 0x48, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, + 0x12, 0x00, 0x00, 0x00, 0x3a, 0x51, 0x75, 0x61, 0x6e, 0x74, 0x69, 0x7a, 0x65, 0x4c, 0x69, 0x6e, + 0x65, 0x61, 0x72, 0x3a, 0x32, 0x34, 0x00, 0x00, 0x0c, 0xf5, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, + 0x70, 0x08, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x5a, 0xf5, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, 0x04, 0xf5, 0xff, 0xff, 0x02, 0x00, 0x00, 0x00, + 0x34, 0xf5, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0x64, 0x09, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x20, 0xf5, 0xff, 0xff, 0x01, 0x00, 0x00, 0x00, 0x50, 0xf5, 0xff, 0xff, + 0x08, 0x00, 0x00, 0x00, 0x78, 0x09, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x2c, 0xf5, 0xff, 0xff, 0x68, 0xf5, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, + 0x03, 0x00, 0x00, 0x00, 0x68, 0x00, 0x00, 0x00, 0x48, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, + 0x12, 0x00, 0x00, 0x00, 0x3a, 0x51, 0x75, 0x61, 0x6e, 0x74, 0x69, 0x7a, 0x65, 0x4c, 0x69, 0x6e, + 0x65, 0x61, 0x72, 0x3a, 0x32, 0x33, 0x00, 0x00, 0x9c, 0xf5, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, + 0xe0, 0x07, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0xea, 0xf5, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, 0x94, 0xf5, 0xff, 0xff, 0x02, 0x00, 0x00, 0x00, + 0xc4, 0xf5, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0xd4, 0x08, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0xb0, 0xf5, 0xff, 0xff, 0x01, 0x00, 0x00, 0x00, 0xe0, 0xf5, 0xff, 0xff, + 0x08, 0x00, 0x00, 0x00, 0xe8, 0x08, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0xbc, 0xf5, 0xff, 0xff, 0xf8, 0xf5, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x48, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, 0x12, 0x00, 0x00, 0x00, + 0x3a, 0x51, 0x75, 0x61, 0x6e, 0x74, 0x69, 0x7a, 0x65, 0x4c, 0x69, 0x6e, 0x65, 0x61, 0x72, 0x3a, + 0x32, 0x31, 0x00, 0x00, 0x28, 0xf6, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0x70, 0x08, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x76, 0xf6, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x01, 0x20, 0xf6, 0xff, 0xff, 0x02, 0x00, 0x00, 0x00, 0x50, 0xf6, 0xff, 0xff, + 0x08, 0x00, 0x00, 0x00, 0x78, 0x08, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x40, 0xf6, 0xff, 0xff, 0x01, 0x00, 0x00, 0x00, 0x38, 0xf6, 0xff, 0xff, + 0x74, 0xf6, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x48, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, 0x12, 0x00, 0x00, 0x00, 0x3a, 0x51, 0x75, 0x61, + 0x6e, 0x74, 0x69, 0x7a, 0x65, 0x4c, 0x69, 0x6e, 0x65, 0x61, 0x72, 0x3a, 0x31, 0x39, 0x00, 0x00, + 0xa4, 0xf6, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0xf4, 0x07, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0xf2, 0xf6, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, + 0x9c, 0xf6, 0xff, 0xff, 0x02, 0x00, 0x00, 0x00, 0xcc, 0xf6, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, + 0xfc, 0x07, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0xbc, 0xf6, 0xff, 0xff, 0x01, 0x00, 0x00, 0x00, 0xb4, 0xf6, 0xff, 0xff, 0xf0, 0xf6, 0xff, 0xff, + 0x08, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x68, 0x00, 0x00, 0x00, + 0x3c, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, 0x12, 0x00, 0x00, 0x00, 0x3a, 0x51, 0x75, 0x61, + 0x6e, 0x74, 0x69, 0x7a, 0x65, 0x4c, 0x69, 0x6e, 0x65, 0x61, 0x72, 0x3a, 0x31, 0x33, 0x00, 0x00, + 0x24, 0xf7, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0x9c, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x10, 0xf7, 0xff, 0xff, 0x01, 0x00, 0x00, 0x00, 0x40, 0xf7, 0xff, 0xff, + 0x08, 0x00, 0x00, 0x00, 0x58, 0x07, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x8e, 0xf7, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, 0x38, 0xf7, 0xff, 0xff, + 0x02, 0x00, 0x00, 0x00, 0x68, 0xf7, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0x60, 0x07, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x44, 0xf7, 0xff, 0xff, 0x80, 0xf7, 0xff, 0xff, + 0x08, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x74, 0x00, 0x00, 0x00, + 0x48, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, 0x12, 0x00, 0x00, 0x00, 0x3a, 0x51, 0x75, 0x61, 0x6e, 0x74, 0x69, 0x7a, 0x65, 0x4c, 0x69, 0x6e, 0x65, 0x61, 0x72, 0x3a, 0x31, 0x30, 0x00, 0x00, - 0xc4, 0xf7, 0xff, 0xff, 0x58, 0x08, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, - 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0xb2, 0xf7, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, - 0xac, 0xf7, 0xff, 0xff, 0x02, 0x00, 0x00, 0x00, 0xec, 0xf7, 0xff, 0xff, 0xf8, 0x07, 0x00, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x08, 0xf8, 0xff, 0xff, - 0x04, 0xf8, 0xff, 0xff, 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0xb4, 0xf7, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x79, 0x5f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x00, - 0xec, 0xf7, 0xff, 0xff, 0x01, 0x00, 0x00, 0x00, 0x2c, 0xf8, 0xff, 0xff, 0x18, 0x00, 0x00, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x60, 0x00, 0x00, 0x00, 0x24, 0x00, 0x00, 0x00, - 0x3c, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x3a, 0x44, 0x65, 0x71, 0x75, 0x61, 0x6e, 0x74, - 0x69, 0x7a, 0x65, 0x4c, 0x69, 0x6e, 0x65, 0x61, 0x72, 0x3a, 0x32, 0x35, 0x00, 0x00, 0x00, 0x00, - 0x64, 0xf8, 0xff, 0xff, 0xb8, 0x07, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x40, 0xf8, 0xff, 0xff, 0x01, 0x00, 0x00, 0x00, 0x80, 0xf8, 0xff, 0xff, - 0x6c, 0x03, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x6a, 0xf8, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, 0x9c, 0xf8, 0xff, 0xff, 0x48, 0x07, 0x00, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x7c, 0xf8, 0xff, 0xff, 0x02, 0x00, 0x00, 0x00, 0xc4, 0xf8, 0xff, 0xff, 0xc0, 0xf8, 0xff, 0xff, - 0x18, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x4c, 0x00, 0x00, 0x00, - 0x60, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, 0x12, 0x00, 0x00, 0x00, 0x3a, 0x51, 0x75, 0x61, - 0x6e, 0x74, 0x69, 0x7a, 0x65, 0x4c, 0x69, 0x6e, 0x65, 0x61, 0x72, 0x3a, 0x32, 0x35, 0x00, 0x00, - 0xf4, 0xf8, 0xff, 0xff, 0xf8, 0x02, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, - 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0xe2, 0xf8, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, - 0xdc, 0xf8, 0xff, 0xff, 0x02, 0x00, 0x00, 0x00, 0x1c, 0xf9, 0xff, 0xff, 0xc8, 0x06, 0x00, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x38, 0xf9, 0xff, 0xff, - 0x34, 0xf9, 0xff, 0xff, 0xe8, 0x06, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x10, 0xf9, 0xff, 0xff, 0x01, 0x00, 0x00, 0x00, 0x50, 0xf9, 0xff, 0xff, - 0x14, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x34, 0x00, 0x00, 0x00, - 0x14, 0x00, 0x00, 0x00, 0x0b, 0x00, 0x00, 0x00, 0x3a, 0x53, 0x71, 0x75, 0x65, 0x65, 0x7a, 0x65, - 0x3a, 0x32, 0x34, 0x00, 0x78, 0xf9, 0xff, 0xff, 0x04, 0x06, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x54, 0xf9, 0xff, 0xff, 0x01, 0x00, 0x00, 0x00, - 0x94, 0xf9, 0xff, 0xff, 0xc0, 0x05, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, - 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x82, 0xf9, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, - 0xbc, 0xf9, 0xff, 0xff, 0xb8, 0xf9, 0xff, 0xff, 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0xac, 0xf7, 0xff, 0xff, 0x01, 0x00, 0x00, 0x00, 0xdc, 0xf7, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, + 0xbc, 0x06, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x2a, 0xf8, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, 0xd4, 0xf7, 0xff, 0xff, 0x02, 0x00, 0x00, 0x00, + 0x04, 0xf8, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0xc4, 0x06, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0xe0, 0xf7, 0xff, 0xff, 0x1c, 0xf8, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, + 0x0c, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, + 0x3a, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x3a, 0x32, 0x35, 0x00, 0x00, 0x00, 0x00, + 0x44, 0xf8, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0xd8, 0x01, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x92, 0xf8, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, + 0x2c, 0xf8, 0xff, 0xff, 0x68, 0xf8, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x3a, 0x49, 0x64, 0x65, - 0x6e, 0x74, 0x69, 0x74, 0x79, 0x3a, 0x32, 0x35, 0x00, 0x00, 0x00, 0x00, 0xe0, 0xf9, 0xff, 0xff, - 0x60, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, - 0x04, 0x00, 0x00, 0x00, 0xce, 0xf9, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, 0x08, 0xfa, 0xff, 0xff, - 0x04, 0xfa, 0xff, 0xff, 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x6e, 0x74, 0x69, 0x74, 0x79, 0x3a, 0x32, 0x34, 0x00, 0x00, 0x00, 0x00, 0x90, 0xf8, 0xff, 0xff, + 0x08, 0x00, 0x00, 0x00, 0x8c, 0x01, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0xde, 0xf8, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, 0x78, 0xf8, 0xff, 0xff, + 0xb4, 0xf8, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x3a, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, - 0x79, 0x3a, 0x31, 0x36, 0x00, 0x00, 0x00, 0x00, 0x2c, 0xfa, 0xff, 0xff, 0x14, 0x00, 0x00, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, - 0x01, 0x00, 0x00, 0x00, 0x56, 0x00, 0x00, 0x00, 0x22, 0xfa, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, - 0x5c, 0xfa, 0xff, 0xff, 0x58, 0xfa, 0xff, 0xff, 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x01, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x0d, 0x00, 0x00, 0x00, 0x3a, 0x54, 0x72, 0x61, - 0x6e, 0x73, 0x70, 0x6f, 0x73, 0x65, 0x3a, 0x32, 0x31, 0x00, 0x00, 0x00, 0x80, 0xfa, 0xff, 0xff, - 0xd4, 0x04, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x6e, 0xfa, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, 0xa8, 0xfa, 0xff, 0xff, - 0xa4, 0xfa, 0xff, 0xff, 0x18, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, - 0x44, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, + 0x79, 0x3a, 0x32, 0x33, 0x00, 0x00, 0x00, 0x00, 0xdc, 0xf8, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, + 0x40, 0x01, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x2a, 0xf9, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, 0xc4, 0xf8, 0xff, 0xff, 0x00, 0xf9, 0xff, 0xff, + 0x08, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, + 0x0c, 0x00, 0x00, 0x00, 0x3a, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x3a, 0x32, 0x31, + 0x00, 0x00, 0x00, 0x00, 0x28, 0xf9, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0xf4, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x76, 0xf9, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x01, 0x10, 0xf9, 0xff, 0xff, 0x4c, 0xf9, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, + 0x0c, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, + 0x3a, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x3a, 0x31, 0x39, 0x00, 0x00, 0x00, 0x00, + 0x74, 0xf9, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0xa8, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0xc2, 0xf9, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, + 0x5c, 0xf9, 0xff, 0xff, 0x98, 0xf9, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x3a, 0x49, 0x64, 0x65, + 0x6e, 0x74, 0x69, 0x74, 0x79, 0x3a, 0x31, 0x36, 0x00, 0x00, 0x00, 0x00, 0xc0, 0xf9, 0xff, 0xff, + 0x08, 0x00, 0x00, 0x00, 0x5c, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x0e, 0xfa, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, 0xa8, 0xf9, 0xff, 0xff, + 0xe4, 0xf9, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x18, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x3a, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, + 0x79, 0x3a, 0x31, 0x34, 0x00, 0x00, 0x00, 0x00, 0x0c, 0xfa, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, + 0x10, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x56, 0x00, 0x00, 0x00, 0x62, 0xfa, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, + 0xfc, 0xf9, 0xff, 0xff, 0x38, 0xfa, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x3a, 0x49, 0x64, 0x65, + 0x6e, 0x74, 0x69, 0x74, 0x79, 0x3a, 0x31, 0x33, 0x00, 0x00, 0x00, 0x00, 0x60, 0xfa, 0xff, 0xff, + 0x08, 0x00, 0x00, 0x00, 0xb8, 0x05, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0xae, 0xfa, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, 0x48, 0xfa, 0xff, 0xff, + 0x84, 0xfa, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x14, 0x00, 0x00, 0x00, 0x0b, 0x00, 0x00, 0x00, 0x3a, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, + 0x79, 0x3a, 0x31, 0x00, 0xa8, 0xfa, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0x70, 0x05, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0xf6, 0xfa, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x01, 0x90, 0xfa, 0xff, 0xff, 0xcc, 0xfa, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, + 0x10, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x34, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, + 0x0a, 0x00, 0x00, 0x00, 0x3a, 0x47, 0x61, 0x74, 0x68, 0x65, 0x72, 0x3a, 0x31, 0x33, 0x00, 0x00, + 0xf4, 0xfa, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0xdc, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0xe0, 0xfa, 0xff, 0xff, 0x01, 0x00, 0x00, 0x00, 0x10, 0xfb, 0xff, 0xff, + 0x08, 0x00, 0x00, 0x00, 0x08, 0x05, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x5e, 0xfb, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, 0xf8, 0xfa, 0xff, 0xff, + 0x34, 0xfb, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x34, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x0a, 0x00, 0x00, 0x00, 0x3a, 0x47, 0x61, 0x74, + 0x68, 0x65, 0x72, 0x3a, 0x31, 0x31, 0x00, 0x00, 0x5c, 0xfb, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, + 0x74, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x48, 0xfb, 0xff, 0xff, + 0x01, 0x00, 0x00, 0x00, 0x78, 0xfb, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0xa0, 0x04, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0xc6, 0xfb, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x01, 0x60, 0xfb, 0xff, 0xff, 0x9c, 0xfb, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, + 0x10, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, + 0x09, 0x00, 0x00, 0x00, 0x3a, 0x47, 0x61, 0x74, 0x68, 0x65, 0x72, 0x3a, 0x31, 0x00, 0x00, 0x00, + 0xc4, 0xfb, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x54, 0x69, 0x6e, 0x64, 0x00, 0x00, 0x00, 0x00, + 0xbc, 0xfb, 0xff, 0xff, 0x01, 0x00, 0x00, 0x00, 0xec, 0xfb, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, + 0x2c, 0x04, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x3a, 0xfc, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, 0xd4, 0xfb, 0xff, 0xff, 0x10, 0xfc, 0xff, 0xff, + 0x08, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x60, 0x00, 0x00, 0x00, + 0x40, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x3a, 0x44, 0x65, 0x71, + 0x75, 0x61, 0x6e, 0x74, 0x69, 0x7a, 0x65, 0x4c, 0x69, 0x6e, 0x65, 0x61, 0x72, 0x3a, 0x32, 0x35, + 0x00, 0x00, 0x00, 0x00, 0x48, 0xfc, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0x34, 0x01, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x92, 0xfc, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, + 0x64, 0xfc, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0x34, 0x02, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x50, 0xfc, 0xff, 0xff, 0x01, 0x00, 0x00, 0x00, 0x80, 0xfc, 0xff, 0xff, + 0x08, 0x00, 0x00, 0x00, 0x48, 0x02, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x70, 0xfc, 0xff, 0xff, 0x02, 0x00, 0x00, 0x00, 0x68, 0xfc, 0xff, 0xff, + 0xa4, 0xfc, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, + 0x60, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x3a, 0x44, 0x65, 0x71, 0x75, 0x61, 0x6e, 0x74, 0x69, 0x7a, 0x65, 0x4c, 0x69, 0x6e, 0x65, 0x61, - 0x72, 0x3a, 0x32, 0x33, 0x00, 0x00, 0x00, 0x00, 0xdc, 0xfa, 0xff, 0xff, 0x10, 0x01, 0x00, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0xc6, 0xfa, 0xff, 0xff, - 0x00, 0x00, 0x00, 0x01, 0xf8, 0xfa, 0xff, 0xff, 0xec, 0x04, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0xd8, 0xfa, 0xff, 0xff, - 0x02, 0x00, 0x00, 0x00, 0x20, 0xfb, 0xff, 0xff, 0x1c, 0xfb, 0xff, 0xff, 0x00, 0x05, 0x00, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0xf8, 0xfa, 0xff, 0xff, - 0x01, 0x00, 0x00, 0x00, 0x38, 0xfb, 0xff, 0xff, 0x14, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x02, 0x00, 0x00, 0x00, 0x38, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x0d, 0x00, 0x00, 0x00, - 0x3a, 0x55, 0x6e, 0x73, 0x71, 0x75, 0x65, 0x65, 0x7a, 0x65, 0x3a, 0x32, 0x34, 0x00, 0x00, 0x00, - 0x64, 0xfb, 0xff, 0xff, 0x18, 0x04, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x40, 0xfb, 0xff, 0xff, 0x01, 0x00, 0x00, 0x00, 0x80, 0xfb, 0xff, 0xff, - 0xd4, 0x03, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x6e, 0xfb, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, 0xa8, 0xfb, 0xff, 0xff, - 0xa4, 0xfb, 0xff, 0xff, 0x18, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, - 0x70, 0x00, 0x00, 0x00, 0x50, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, 0x12, 0x00, 0x00, 0x00, - 0x3a, 0x51, 0x75, 0x61, 0x6e, 0x74, 0x69, 0x7a, 0x65, 0x4c, 0x69, 0x6e, 0x65, 0x61, 0x72, 0x3a, - 0x32, 0x33, 0x00, 0x00, 0xd8, 0xfb, 0xff, 0xff, 0x14, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x72, 0x3a, 0x32, 0x34, 0x00, 0x00, 0x00, 0x00, 0xdc, 0xfc, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, + 0xa0, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x26, 0xfd, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x01, 0xf8, 0xfc, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0xa0, 0x01, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0xe4, 0xfc, 0xff, 0xff, 0x01, 0x00, 0x00, 0x00, + 0x14, 0xfd, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0xb4, 0x01, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x04, 0xfd, 0xff, 0xff, 0x02, 0x00, 0x00, 0x00, + 0xfc, 0xfc, 0xff, 0xff, 0x38, 0xfd, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, + 0x03, 0x00, 0x00, 0x00, 0x68, 0x00, 0x00, 0x00, 0x48, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, + 0x14, 0x00, 0x00, 0x00, 0x3a, 0x44, 0x65, 0x71, 0x75, 0x61, 0x6e, 0x74, 0x69, 0x7a, 0x65, 0x4c, + 0x69, 0x6e, 0x65, 0x61, 0x72, 0x3a, 0x32, 0x33, 0x00, 0x00, 0x00, 0x00, 0x70, 0xfd, 0xff, 0xff, + 0x08, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x54, 0x33, 0x00, 0x00, 0xc2, 0xfd, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, + 0x94, 0xfd, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0x04, 0x01, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x80, 0xfd, 0xff, 0xff, 0x01, 0x00, 0x00, 0x00, 0xb0, 0xfd, 0xff, 0xff, + 0x08, 0x00, 0x00, 0x00, 0x18, 0x01, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0xa0, 0xfd, 0xff, 0xff, 0x02, 0x00, 0x00, 0x00, 0x98, 0xfd, 0xff, 0xff, + 0xd4, 0xfd, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x4c, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x3a, 0x44, 0x65, 0x71, + 0x75, 0x61, 0x6e, 0x74, 0x69, 0x7a, 0x65, 0x4c, 0x69, 0x6e, 0x65, 0x61, 0x72, 0x3a, 0x32, 0x31, + 0x00, 0x00, 0x00, 0x00, 0x08, 0xfe, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0x90, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x56, 0xfe, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x01, 0x00, 0xfe, 0xff, 0xff, 0x01, 0x00, 0x00, 0x00, 0x30, 0xfe, 0xff, 0xff, + 0x08, 0x00, 0x00, 0x00, 0x98, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x20, 0xfe, 0xff, 0xff, 0x02, 0x00, 0x00, 0x00, 0x18, 0xfe, 0xff, 0xff, + 0x54, 0xfe, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x54, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x3a, 0x44, 0x65, 0x71, + 0x75, 0x61, 0x6e, 0x74, 0x69, 0x7a, 0x65, 0x4c, 0x69, 0x6e, 0x65, 0x61, 0x72, 0x3a, 0x31, 0x39, + 0x00, 0x00, 0x00, 0x00, 0x88, 0xfe, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, - 0x54, 0x33, 0x00, 0x00, 0xce, 0xfb, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, 0xc8, 0xfb, 0xff, 0xff, - 0x02, 0x00, 0x00, 0x00, 0x08, 0xfc, 0xff, 0xff, 0x14, 0x04, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0xe4, 0xfb, 0xff, 0xff, 0x01, 0x00, 0x00, 0x00, - 0x24, 0xfc, 0xff, 0xff, 0xc0, 0x03, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x40, 0xfc, 0xff, 0xff, 0x3c, 0xfc, 0xff, 0xff, 0x14, 0x00, 0x00, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x38, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, - 0x0d, 0x00, 0x00, 0x00, 0x3a, 0x55, 0x6e, 0x73, 0x71, 0x75, 0x65, 0x65, 0x7a, 0x65, 0x3a, 0x32, - 0x33, 0x00, 0x00, 0x00, 0x68, 0xfc, 0xff, 0xff, 0x14, 0x03, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x44, 0xfc, 0xff, 0xff, 0x01, 0x00, 0x00, 0x00, - 0x84, 0xfc, 0xff, 0xff, 0xd0, 0x02, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, - 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x72, 0xfc, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, - 0xac, 0xfc, 0xff, 0xff, 0xa8, 0xfc, 0xff, 0xff, 0x14, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x02, 0x00, 0x00, 0x00, 0x4c, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, - 0x3a, 0x44, 0x65, 0x71, 0x75, 0x61, 0x6e, 0x74, 0x69, 0x7a, 0x65, 0x4c, 0x69, 0x6e, 0x65, 0x61, - 0x72, 0x3a, 0x32, 0x31, 0x00, 0x00, 0x00, 0x00, 0xdc, 0xfc, 0xff, 0xff, 0x40, 0x03, 0x00, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0xca, 0xfc, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, 0xc4, 0xfc, 0xff, 0xff, 0x01, 0x00, 0x00, 0x00, - 0x04, 0xfd, 0xff, 0xff, 0xe0, 0x02, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, - 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0xe4, 0xfc, 0xff, 0xff, 0x02, 0x00, 0x00, 0x00, - 0x2c, 0xfd, 0xff, 0xff, 0x28, 0xfd, 0xff, 0xff, 0x14, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x02, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, - 0x3a, 0x47, 0x61, 0x74, 0x68, 0x65, 0x72, 0x3a, 0x31, 0x00, 0x00, 0x00, 0x50, 0xfd, 0xff, 0xff, - 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x54, 0x69, 0x6e, 0x64, 0x00, 0x00, 0x00, 0x00, 0x38, 0xfd, 0xff, 0xff, - 0x01, 0x00, 0x00, 0x00, 0x78, 0xfd, 0xff, 0xff, 0xdc, 0x01, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x66, 0xfd, 0xff, 0xff, - 0x00, 0x00, 0x00, 0x01, 0xa0, 0xfd, 0xff, 0xff, 0x9c, 0xfd, 0xff, 0xff, 0x10, 0x00, 0x00, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x0d, 0x00, 0x00, 0x00, - 0x3a, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x70, 0x6f, 0x73, 0x65, 0x3a, 0x32, 0x34, 0x00, 0x00, 0x00, - 0xc4, 0xfd, 0xff, 0xff, 0x90, 0x01, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, - 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0xb2, 0xfd, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, - 0xec, 0xfd, 0xff, 0xff, 0xe8, 0xfd, 0xff, 0xff, 0x14, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x02, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x38, 0x00, 0x00, 0x00, 0x0b, 0x00, 0x00, 0x00, - 0x3a, 0x53, 0x71, 0x75, 0x65, 0x65, 0x7a, 0x65, 0x3a, 0x31, 0x33, 0x00, 0x10, 0xfe, 0xff, 0xff, - 0x44, 0x01, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, - 0x04, 0x00, 0x00, 0x00, 0xfe, 0xfd, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, 0x38, 0xfe, 0xff, 0xff, - 0x34, 0xfe, 0xff, 0xff, 0x48, 0x01, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x10, 0xfe, 0xff, 0xff, 0x01, 0x00, 0x00, 0x00, 0x50, 0xfe, 0xff, 0xff, - 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, - 0x0b, 0x00, 0x00, 0x00, 0x3a, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x3a, 0x31, 0x00, - 0x74, 0xfe, 0xff, 0xff, 0xe0, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, - 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x62, 0xfe, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, - 0x9c, 0xfe, 0xff, 0xff, 0x98, 0xfe, 0xff, 0xff, 0x14, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x02, 0x00, 0x00, 0x00, 0x4c, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, + 0x54, 0x32, 0x00, 0x00, 0xde, 0xfe, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, 0x88, 0xfe, 0xff, 0xff, + 0x01, 0x00, 0x00, 0x00, 0xb8, 0xfe, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x54, 0x31, 0x00, 0x00, 0xb0, 0xfe, 0xff, 0xff, 0x02, 0x00, 0x00, 0x00, 0xa8, 0xfe, 0xff, 0xff, + 0xe4, 0xfe, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, + 0x60, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x3a, 0x44, 0x65, 0x71, 0x75, 0x61, 0x6e, 0x74, 0x69, 0x7a, 0x65, 0x4c, 0x69, 0x6e, 0x65, 0x61, - 0x72, 0x3a, 0x31, 0x39, 0x00, 0x00, 0x00, 0x00, 0xcc, 0xfe, 0xff, 0xff, 0x50, 0x01, 0x00, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0xba, 0xfe, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, 0xb4, 0xfe, 0xff, 0xff, 0x01, 0x00, 0x00, 0x00, - 0xf4, 0xfe, 0xff, 0xff, 0xf0, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, - 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0xd4, 0xfe, 0xff, 0xff, 0x02, 0x00, 0x00, 0x00, - 0x1c, 0xff, 0xff, 0xff, 0x18, 0xff, 0xff, 0xff, 0x14, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x02, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x0b, 0x00, 0x00, 0x00, - 0x3a, 0x53, 0x71, 0x75, 0x65, 0x65, 0x7a, 0x65, 0x3a, 0x32, 0x33, 0x00, 0x40, 0xff, 0xff, 0xff, - 0x14, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, - 0x0c, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x54, 0x00, 0x00, 0x00, 0x36, 0xff, 0xff, 0xff, - 0x00, 0x00, 0x00, 0x01, 0x70, 0xff, 0xff, 0xff, 0x6c, 0xff, 0xff, 0xff, 0x10, 0x00, 0x00, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x61, 0x78, 0x65, 0x73, 0x00, 0x00, 0x00, 0x00, 0x54, 0xff, 0xff, 0xff, 0x01, 0x00, 0x00, 0x00, - 0x94, 0xff, 0xff, 0xff, 0x14, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, - 0x2c, 0x00, 0x00, 0x00, 0x60, 0x00, 0x00, 0x00, 0x1e, 0x00, 0x00, 0x00, 0x63, 0x6f, 0x6d, 0x2e, - 0x6d, 0x69, 0x63, 0x72, 0x6f, 0x73, 0x6f, 0x66, 0x74, 0x3a, 0x51, 0x75, 0x61, 0x6e, 0x74, 0x69, - 0x7a, 0x65, 0x4c, 0x69, 0x6e, 0x65, 0x61, 0x72, 0x3a, 0x31, 0x00, 0x00, 0xd0, 0xff, 0xff, 0xff, - 0x14, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, - 0x0c, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x54, 0x31, 0x00, 0x00, 0xb8, 0xff, 0xff, 0xff, - 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x04, 0x00, 0x04, 0x00, 0x00, 0x00, 0x08, 0x00, 0x0c, 0x00, - 0x04, 0x00, 0x08, 0x00, 0x08, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x02, 0x00, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, - 0x54, 0x32, 0x00, 0x00, 0x00, 0x00, 0x06, 0x00, 0x08, 0x00, 0x07, 0x00, 0x06, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x01, 0x08, 0x00, 0x08, 0x00, 0x00, 0x00, 0x04, 0x00, 0x08, 0x00, 0x00, 0x00, - 0x02, 0x00, 0x00, 0x00, + 0x72, 0x3a, 0x31, 0x33, 0x00, 0x00, 0x00, 0x00, 0x1c, 0xff, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, + 0xa0, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x66, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x01, 0x38, 0xff, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0xac, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x24, 0xff, 0xff, 0xff, 0x01, 0x00, 0x00, 0x00, + 0x54, 0xff, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0xc4, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x44, 0xff, 0xff, 0xff, 0x02, 0x00, 0x00, 0x00, + 0x3c, 0xff, 0xff, 0xff, 0x78, 0xff, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, + 0x03, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x4c, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, + 0x14, 0x00, 0x00, 0x00, 0x3a, 0x44, 0x65, 0x71, 0x75, 0x61, 0x6e, 0x74, 0x69, 0x7a, 0x65, 0x4c, + 0x69, 0x6e, 0x65, 0x61, 0x72, 0x3a, 0x31, 0x30, 0x00, 0x00, 0x00, 0x00, 0xb0, 0xff, 0xff, 0xff, + 0x08, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x79, 0x00, 0x06, 0x00, 0x08, 0x00, 0x07, 0x00, 0x06, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x01, 0xd8, 0xff, 0xff, 0xff, 0x08, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x78, 0x5f, 0x73, 0x63, + 0x61, 0x6c, 0x65, 0x00, 0xd0, 0xff, 0xff, 0xff, 0x01, 0x00, 0x00, 0x00, 0x08, 0x00, 0x0c, 0x00, + 0x08, 0x00, 0x04, 0x00, 0x08, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x24, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x54, 0x00, 0x00, 0x00, 0x08, 0x00, 0x08, 0x00, 0x00, 0x00, 0x04, 0x00, 0x08, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x04, 0x00, 0x04, 0x00, 0x04, 0x00, 0x00, 0x00, }; // clang-format on diff --git a/onnxruntime/test/contrib_ops/dequantize_fp8_bf16_test.cc b/onnxruntime/test/contrib_ops/dequantize_fp8_bf16_test.cc index 7e932c799fd0a..172f6bbb98d7f 100644 --- a/onnxruntime/test/contrib_ops/dequantize_fp8_bf16_test.cc +++ b/onnxruntime/test/contrib_ops/dequantize_fp8_bf16_test.cc @@ -11,6 +11,42 @@ namespace onnxruntime { namespace test { #if defined(USE_CUDA) && !defined(DISABLE_FLOAT8_TYPES) +static void RunFp8Bf16Case(const std::vector& shape, int64_t axis, + const std::vector& values, const std::vector& scale_shape, + const std::vector& scales, const std::vector& zeros, + const std::vector& expected, + const std::string& error = "") { + auto provider = DefaultCudaExecutionProvider(); + if (provider == nullptr) { + GTEST_SKIP() << "CUDA execution provider is not available"; + } + std::vector> providers; + providers.push_back(std::move(provider)); + + OpTester test("DequantizeLinear", 1, kMSDomain); + test.AddAttribute("axis", axis); + test.AddAttribute("output_dtype", ONNX_NAMESPACE::TensorProto_DataType_BFLOAT16); + std::vector input; + for (float value : values) { + input.emplace_back(value); + } + test.AddInput("x", shape, input); + test.AddInput("x_scale", scale_shape, scales); + if (zeros.empty()) { + test.AddOptionalInputEdge(); + } else { + std::vector zero_point; + for (float value : zeros) { + zero_point.emplace_back(value); + } + test.AddInput("x_zero_point", scale_shape, zero_point); + } + test.AddOutput("y", shape, ToBFloat16(expected)); + test.SetOutputTolerance(0.0f, 0.0f); + test.Run(error.empty() ? OpTester::ExpectResult::kExpectSuccess : OpTester::ExpectResult::kExpectFailure, + error, {}, nullptr, &providers); +} + TEST(DequantizeLinearTest, Fp8ToBf16) { auto provider = DefaultCudaExecutionProvider(); if (provider == nullptr) { @@ -28,6 +64,35 @@ TEST(DequantizeLinearTest, Fp8ToBf16) { test.AddOutput("weight_bf16", {1, 8}, ToBFloat16(std::vector(8, 2.0f))); test.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &providers); } + +TEST(DequantizeLinearTest, Fp8ToBf16ScalarFp32Scale) { + // 1.00390625 rounds to 1 in BF16. Multiplication must use the original FP32 scale. + RunFp8Bf16Case({4}, 0, {1.0f, 3.0f, -3.0f, 0.0f}, {}, {1.00390625f}, {}, + {1.00390625f, 3.01171875f, -3.01171875f, 0.0f}); +} + +TEST(DequantizeLinearTest, Fp8ToBf16NegativeNonLastAxisAndZeroPoint) { + RunFp8Bf16Case({2, 2, 2}, -2, {1, 2, 3, 4, -1, -2, -3, -4}, {2}, {2, 3}, {1, -1}, + {0, 2, 12, 15, -4, -6, -6, -9}); +} + +TEST(DequantizeLinearTest, Fp8ToBf16RankOnePerAxis) { + RunFp8Bf16Case({3}, 0, {1, -2, 4}, {3}, {0.5f, 2, 3}, {}, + {0.5f, -4, 12}); +} + +TEST(DequantizeLinearTest, Fp8ToBf16EmptyInput) { + RunFp8Bf16Case({0, 2}, 1, {}, {2}, {1, 2}, {}, {}); +} + +TEST(DequantizeLinearTest, Fp8ToBf16RejectInvalidScales) { + RunFp8Bf16Case({2, 2}, 1, {1, 2, 3, 4}, {3}, {1, 2, 3}, {}, {1, 2, 3, 4}, + "x_scale must match the input dimension along axis"); + RunFp8Bf16Case({2, 2}, 1, {1, 2, 3, 4}, {0}, {}, {}, {1, 2, 3, 4}, + "x_scale must not be empty"); + RunFp8Bf16Case({2, 2}, 1, {1, 2, 3, 4}, {1, 2}, {1, 2}, {}, {1, 2, 3, 4}, + "x_scale must be a scalar or 1D tensor"); +} #endif } // namespace test From 3a8c804e3373657da4268d411e4a5f82c082f6fc Mon Sep 17 00:00:00 2001 From: Akshay Sonawane Date: Mon, 19 Oct 2026 22:15:30 +0000 Subject: [PATCH 03/12] Guard new dequantization contracts in QDQ propagation and CPU dispatch --- docs/OperatorKernels.md | 2 +- .../quantize_dequantize_linear.cc | 2 +- .../qdq_transformer/qdq_propagation.cc | 15 +++++ .../cpu/quantization/quantize_linear.cc | 7 +++ .../cuda_kernels/qmoe_fp8_compaction_test.cc | 30 ++++++---- .../contrib_ops/dequantize_fp8_bf16_test.cc | 56 ++++++++++++++++++ .../test/contrib_ops/quantize_ops_test.cc | 52 +++++++++++++++++ .../test/optimizer/qdq_transformer_test.cc | 57 +++++++++++++++++++ 8 files changed, 207 insertions(+), 14 deletions(-) diff --git a/docs/OperatorKernels.md b/docs/OperatorKernels.md index 9bc7548677773..82bfcfbb59a4a 100644 --- a/docs/OperatorKernels.md +++ b/docs/OperatorKernels.md @@ -581,7 +581,7 @@ The **OpSet Version** column uses the following notation: |ConvTransposeWithDynamicPads|*in* X:**T**
*in* W:**T**
*in* Pads:**tensor(int64)**
*in* B:**T**
*out* Y:**T**|1+|**T** = tensor(float)| |CropAndResize|*in* X:**T1**
*in* rois:**T1**
*in* batch_indices:**T2**
*in* crop_size:**T2**
*out* Y:**T1**|1+|**T1** = tensor(float)
**T2** = tensor(int32)| |DecoderMaskedMultiHeadAttention|*in* query:**T**
*in* key:**T**
*in* value:**T**
*in* mask_index:**M**
*in* attention_bias:**T**
*in* past_key:**T**
*in* past_value:**T**
*in* past_sequence_length:**M**
*in* beam_width:**M**
*in* cache_indirection:**M**
*in* bias:**T**
*out* output:**T**
*out* present_key:**T**
*out* present_value:**T**
*out* qk:**QK**|1+|**T** = tensor(float)| -|DequantizeLinear|*in* x:**T1**
*in* x_scale:**TS**
*in* x_zero_point:**T1**
*out* y:**T2**|1+|**T1** = tensor(int16), tensor(int32), tensor(int4), tensor(int8), tensor(uint16), tensor(uint4), tensor(uint8)
**T2** = tensor(float)| +|DequantizeLinear|*in* x:**T1**
*in* x_scale:**TS**
*in* x_zero_point:**T1**
*out* y:**T2**|1+|**T1** = tensor(int16), tensor(int32), tensor(int4), tensor(int8), tensor(uint16), tensor(uint4), tensor(uint8)
**T2** = tensor(float)
**TS** = tensor(float), tensor(float16)| |DynamicQuantizeLSTM|*in* X:**T**
*in* W:**T2**
*in* R:**T2**
*in* B:**T**
*in* sequence_lens:**T1**
*in* initial_h:**T**
*in* initial_c:**T**
*in* P:**T**
*in* W_scale:**T**
*in* W_zero_point:**T2**
*in* R_scale:**T**
*in* R_zero_point:**T2**
*out* Y:**T**
*out* Y_h:**T**
*out* Y_c:**T**|1+|**T** = tensor(float)
**T1** = tensor(int32)
**T2** = tensor(int8), tensor(uint8)| |DynamicQuantizeMatMul|*in* A:**T1**
*in* B:**T2**
*in* b_scale:**T1**
*in* b_zero_point:**T2**
*in* bias:**T1**
*out* Y:**T1**|1+|**T1** = tensor(float)
**T2** = tensor(int8), tensor(uint8)| |DynamicTimeWarping|*in* input:**F**
*out* output:**I**|1+|**F** = tensor(float)
**I** = tensor(int32)| diff --git a/onnxruntime/contrib_ops/cuda/quantization/quantize_dequantize_linear.cc b/onnxruntime/contrib_ops/cuda/quantization/quantize_dequantize_linear.cc index af94287eb9f50..1d9e9794de2d4 100644 --- a/onnxruntime/contrib_ops/cuda/quantization/quantize_dequantize_linear.cc +++ b/onnxruntime/contrib_ops/cuda/quantization/quantize_dequantize_linear.cc @@ -64,7 +64,7 @@ class DequantizeLinearFp8Bf16 final : public CudaKernel { "x_zero_point and x_scale must have the same shape."); const int64_t scale_count = scale.Shape().Size(); - ORT_RETURN_IF_NOT(scale_count > 0, "x_scale must not be empty."); + ORT_RETURN_IF_NOT(scale_count > 0 || count == 0, "x_scale must not be empty."); int64_t axis_stride = 1; if (scale_count != 1) { const int64_t axis = HandleNegativeAxis(axis_, shape.NumDimensions()); diff --git a/onnxruntime/core/optimizer/qdq_transformer/qdq_propagation.cc b/onnxruntime/core/optimizer/qdq_transformer/qdq_propagation.cc index ab491c134b5e5..f5093dbdc3db5 100644 --- a/onnxruntime/core/optimizer/qdq_transformer/qdq_propagation.cc +++ b/onnxruntime/core/optimizer/qdq_transformer/qdq_propagation.cc @@ -365,6 +365,21 @@ Status PropagateDQForward(Graph& graph, gsl::span node_indices, continue; } + if (dq_node.Domain() == kMSDomain) { + const auto* scale_type = dq_node.InputDefs()[QDQ::InputIndex::SCALE_ID]->TypeAsProto(); + const auto* output_type = dq_node.OutputDefs()[0]->TypeAsProto(); + const auto* input_type = dq_node.InputDefs()[QDQ::InputIndex::INPUT_ID]->TypeAsProto(); + // The MS quantizer cannot invert FP8, BF16, or independently typed scale/output dequantization. + if (graph_utils::GetNodeAttribute(dq_node, "output_dtype") != nullptr || + scale_type == nullptr || output_type == nullptr || input_type == nullptr || + scale_type->tensor_type().elem_type() != output_type->tensor_type().elem_type() || + (scale_type->tensor_type().elem_type() != ONNX_NAMESPACE::TensorProto::FLOAT && + scale_type->tensor_type().elem_type() != ONNX_NAMESPACE::TensorProto::FLOAT16) || + input_type->tensor_type().elem_type() == ONNX_NAMESPACE::TensorProto::FLOAT8E4M3FN) { + continue; + } + } + bool dq_zero_point_exists = false; if (!QDQ::QOrDQNodeHasConstantScalarScaleAndZeroPoint(dq_node, GraphConstantInitializerGetter{graph}, dq_zero_point_exists)) { diff --git a/onnxruntime/core/providers/cpu/quantization/quantize_linear.cc b/onnxruntime/core/providers/cpu/quantization/quantize_linear.cc index 360ad9381e750..8c76ec0a78bdb 100644 --- a/onnxruntime/core/providers/cpu/quantization/quantize_linear.cc +++ b/onnxruntime/core/providers/cpu/quantization/quantize_linear.cc @@ -274,6 +274,7 @@ ONNX_CPU_OPERATOR_TYPED_MS_KERNEL( uint8_t, KernelDefBuilder() .TypeConstraint("T1", DataTypeImpl::GetTensorType()) + .TypeConstraint("TS", {DataTypeImpl::GetTensorType(), DataTypeImpl::GetTensorType()}) .TypeConstraint("T2", DataTypeImpl::GetTensorType()), DequantizeLinear); @@ -283,6 +284,7 @@ ONNX_CPU_OPERATOR_TYPED_MS_KERNEL( int8_t, KernelDefBuilder() .TypeConstraint("T1", DataTypeImpl::GetTensorType()) + .TypeConstraint("TS", {DataTypeImpl::GetTensorType(), DataTypeImpl::GetTensorType()}) .TypeConstraint("T2", DataTypeImpl::GetTensorType()), DequantizeLinear); @@ -292,6 +294,7 @@ ONNX_CPU_OPERATOR_TYPED_MS_KERNEL( uint16_t, KernelDefBuilder() .TypeConstraint("T1", DataTypeImpl::GetTensorType()) + .TypeConstraint("TS", {DataTypeImpl::GetTensorType(), DataTypeImpl::GetTensorType()}) .TypeConstraint("T2", DataTypeImpl::GetTensorType()), DequantizeLinear); @@ -301,6 +304,7 @@ ONNX_CPU_OPERATOR_TYPED_MS_KERNEL( int16_t, KernelDefBuilder() .TypeConstraint("T1", DataTypeImpl::GetTensorType()) + .TypeConstraint("TS", {DataTypeImpl::GetTensorType(), DataTypeImpl::GetTensorType()}) .TypeConstraint("T2", DataTypeImpl::GetTensorType()), DequantizeLinear); @@ -310,6 +314,7 @@ ONNX_CPU_OPERATOR_TYPED_MS_KERNEL( int32_t, KernelDefBuilder() .TypeConstraint("T1", DataTypeImpl::GetTensorType()) + .TypeConstraint("TS", {DataTypeImpl::GetTensorType(), DataTypeImpl::GetTensorType()}) .TypeConstraint("T2", DataTypeImpl::GetTensorType()), DequantizeLinear); @@ -319,6 +324,7 @@ ONNX_CPU_OPERATOR_TYPED_MS_KERNEL( Int4x2, KernelDefBuilder() .TypeConstraint("T1", DataTypeImpl::GetTensorType()) + .TypeConstraint("TS", {DataTypeImpl::GetTensorType(), DataTypeImpl::GetTensorType()}) .TypeConstraint("T2", DataTypeImpl::GetTensorType()), DequantizeLinear); @@ -328,6 +334,7 @@ ONNX_CPU_OPERATOR_TYPED_MS_KERNEL( UInt4x2, KernelDefBuilder() .TypeConstraint("T1", DataTypeImpl::GetTensorType()) + .TypeConstraint("TS", {DataTypeImpl::GetTensorType(), DataTypeImpl::GetTensorType()}) .TypeConstraint("T2", DataTypeImpl::GetTensorType()), DequantizeLinear); diff --git a/onnxruntime/test/contrib_ops/cuda_kernels/qmoe_fp8_compaction_test.cc b/onnxruntime/test/contrib_ops/cuda_kernels/qmoe_fp8_compaction_test.cc index 3888cd665e71e..86b0bb6f83ab8 100644 --- a/onnxruntime/test/contrib_ops/cuda_kernels/qmoe_fp8_compaction_test.cc +++ b/onnxruntime/test/contrib_ops/cuda_kernels/qmoe_fp8_compaction_test.cc @@ -26,6 +26,16 @@ namespace test { namespace { namespace qmoe = contrib::cuda; +#if defined(ORT_TEST_NVFP4_GEMV) || (defined(ENABLE_FP4) && defined(USE_FP4_QMOE)) +bool SupportsSm80OrLater() { + int device = 0; + cudaDeviceProp properties{}; + CUDA_CALL_THROW(cudaGetDevice(&device)); + CUDA_CALL_THROW(cudaGetDeviceProperties(&properties, device)); + return properties.major >= 8; +} +#endif + #ifdef ORT_TEST_NVFP4_GEMV namespace gemv = llm::kernels::moe_gemv; #endif @@ -131,6 +141,9 @@ TEST(CUDA_EP_Unittest, QMoENvfp4SmallDecodeMapsPreserveStableOrder) { template void TestNvfp4KPackedGateUp(int k, bool use_bias, bool varied_scales = false, int n = 48) { + if (!SupportsSm80OrLater()) { + GTEST_SKIP() << "NVFP4 K-packed gate/up requires SM80 or newer"; + } constexpr int experts = 3; constexpr int rows = 4; constexpr std::array routes{2, 0, 2, 1}; @@ -225,6 +238,9 @@ TEST(CUDA_EP_Unittest, QMoENvfp4KPackedGateUpPartialTiles) { template void TestNvfp4FusedFinalize(int num_rows, int topk, int k, bool use_bias) { + if (!SupportsSm80OrLater()) { + GTEST_SKIP() << "NVFP4 fused finalization requires SM80 or newer"; + } constexpr int n = 50, experts = 3; const int expanded = num_rows * topk; std::vector activations(expanded * k), biases(experts * n); @@ -418,16 +434,6 @@ TEST(CUDA_EP_Unittest, QMoEFp8DequantizesOnlySelectedExpertsAndGathersBias) { return code & 128 ? -magnitude : magnitude; } -#if defined(ENABLE_FP4) && defined(USE_FP4_QMOE) -bool SupportsFp8Gemv() { - int device = 0; - cudaDeviceProp properties{}; - CUDA_CALL_THROW(cudaGetDevice(&device)); - CUDA_CALL_THROW(cudaGetDeviceProperties(&properties, device)); - return properties.major >= 8; -} -#endif - template void TestFp8GemvCodeDecoding(int scale_type, int block_size, int k, int fusion, bool graph) { constexpr int n = 256; @@ -668,7 +674,7 @@ void TestNvfp4GemvScaleCodeConfigurations() { } TEST(CUDA_EP_Unittest, QMoENvfp4GemvAllScaleCodesPreserveScaledValuesAndNaNs) { - if (!SupportsFp8Gemv()) { + if (!SupportsSm80OrLater()) { GTEST_SKIP() << "NVFP4 GEMV requires SM80+."; } TestNvfp4GemvScaleCodeConfigurations(); @@ -678,7 +684,7 @@ TEST(CUDA_EP_Unittest, QMoENvfp4GemvNativeScale) { if (!ParseEnvironmentVariableWithDefault("ORT_NVFP4_GEMV_NATIVE_SCALE", false)) { GTEST_SKIP() << "Native-scale coverage runs in its flag-enabled CTest process."; } - if (!SupportsFp8Gemv() || !onnxruntime::llm::kernels::moe_gemv::Fp4MoeGemvSupportsNativeScale()) { + if (!SupportsSm80OrLater() || !onnxruntime::llm::kernels::moe_gemv::Fp4MoeGemvSupportsNativeScale()) { GTEST_SKIP() << "Native E4M3 scale conversion requires FP8-enabled kernels compiled for SM90+."; } TestNvfp4GemvScaleCodeConfigurations(); diff --git a/onnxruntime/test/contrib_ops/dequantize_fp8_bf16_test.cc b/onnxruntime/test/contrib_ops/dequantize_fp8_bf16_test.cc index 172f6bbb98d7f..2face506a20a3 100644 --- a/onnxruntime/test/contrib_ops/dequantize_fp8_bf16_test.cc +++ b/onnxruntime/test/contrib_ops/dequantize_fp8_bf16_test.cc @@ -3,9 +3,16 @@ #include "gtest/gtest.h" #include "core/common/float8.h" +#include "core/graph/model.h" #include "test/common/tensor_op_test_utils.h" #include "test/providers/provider_test_utils.h" +#include "test/unittest_util/graph_transform_test_builder.h" +#include "test/util/include/asserts.h" #include "test/util/include/default_providers.h" +#include "test/util/include/inference_session_wrapper.h" +#include "test/util/include/test_environment.h" + +#include namespace onnxruntime { namespace test { @@ -83,6 +90,55 @@ TEST(DequantizeLinearTest, Fp8ToBf16RankOnePerAxis) { TEST(DequantizeLinearTest, Fp8ToBf16EmptyInput) { RunFp8Bf16Case({0, 2}, 1, {}, {2}, {1, 2}, {}, {}); + RunFp8Bf16Case({0, 2}, 0, {}, {0}, {}, {}, {}); + RunFp8Bf16Case({0, 2}, 1, {}, {0}, {}, {}, {}, + "x_scale must match the input dimension along axis"); +} + +TEST(DequantizeLinearTest, Fp8ToBf16OptimizedReshapeTranspose) { + auto provider = DefaultCudaExecutionProvider(); + if (provider == nullptr) GTEST_SKIP() << "CUDA execution provider is not available"; + std::unordered_map versions{{kOnnxDomain, 19}, {kMSDomain, 1}}; + Model model("Fp8Bf16Optimized", false, ModelMetaData(), PathString(), + IOnnxRuntimeOpSchemaRegistryList(), versions, {}, DefaultLoggingManager().DefaultLogger()); + auto& graph = model.MainGraph(); + ModelTestBuilder builder(graph); + std::vector values; + for (float value : {1.0f, 2.0f, -1.0f, -2.0f}) values.emplace_back(value); + auto* input = builder.MakeInput({4}, values); + auto* scale = builder.MakeScalarInitializer(2.0f); + auto* dequantized = builder.MakeIntermediate(); + builder.AddNode("DequantizeLinear", {input, scale}, {dequantized}, kMSDomain) + .AddAttribute("output_dtype", static_cast(ONNX_NAMESPACE::TensorProto::BFLOAT16)); + auto* reshaped = builder.MakeIntermediate(); + builder.AddNode("Reshape", {dequantized, builder.Make1DInitializer({2, 2})}, {reshaped}); + builder.AddNode("Transpose", {reshaped}, {builder.MakeOutput()}) + .AddAttribute("perm", std::vector{1, 0}); + builder.SetGraphOutputs(); + ASSERT_STATUS_OK(graph.Resolve()); + std::string serialized; + ASSERT_TRUE(model.ToProto().SerializeToString(&serialized)); + SessionOptions options; + options.graph_optimization_level = TransformerLevel::Level3; + InferenceSessionWrapper session(options, GetEnvironment()); + ASSERT_STATUS_OK(session.RegisterExecutionProvider(std::move(provider))); + std::istringstream stream(serialized); + ASSERT_STATUS_OK(session.Load(stream)); + ASSERT_STATUS_OK(session.Initialize()); + for (const auto& node : session.GetGraph().Nodes()) { + EXPECT_NE(node.OpType(), "QuantizeLinear"); + } + std::vector fetches; + ASSERT_STATUS_OK(session.Run(RunOptions{}, builder.feeds_, builder.output_names_, &fetches)); + ASSERT_EQ(fetches.size(), 1u); + const auto& output = fetches[0].Get(); + ASSERT_EQ(output.Shape(), TensorShape({2, 2})); + const auto actual = output.DataAsSpan(); + const auto expected = ToBFloat16({2, -2, 4, -4}); + ASSERT_EQ(actual.size(), expected.size()); + for (size_t i = 0; i < actual.size(); ++i) { + EXPECT_EQ(actual[i], expected[i]); + } } TEST(DequantizeLinearTest, Fp8ToBf16RejectInvalidScales) { diff --git a/onnxruntime/test/contrib_ops/quantize_ops_test.cc b/onnxruntime/test/contrib_ops/quantize_ops_test.cc index f3bf09c9533d1..3baf97a70c86a 100644 --- a/onnxruntime/test/contrib_ops/quantize_ops_test.cc +++ b/onnxruntime/test/contrib_ops/quantize_ops_test.cc @@ -4,11 +4,63 @@ #include "gtest/gtest.h" #include "test/common/tensor_op_test_utils.h" #include "test/providers/provider_test_utils.h" +#include "test/unittest_util/graph_transform_test_builder.h" +#include "test/util/include/asserts.h" #include "test/util/include/default_providers.h" +#include "test/util/include/inference_session_wrapper.h" +#include "test/util/include/test_environment.h" + +#include namespace onnxruntime { namespace test { +TEST(DequantizeLinearOpTest, CpuScaleTypeConstraints) { + for (int scale_type : {ONNX_NAMESPACE::TensorProto::FLOAT, ONNX_NAMESPACE::TensorProto::FLOAT16, + ONNX_NAMESPACE::TensorProto::BFLOAT16}) { + std::unordered_map versions{{kOnnxDomain, 19}, {kMSDomain, 1}}; + Model model("DequantizeLinearCpuScales", false, ModelMetaData(), PathString(), + IOnnxRuntimeOpSchemaRegistryList(), versions, {}, DefaultLoggingManager().DefaultLogger()); + auto& graph = model.MainGraph(); + ModelTestBuilder builder(graph); + auto* input = builder.MakeInput({2}, {-2, 4}); + NodeArg* scale; + if (scale_type == ONNX_NAMESPACE::TensorProto::FLOAT) { + scale = builder.MakeScalarInitializer(0.5f); + } else if (scale_type == ONNX_NAMESPACE::TensorProto::FLOAT16) { + scale = builder.MakeScalarInitializer(MLFloat16(0.5f)); + } else { + scale = builder.MakeScalarInitializer(BFloat16(0.5f)); + } + builder.AddNode("DequantizeLinear", {input, scale}, {builder.MakeOutput()}, kMSDomain) + .AddAttribute("output_dtype", static_cast(ONNX_NAMESPACE::TensorProto::FLOAT)); + builder.SetGraphOutputs(); + ASSERT_STATUS_OK(graph.Resolve()); + std::string serialized; + ASSERT_TRUE(model.ToProto().SerializeToString(&serialized)); + SessionOptions options; + options.graph_optimization_level = TransformerLevel::Default; + InferenceSessionWrapper session(options, GetEnvironment()); + ASSERT_STATUS_OK(session.RegisterExecutionProvider(DefaultCpuExecutionProvider())); + std::istringstream stream(serialized); + ASSERT_STATUS_OK(session.Load(stream)); + const auto status = session.Initialize(); + if (scale_type == ONNX_NAMESPACE::TensorProto::BFLOAT16) { + EXPECT_EQ(status.Code(), common::NOT_IMPLEMENTED) << status.ErrorMessage(); + EXPECT_NE(status.ErrorMessage().find("DequantizeLinear"), std::string::npos); + } else { + ASSERT_STATUS_OK(status); + std::vector fetches; + ASSERT_STATUS_OK(session.Run(RunOptions{}, builder.feeds_, builder.output_names_, &fetches)); + ASSERT_EQ(fetches.size(), 1u); + const auto actual = fetches[0].Get().DataAsSpan(); + ASSERT_EQ(actual.size(), 2u); + EXPECT_EQ(actual[0], -1.0f); + EXPECT_EQ(actual[1], 2.0f); + } + } +} + // scalar zero & scale with uint8 void TestDequantizeLinearPerTensorFloatUint8(bool use_initializer_except_x) { OpTester test("DequantizeLinear", 1, onnxruntime::kMSDomain); diff --git a/onnxruntime/test/optimizer/qdq_transformer_test.cc b/onnxruntime/test/optimizer/qdq_transformer_test.cc index e6139363ade79..4b263a39fc767 100644 --- a/onnxruntime/test/optimizer/qdq_transformer_test.cc +++ b/onnxruntime/test/optimizer/qdq_transformer_test.cc @@ -3,6 +3,7 @@ #include #include +#include "core/common/float8.h" #include "core/common/inlined_containers_fwd.h" #include "core/common/span_utils.h" #include "core/framework/compute_capability.h" @@ -4586,6 +4587,62 @@ TEST(QDQTransformerTests, QDQPropagation_DQForward_ConstantInput_NoPropagation) } } +TEST(QDQTransformerTests, QDQPropagation_MSIndependentScaleOutput_NoPropagation) { + for (bool explicit_output_dtype : {false, true}) { + auto build_test_case = [=](ModelTestBuilder& builder) { + auto* input = builder.MakeInput({4}, {-2, -1, 1, 2}); + NodeArg* scale = explicit_output_dtype + ? builder.MakeScalarInitializer(0.5f) + : builder.MakeScalarInitializer(BFloat16(0.5f)); + auto* dequantized = builder.MakeIntermediate(); + auto& dq = builder.AddNode("DequantizeLinear", {input, scale}, {dequantized}, kMSDomain); + if (explicit_output_dtype) { + dq.AddAttribute("output_dtype", static_cast(ONNX_NAMESPACE::TensorProto::FLOAT16)); + } + auto* reshaped = builder.MakeIntermediate(); + builder.AddNode("Reshape", {dequantized, builder.Make1DInitializer({2, 2})}, {reshaped}); + builder.AddNode("Transpose", {reshaped}, {builder.MakeOutput()}) + .AddAttribute("perm", std::vector{1, 0}); + }; + auto check_graph = [](Graph& graph) -> Status { + const auto counts = CountOpsInGraph(graph); + TEST_RETURN_IF_NOT(counts.count("com.microsoft.QuantizeLinear") == 0); + TEST_RETURN_IF_NOT(counts.at("com.microsoft.DequantizeLinear") == 1); + TEST_RETURN_IF_NOT(counts.at("Reshape") == 1 && counts.at("Transpose") == 1); + return graph.Resolve(); + }; + ASSERT_STATUS_OK(TestGraphTransformer(build_test_case, 19, DefaultLoggingManager().DefaultLogger(), + std::make_unique(), + TransformerLevel::Level1, 2, nullptr, check_graph)); + } +} + +#if !defined(DISABLE_FLOAT8_TYPES) +TEST(QDQTransformerTests, QDQPropagation_MSFp8Bf16_NoPropagation) { + auto build_test_case = [](ModelTestBuilder& builder) { + auto* input = builder.MakeInput({4}, std::vector(4, Float8E4M3FN(1.0f))); + auto* scale = builder.MakeScalarInitializer(0.5f); + auto* dequantized = builder.MakeIntermediate(); + builder.AddNode("DequantizeLinear", {input, scale}, {dequantized}, kMSDomain) + .AddAttribute("output_dtype", static_cast(ONNX_NAMESPACE::TensorProto::BFLOAT16)); + auto* reshaped = builder.MakeIntermediate(); + builder.AddNode("Reshape", {dequantized, builder.Make1DInitializer({2, 2})}, {reshaped}); + builder.AddNode("Transpose", {reshaped}, {builder.MakeOutput()}) + .AddAttribute("perm", std::vector{1, 0}); + }; + auto check_graph = [](Graph& graph) -> Status { + const auto counts = CountOpsInGraph(graph); + TEST_RETURN_IF_NOT(counts.count("com.microsoft.QuantizeLinear") == 0); + TEST_RETURN_IF_NOT(counts.at("com.microsoft.DequantizeLinear") == 1); + TEST_RETURN_IF_NOT(counts.at("Reshape") == 1 && counts.at("Transpose") == 1); + return graph.Resolve(); + }; + ASSERT_STATUS_OK(TestGraphTransformer(build_test_case, 19, DefaultLoggingManager().DefaultLogger(), + std::make_unique(), + TransformerLevel::Level1, 2, nullptr, check_graph)); +} +#endif + TEST(QDQTransformerTests, QDQPropagation_StopAtOtherQDQ) { auto test_case = [&](const std::vector& input_shape, bool same_scale, bool same_zp, bool use_contrib_qdq) { From 63026c2c0b20809789482a9c9c50f0e0d27166a4 Mon Sep 17 00:00:00 2001 From: Akshay Sonawane Date: Mon, 19 Oct 2026 23:50:58 +0000 Subject: [PATCH 04/12] Increase fused NVFP4 SwiGLU reduction loop unroll factor --- onnxruntime/contrib_ops/cuda/llm/moe_gemm/moe_gemv_fp4.cu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onnxruntime/contrib_ops/cuda/llm/moe_gemm/moe_gemv_fp4.cu b/onnxruntime/contrib_ops/cuda/llm/moe_gemm/moe_gemv_fp4.cu index dbe438cc16a7c..039171c7c514a 100644 --- a/onnxruntime/contrib_ops/cuda/llm/moe_gemm/moe_gemv_fp4.cu +++ b/onnxruntime/contrib_ops/cuda/llm/moe_gemm/moe_gemv_fp4.cu @@ -376,7 +376,7 @@ __global__ void MoeGemvFp4RawKPackedKernel( constexpr int kKLanes = KLanes; constexpr int kBlockSize = 16; constexpr int kColsPerBlock = FusedFinalize ? 8 : 128 / kKLanes; - constexpr int kReductionUnroll = FusedSwiGlu ? 2 : 1; + constexpr int kReductionUnroll = FusedSwiGlu ? 4 : 1; static_assert(kKLanes == 4 || kKLanes == 8); constexpr unsigned kFullMask = 0xffffffffu; using PackedT = std::conditional_t, half2, __nv_bfloat162>; From 25d286b5a0acb04957e81896b8bfb0e28e89b235 Mon Sep 17 00:00:00 2001 From: Akshay Sonawane Date: Tue, 20 Oct 2026 22:08:01 +0000 Subject: [PATCH 05/12] Share packed INT2 MoE weights and support offline single-copy layouts Enable compatible 512-expert/top-10 packed execution, keep shared allocations independent of session allocators, and consume explicitly prepacked INT2 initializers directly. Reject raw fallback for prepacked bytes and document the layout contract. Add large-expert parity and unsupported-prepacked regression coverage. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/contrib_ops/cuda/moe_qmoe.md | 38 ++++++ .../contrib_ops/cuda/llm/moe_gemm/moe_gemv.cu | 7 +- .../contrib_ops/cuda/moe/moe_quantization.cc | 128 +++++++++++++++--- .../contrib_ops/cuda/moe/moe_quantization.h | 7 +- .../core/graph/contrib_ops/contrib_defs.cc | 4 + onnxruntime/test/contrib_ops/moe_test.cc | 58 ++++++-- 6 files changed, 204 insertions(+), 38 deletions(-) diff --git a/docs/contrib_ops/cuda/moe_qmoe.md b/docs/contrib_ops/cuda/moe_qmoe.md index 73682f6a237c6..0ad4bd6ad48de 100644 --- a/docs/contrib_ops/cuda/moe_qmoe.md +++ b/docs/contrib_ops/cuda/moe_qmoe.md @@ -843,6 +843,44 @@ When native CUTLASS is enabled, weights and scales are dual-prepacked (native la raw e8m0 block scales for GEMV), so `ORT_FP4_PREFILL_MIN_TOKENS` selects per call between the prefill and decode kernels with no extra conversion. +### Large-expert INT2 execution and single-copy packed weights + +Compatible INT2 execution supports up to 512 experts and top-10 routing by +default. `ORT_QMOE_INT2_LARGE_EXPERTS=0` restores the previous 256-expert +eligibility. Packed INT2 GEMV accepts up to 20 expanded rows, covering one-token +decode and two-token verification with top-10 routing. Existing GPU, alignment, +block-size, symmetric-weight and fused-SwiGLU checks remain in force. Larger +calls use supported packed grouped GEMM; raw inputs retain the bounded dense +fallback for unsupported configurations. + +For raw INT2 initializers (`weights_prepacked=0`), the runtime automatically +shares packed weights between sessions retaining the same immutable GPU +initializer. `ORT_QMOE_INT2_SHARE_PACKED=0` disables this cache. Keys include CUDA +device, buffer identity, dimensions and bit width. Publication follows stream +synchronization; weak entries do not extend ownership. Shared CUDA allocations +are independent of session allocators, so the first session may be destroyed +before surviving users. Callers must preserve the original initializer lifetime +and immutability required by `AddInitializer`. CPU initializers and other bit +widths retain their existing allocation behavior. + +Uniform INT2 can instead set **explicit** `weights_prepacked=1` and provide the +SM80 CUTLASS `W2_A16` bytes produced by `PrePackIntExpertWeights`, retaining +logical `[E,N,K/4]` dimensions. Scales remain activation-dtype +`[E,N,K/block_size]`. Packed GEMV and grouped GEMM read these initializer tensors +directly without another packed expert allocation. Shared GPU `AddInitializer` +tensors therefore let prefill and decode sessions read one pool-owned weight +representation. The pool owner must outlive all sessions. CPU initializers use +normal ORT device transfers; cross-session sharing requires shared GPU tensors. +Scale transposes and execution workspaces remain per session. + +Unsupported prepacked INT2 configurations fail explicitly: the raw dequantization +fallback cannot interpret CUTLASS-packed bytes. This includes disabling both +packed paths, unsupported shapes/hardware, zero points or non-fused activations. +Mixed-width offline prepacking and implicit `weights_prepacked=-1` for INT2 are +not supported. Sharing alone removes duplicate packed copies, not the retained +raw initializer; offline prepacking is necessary to eliminate raw-plus-packed +GPU duplication. + ### 9.10 Interleaved GEMV layout + dtype-conditional accumulation **Opt-in (`ORT_FP4_GEMV_INTERLEAVED=1`, default off).** The shipping MXFP4 decode GEMV uses a diff --git a/onnxruntime/contrib_ops/cuda/llm/moe_gemm/moe_gemv.cu b/onnxruntime/contrib_ops/cuda/llm/moe_gemm/moe_gemv.cu index 36b4435a86c76..168e7980c5560 100644 --- a/onnxruntime/contrib_ops/cuda/llm/moe_gemm/moe_gemv.cu +++ b/onnxruntime/contrib_ops/cuda/llm/moe_gemm/moe_gemv.cu @@ -944,7 +944,12 @@ bool is_moe_gemv_shape_supported(int sm, int64_t expanded_num_rows, int64_t n, i if (group_size > 0 && k % group_size != 0) { return false; } - if (expanded_num_rows <= 0 || expanded_num_rows > kMaxProfiledExpandedRows) { + const int64_t max_expanded_rows = + weight_bits == 2 && + onnxruntime::ParseEnvironmentVariableWithDefault("ORT_QMOE_INT2_LARGE_EXPERTS", 1) == 1 + ? 20 + : kMaxProfiledExpandedRows; + if (expanded_num_rows <= 0 || expanded_num_rows > max_expanded_rows) { return false; } if (n < kMinProfiledProblemDim || k < kMinProfiledProblemDim) { diff --git a/onnxruntime/contrib_ops/cuda/moe/moe_quantization.cc b/onnxruntime/contrib_ops/cuda/moe/moe_quantization.cc index b0cea76b5514b..ae224961905aa 100644 --- a/onnxruntime/contrib_ops/cuda/moe/moe_quantization.cc +++ b/onnxruntime/contrib_ops/cuda/moe/moe_quantization.cc @@ -42,7 +42,9 @@ #include #include #include +#include #include +#include using namespace onnxruntime::cuda; using namespace ::onnxruntime::common; @@ -362,8 +364,10 @@ QMoE::QMoE(const OpKernelInfo& op_kernel_info) : CudaKernel(op_kernel_info), MoE "weights_prepacked must be -1 (default), 0, or 1, but got ", weights_prepacked_mode); weights_prepacked_ = (weights_prepacked_mode != 0); if (use_int_dequant_fallback) { - ORT_ENFORCE(!weights_prepacked_, - "INT2 or mixed-width CUDA QMoE requires raw weights with weights_prepacked=0."); + ORT_ENFORCE(!weights_prepacked_ || + (weights_prepacked_mode == 1 && expert_weight_bits_ == 2 && !is_mixed_width), + "INT2 or mixed-width CUDA QMoE requires raw weights with weights_prepacked=0, " + "or uniform INT2 SM80 CUTLASS weights with explicit weights_prepacked=1."); ORT_ENFORCE(block_size_ >= 16 && block_size_ <= 256 && (block_size_ & (block_size_ - 1)) == 0, "INT2 or mixed-width CUDA QMoE requires block_size to be a power of two in [16, 256], got ", @@ -398,6 +402,8 @@ QMoE::QMoE(const OpKernelInfo& op_kernel_info) : CudaKernel(op_kernel_info), MoE (fc2_expert_weight_bits_ == 2 || fc2_expert_weight_bits_ == 4) && (fc1_expert_weight_bits_ == 2 || fc2_expert_weight_bits_ == 2) && onnxruntime::ParseEnvironmentVariableWithDefault("ORT_ENABLE_QMOE_INT2_PREFILL", 1) != 0; + enable_int2_large_experts_ = + onnxruntime::ParseEnvironmentVariableWithDefault("ORT_QMOE_INT2_LARGE_EXPERTS", 1) == 1; fp4_deep_gemm_num_experts_ = StaticFp4DeepGemmNumExperts(op_kernel_info); enable_fp4_deep_gemm_ = @@ -759,6 +765,15 @@ Status QMoE::ComputeInternal(OpKernelContext* context) const { const Tensor* fc1_experts_bias_optional = context->Input(4); const Tensor* fc2_experts_weights = weights_consumed_by_prepack ? nullptr : context->Input(5); const Tensor* fc2_scales = (is_int && !packed_fc2_scales_) ? context->Input(6) : nullptr; + const bool preloaded_int2 = is_int && expert_weight_bits_ == 2 && weights_prepacked_; + if (preloaded_int2) { + ORT_RETURN_IF_NOT(fc1_experts_weights && fc2_experts_weights && + fc1_experts_weights->IsDataType() && + fc2_experts_weights->IsDataType(), + "Prepacked INT2 requires retained UINT8 fc1/fc2 initializers."); + } + const void* packed_int_fc1 = preloaded_int2 ? fc1_experts_weights->DataRaw() : packed_fc1_weights_.get(); + const void* packed_int_fc2 = preloaded_int2 ? fc2_experts_weights->DataRaw() : packed_fc2_weights_.get(); if (is_int) { for (const auto& [input_idx, scales] : std::array{ std::pair{3, fc1_scales}, @@ -1200,11 +1215,13 @@ Status QMoE::ComputeInternal(OpKernelContext* context) const { sm_, packed_int_expanded, moe_params.hidden_size, moe_params.inter_size, static_cast(fc2_expert_weight_bits_), static_cast(block_size_)); const bool packed_int_routing_supported = - moe_params.num_rows <= 256 && moe_params.num_experts <= 256 && - (k_ == 1 || k_ == 2 || k_ == 4 || k_ == 6 || k_ == 8); + moe_params.num_rows <= 256 && + moe_params.num_experts <= (enable_int2_large_experts_ ? 512 : 256) && + (k_ == 1 || k_ == 2 || k_ == 4 || k_ == 6 || k_ == 8 || + (enable_int2_large_experts_ && k_ == 10)); if (enable_int2_gemv_ && !has_any_zero_point && is_fused_swiglu && swiglu_fusion == 1 && - (block_size_ == 64 || block_size_ == 128) && packed_fc1_weights_ != nullptr && - packed_fc2_weights_ != nullptr) { + (block_size_ == 64 || block_size_ == 128) && packed_int_fc1 != nullptr && + packed_int_fc2 != nullptr) { use_packed_int_gemv = packed_int_fc1_shape_supported && packed_int_fc2_shape_supported && packed_int_routing_supported; } @@ -1216,20 +1233,21 @@ Status QMoE::ComputeInternal(OpKernelContext* context) const { << " block_size=" << block_size_ << " fc1_bits=" << fc1_expert_weight_bits_ << " fc2_bits=" << fc2_expert_weight_bits_ - << " fc1_packed=" << (packed_fc1_weights_ != nullptr) - << " fc2_packed=" << (packed_fc2_weights_ != nullptr) + << " fc1_packed=" << (packed_int_fc1 != nullptr) + << " fc2_packed=" << (packed_int_fc2 != nullptr) << " fc1_shape=" << packed_int_fc1_shape_supported << " fc2_shape=" << packed_int_fc2_shape_supported << " routing=" << packed_int_routing_supported; } const bool use_packed_int_prefill = enable_int2_prefill_ && !use_packed_int_gemv && !has_any_zero_point && is_fused_swiglu && - swiglu_fusion == 1 && packed_fc1_weights_ != nullptr && packed_fc2_weights_ != nullptr && + swiglu_fusion == 1 && packed_int_fc1 != nullptr && packed_int_fc2 != nullptr && moe_params.num_rows > 0 && packed_int_expanded <= std::numeric_limits::max() && moe_params.hidden_size > 0 && moe_params.hidden_size <= std::numeric_limits::max() && moe_params.hidden_size % 64 == 0 && moe_params.inter_size > 0 && moe_params.inter_size <= std::numeric_limits::max() / 2 && moe_params.inter_size % 64 == 0 && - moe_params.num_experts > 0 && moe_params.num_experts <= 256; + moe_params.num_experts > 0 && + moe_params.num_experts <= (enable_int2_large_experts_ ? 512 : 256); int64_t effective_row_tile_size = row_tile_size_; size_t packed_prefill_workspace_bytes = 0; if (use_packed_int_prefill) { @@ -1270,6 +1288,10 @@ Status QMoE::ComputeInternal(OpKernelContext* context) const { } } const bool use_packed_int = use_packed_int_gemv || use_packed_int_prefill; + ORT_RETURN_IF(preloaded_int2 && !use_packed_int, + "Prepacked INT2 requires a supported packed GEMV or grouped-GEMM configuration. " + "Raw-weight dequantization fallback cannot read SM80 CUTLASS bytes. " + "Check GPU architecture, dimensions, block_size, routing, zero_points and packed-path overrides."); const bool use_fp8_fused = is_block_fp8 && enable_fp8_fused_ && (kernel_activation_type == ActivationType::Silu || kernel_activation_type == ActivationType::Swiglu) && @@ -2045,8 +2067,8 @@ Status QMoE::ComputeInternal(OpKernelContext* context) const { namespace ck = onnxruntime::llm::kernels::cutlass_kernels; ck::Int2MoePrefillParams params; params.is_bf16 = !is_fp16_; - params.fc1_weights = static_cast(packed_fc1_weights_.get()); - params.fc2_weights = static_cast(packed_fc2_weights_.get()); + params.fc1_weights = static_cast(packed_int_fc1); + params.fc2_weights = static_cast(packed_int_fc2); params.fc1_scales = p_fc1_scales; params.fc2_scales = p_fc2_scales; params.fc1_bias = fc1_experts_bias_optional ? fc1_experts_bias_optional->DataRaw() : nullptr; @@ -2158,7 +2180,7 @@ Status QMoE::ComputeInternal(OpKernelContext* context) const { using WeightType = std::remove_pointer_t; gemv::launch_moe_gemv_int_symmetric_interleaved_swiglu( reinterpret_cast(input->DataRaw()), - static_cast(packed_fc1_weights_.get()), + static_cast(packed_int_fc1), static_cast(p_fc1_scales), fc1_bias, fc1_output, p_efto, p_exp, num_experts, expanded, inter, hidden, static_cast(block_size_), sm_, act_params, p_r2u, num_rows, nullptr, stream); @@ -2166,7 +2188,7 @@ Status QMoE::ComputeInternal(OpKernelContext* context) const { auto launch_fc2 = [&](auto* weight_type) { using WeightType = std::remove_pointer_t; gemv::launch_moe_gemv_int_symmetric( - fc1_output, static_cast(packed_fc2_weights_.get()), + fc1_output, static_cast(packed_int_fc2), static_cast(p_fc2_scales), fc2_bias, fc2_output, p_efto, p_exp, num_experts, expanded, hidden, inter, static_cast(block_size_), sm_, stream); }; @@ -2935,7 +2957,7 @@ Status QMoE::PrePack(const Tensor& tensor, int input_idx, AllocatorPtr alloc, } const int64_t reduction_size = shape[2] * pack_factor; - if (enable_int2_prefill_ && shape[0] > 0 && shape[0] <= 256 && + if (enable_int2_prefill_ && shape[0] > 0 && shape[0] <= (enable_int2_large_experts_ ? 512 : 256) && shape[1] > 0 && shape[1] <= std::numeric_limits::max() && shape[1] % 64 == 0 && reduction_size > 0 && reduction_size <= std::numeric_limits::max() && reduction_size % 64 == 0) { return true; @@ -3157,7 +3179,7 @@ Status QMoE::PrePack(const Tensor& tensor, int input_idx, AllocatorPtr alloc, } } else if (quant_type_ == "int") { if (uses_packed_int_decode) { - if (packed_fc1_weights_) { + if (packed_fc1_weights_ || (expert_weight_bits_ == 2 && weights_prepacked_)) { bool local_packed = false; PrePackTransposeAndPack(tensor, stream, alloc, gemv_int_fc1_scales_, local_packed); } @@ -3208,7 +3230,7 @@ Status QMoE::PrePack(const Tensor& tensor, int input_idx, AllocatorPtr alloc, } } else if (quant_type_ == "int") { if (uses_packed_int_decode) { - if (packed_fc2_weights_) { + if (packed_fc2_weights_ || (expert_weight_bits_ == 2 && weights_prepacked_)) { bool local_packed = false; PrePackTransposeAndPack(tensor, stream, alloc, gemv_int_fc2_scales_, local_packed); } @@ -3356,7 +3378,7 @@ void QMoE::PrePackCopyToGpu(const Tensor& tensor, cudaStream_t stream, Allocator // quantized weight initializer; output is a GPU buffer in the // kernel-expected ``[E, K, N/(8/bits)]`` layout. void QMoE::PrePackIntExpertWeights(const Tensor& tensor, cudaStream_t stream, AllocatorPtr alloc, - IAllocatorUniquePtr& packed_buf, bool& is_packed, + std::shared_ptr& packed_buf, bool& is_packed, int64_t weight_bits) { ORT_ENFORCE(weight_bits == 2 || weight_bits == 4 || weight_bits == 8, "PrePackIntExpertWeights: only 2, 4, and 8 bits are supported, got ", weight_bits); @@ -3375,6 +3397,39 @@ void QMoE::PrePackIntExpertWeights(const Tensor& tensor, cudaStream_t stream, Al const int64_t k_packed = shape[2]; const int64_t k = k_packed * pack_factor; + // Only the retained GPU raw INT2 inputs provide a stable identity shared across sessions. + // Weak entries do not extend model/allocator lifetime, and publication follows stream completion. + using CacheKey = std::tuple; + static std::mutex cache_mutex; + static std::map> cache; + const bool share = expert_weight_bits_ == 2 && weight_bits == 2 && + tensor.Location().device.Type() == OrtDevice::GPU && + onnxruntime::ParseEnvironmentVariableWithDefault("ORT_QMOE_INT2_SHARE_PACKED", 1) == 1; + std::unique_lock cache_lock(cache_mutex, std::defer_lock); + int device = -1; + if (share) { + CUDA_CALL_THROW(cudaGetDevice(&device)); + cache_lock.lock(); + for (auto it = cache.begin(); it != cache.end();) { + if (it->second.expired()) { + it = cache.erase(it); + } else { + ++it; + } + } + } + const CacheKey cache_key{device, tensor.DataRaw(), num_experts, n, k, bits}; + if (share) { + auto it = cache.find(cache_key); + if (it != cache.end()) { + if (auto existing = it->second.lock()) { + packed_buf = std::move(existing); + is_packed = true; + return; + } + } + } + // The CUDA QMoE int4/int8 MoE GEMM always dispatches to the Ampere (SM80) // grouped-GEMM kernel -- even on SM90 -- because mixed int-weight + fp16/bf16 // is not a valid Hopper TMA warp-specialized specialisation. The kernel thus @@ -3391,8 +3446,37 @@ void QMoE::PrePackIntExpertWeights(const Tensor& tensor, cudaStream_t stream, Al // Output buffer holds all E prepacked experts back-to-back in // [E, K, N/pack_factor] layout. - packed_buf = IAllocator::MakeUniquePtr(alloc, total_bytes, /*use_reserve=*/true); - int8_t* dst_all = reinterpret_cast(packed_buf.get()); + std::shared_ptr allocation; + if (share) { + // A shared buffer can outlive the session allocator that first packed it. + void* pointer = nullptr; + CUDA_CALL_THROW(cudaMalloc(&pointer, total_bytes)); + allocation = std::shared_ptr(pointer, [device](void* buffer) { + int previous = -1; + const cudaError_t current = cudaGetDevice(&previous); + if (current != cudaSuccess) { + LOGS_DEFAULT(ERROR) << "QMoE shared INT2 cleanup cudaGetDevice: " << cudaGetErrorString(current); + } + const cudaError_t selected = cudaSetDevice(device); + if (selected != cudaSuccess) { + LOGS_DEFAULT(ERROR) << "QMoE shared INT2 cleanup cudaSetDevice: " << cudaGetErrorString(selected); + return; + } + const cudaError_t freed = cudaFree(buffer); + if (freed != cudaSuccess) { + LOGS_DEFAULT(ERROR) << "QMoE shared INT2 cleanup cudaFree: " << cudaGetErrorString(freed); + } + if (current == cudaSuccess && previous != device) { + const cudaError_t restored = cudaSetDevice(previous); + if (restored != cudaSuccess) { + LOGS_DEFAULT(ERROR) << "QMoE shared INT2 cleanup restore device: " << cudaGetErrorString(restored); + } + } + }); + } else { + allocation = std::shared_ptr(IAllocator::MakeUniquePtr(alloc, total_bytes, /*use_reserve=*/true)); + } + int8_t* dst_all = reinterpret_cast(allocation.get()); // Two transient per-expert scratch buffers reused across experts. IAllocatorUniquePtr transposed_scratch = @@ -3455,6 +3539,10 @@ void QMoE::PrePackIntExpertWeights(const Tensor& tensor, cudaStream_t stream, Al // per-expert transform (and the CPU->GPU staging copy above) is complete, so // the transient scratch buffers are safe to free on return. CUDA_CALL_THROW(cudaStreamSynchronize(stream)); + packed_buf = std::move(allocation); + if (share) { + cache[cache_key] = packed_buf; + } is_packed = true; } diff --git a/onnxruntime/contrib_ops/cuda/moe/moe_quantization.h b/onnxruntime/contrib_ops/cuda/moe/moe_quantization.h index c77bf8a27c389..a63bfc2e148ca 100644 --- a/onnxruntime/contrib_ops/cuda/moe/moe_quantization.h +++ b/onnxruntime/contrib_ops/cuda/moe/moe_quantization.h @@ -68,7 +68,7 @@ class QMoE final : public CudaKernel, public MoEBase { // ``[E, N, K / (8 / bits)]`` weight initializer; ``packed_buf`` receives a // GPU buffer in the kernel-expected ``[E, K, N / (8 / bits)]`` layout. void PrePackIntExpertWeights(const Tensor& tensor, cudaStream_t stream, AllocatorPtr alloc, - IAllocatorUniquePtr& packed_buf, bool& is_packed, int64_t weight_bits); + std::shared_ptr& packed_buf, bool& is_packed, int64_t weight_bits); int64_t expert_weight_bits_; int64_t fc1_expert_weight_bits_; int64_t fc2_expert_weight_bits_; @@ -115,6 +115,7 @@ class QMoE final : public CudaKernel, public MoEBase { bool enable_kernel_debug_info_ = false; bool enable_int2_gemv_ = false; bool enable_int2_prefill_ = false; + bool enable_int2_large_experts_ = false; int64_t row_tile_size_ = qmoe::kDisabledRowTileSize; int64_t int_dequant_max_scratch_bytes_ = int64_t{1} << 30; @@ -139,8 +140,8 @@ class QMoE final : public CudaKernel, public MoEBase { // pre-prepack the weights offline using ``pack_weights_for_cuda_mixed_gemm``, // which is asymmetric with how ``MatMulNBits`` is consumed and forces // a CUDA-enabled ORT build for any offline quantization tooling. - IAllocatorUniquePtr packed_fc1_weights_; - IAllocatorUniquePtr packed_fc2_weights_; + std::shared_ptr packed_fc1_weights_; + std::shared_ptr packed_fc2_weights_; IAllocatorUniquePtr packed_fc1_scales_; IAllocatorUniquePtr packed_fc1_bias_; IAllocatorUniquePtr packed_fc2_scales_; diff --git a/onnxruntime/core/graph/contrib_ops/contrib_defs.cc b/onnxruntime/core/graph/contrib_ops/contrib_defs.cc index 477b8b2b6dd69..1acf856971369 100644 --- a/onnxruntime/core/graph/contrib_ops/contrib_defs.cc +++ b/onnxruntime/core/graph/contrib_ops/contrib_defs.cc @@ -1604,6 +1604,10 @@ ONNX_MS_OPERATOR_SET_SCHEMA( "Expert-weight layout selector with quantization-specific meanings. For quant_type='int', " "-1 and 1 select execution-provider-specific prepacked int4/int8 fc1/fc2 layouts. " "0 selects raw [E,N,K/pack] tensors as produced by quantize_matmul_{4,8}bits. " + "Uniform INT2 on CUDA also accepts explicit 1 for SM80 CUTLASS W2_A16 bytes, " + "retaining logical [E,N,K/4] dimensions and ordinary block scales. It requires " + "supported packed GEMV/grouped-GEMM execution; raw fallback is unavailable. " + "Mixed-width CUDA INT execution requires 0; INT2 does not accept -1. " "For quant_type='nvfp4' on CUDA, -1 (default) and 0 select legacy N-packed [E,K,N/2] bytes; " "1 selects K-packed row-major [E,N,K/2] bytes while retaining logical [E,K,N/2] dimensions. " "NVFP4 block scales and global scales retain their original layouts and values. " diff --git a/onnxruntime/test/contrib_ops/moe_test.cc b/onnxruntime/test/contrib_ops/moe_test.cc index 8864069727f84..72c40e7042b25 100644 --- a/onnxruntime/test/contrib_ops/moe_test.cc +++ b/onnxruntime/test/contrib_ops/moe_test.cc @@ -2029,7 +2029,9 @@ static void RunQMoEMixedWidthCudaIdentityTest(int64_t fc1_bits, int64_t fc2_bits bool expect_scratch_failure = true, bool use_initializers = false, int64_t num_rows = 1, - int64_t row_tile_size = 0) { + int64_t row_tile_size = 0, + int64_t weights_prepacked = 0, + const char* expected_error = "") { constexpr int64_t num_experts = 1; auto make_identity = [block_size](int64_t bits, int64_t rows, int64_t columns, @@ -2094,7 +2096,7 @@ static void RunQMoEMixedWidthCudaIdentityTest(int64_t fc1_bits, int64_t fc2_bits if (fc2_bits != expert_bits) { tester.AddAttribute("fc2_expert_weight_bits", fc2_bits); } - tester.AddAttribute("weights_prepacked", 0); + tester.AddAttribute("weights_prepacked", weights_prepacked); tester.AddAttribute("block_size", block_size); if (use_bf16) { tester.AddInput("input", {num_rows, hidden_size}, ToBFloat16(input)); @@ -2157,10 +2159,14 @@ static void RunQMoEMixedWidthCudaIdentityTest(int64_t fc1_bits, int64_t fc2_bits ASSERT_STATUS_OK(session_options.config_options.AddConfigEntry( "ep.cuda.qmoe_row_tile_size", std::to_string(row_tile_size).c_str())); } + const bool scratch_failure = max_scratch_bytes > 0 && expect_scratch_failure; + const bool expect_failure = expected_error[0] != '\0' || scratch_failure; + const char* failure_message = expected_error[0] != '\0' + ? expected_error + : (scratch_failure ? "exceeding the configured limit" : ""); tester.Run(session_options, - max_scratch_bytes > 0 && expect_scratch_failure ? OpTester::ExpectResult::kExpectFailure - : OpTester::ExpectResult::kExpectSuccess, - max_scratch_bytes > 0 && expect_scratch_failure ? "exceeding the configured limit" : "", + expect_failure ? OpTester::ExpectResult::kExpectFailure : OpTester::ExpectResult::kExpectSuccess, + failure_message, {}, nullptr, &execution_providers); } @@ -2238,14 +2244,22 @@ TEST(MoETest, QMoETest_Int2CudaPackedDecodeBlock128) { /*expect_scratch_failure=*/false, /*use_initializers=*/true); } +TEST(MoETest, QMoETest_Int2CudaPrepackedRejectsRawFallback) { + if (!HasCudaEnvironment(800)) { + GTEST_SKIP() << "CUDA device with compute capability 8.0 or newer is required."; + } + // No fused SwiGLU: the packed paths must reject before interpreting any weight bytes. + RunQMoEMixedWidthCudaIdentityTest( + 2, 2, 0, false, false, false, 64, 64, 64, false, true, 1, 0, 1, + "Prepacked INT2 requires a supported packed GEMV or grouped-GEMM configuration"); +} + // Runs the same multi-row, multi-expert, k=2 INT2 SwiGLU model once with the packed GEMV // path selected and once with it disabled (forcing the already-validated dense fallback from // #32743), so a stride/permutation/top-k bug in the new packed pipeline shows up as a numeric // mismatch instead of only a shape or gate mismatch. -static std::vector RunQMoEPackedIntGemvParityCase(bool disable_packed_gemv) { - constexpr int64_t num_rows = 4; - constexpr int64_t num_experts = 4; - constexpr int64_t k = 2; +static std::vector RunQMoEPackedIntGemvParityCase( + bool disable_packed_gemv, int64_t num_rows = 4, int64_t num_experts = 4, int64_t k = 2) { constexpr int64_t hidden_size = 512; constexpr int64_t inter_size = 512; constexpr int64_t block_size = 64; @@ -2264,16 +2278,16 @@ static std::vector RunQMoEPackedIntGemvParityCase(bool disable_packed_gem value = static_cast(next_rand() % 17) * 0.05f - 0.4f; } - // Give each row a distinct preferred expert pair so k=2 top-k selection and the resulting - // expert permutation differ across rows. - constexpr int64_t preferred_pairs[num_rows][2] = {{0, 1}, {1, 2}, {2, 3}, {3, 0}}; + // Favor high-numbered experts too, exercising the full expert-map range. std::vector router_probs(static_cast(num_rows * num_experts)); for (int64_t row = 0; row < num_rows; ++row) { for (int64_t e = 0; e < num_experts; ++e) { router_probs[static_cast(row * num_experts + e)] = 0.1f * static_cast(e); } - router_probs[static_cast(row * num_experts + preferred_pairs[row][0])] = 3.0f; - router_probs[static_cast(row * num_experts + preferred_pairs[row][1])] = 2.0f; + for (int64_t rank = 0; rank < k; ++rank) { + const int64_t expert = (num_experts - 1 - row - rank + num_experts) % num_experts; + router_probs[static_cast(row * num_experts + expert)] = 100.0f - static_cast(rank); + } } // Each expert's weight is a "shifted identity": row r picks out input column @@ -2357,6 +2371,7 @@ static std::vector RunQMoEPackedIntGemvParityCase(bool disable_packed_gem EnvVarMap env_overrides; if (disable_packed_gemv) { env_overrides["ORT_DISABLE_MOE_GEMV"] = "1"; + env_overrides["ORT_ENABLE_QMOE_INT2_PREFILL"] = "0"; } ScopedEnvironmentVariables scoped_env_vars{env_overrides}; @@ -2384,6 +2399,21 @@ TEST(MoETest, QMoETest_Int2CudaPackedDecodeMultiRowExpertParity) { } } +TEST(MoETest, QMoETest_Int2Cuda512ExpertsTop10Parity) { + if (!HasCudaEnvironment(800)) { + GTEST_SKIP() << "CUDA device with compute capability 8.0 or newer is required."; + } + for (const int64_t rows : {1, 2, 3, 32}) { + SCOPED_TRACE(rows); + const auto packed = RunQMoEPackedIntGemvParityCase(false, rows, 512, 10); + const auto dense = RunQMoEPackedIntGemvParityCase(true, rows, 512, 10); + ASSERT_EQ(packed.size(), dense.size()); + for (size_t i = 0; i < packed.size(); ++i) { + EXPECT_NEAR(packed[i], dense[i], 0.02f) << "mismatch at index " << i; + } + } +} + #ifdef USE_CUDA static std::string BuildQMoEInt2DynamicRowsModel() { constexpr int64_t num_experts = 1; From 661351202b50d00864f4d93ba0496d7737640121 Mon Sep 17 00:00:00 2001 From: Akshay Sonawane Date: Wed, 21 Oct 2026 22:13:53 +0000 Subject: [PATCH 06/12] Specialize two-row FP8 projections and small-decode MoE routing Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../matmul_block_scaled_fp8_experiments.md | 15 ++ .../contrib_ops/cuda/qmoe_gemv_experiments.md | 16 ++ .../cuda/math/matmul_block_scaled_fp8.cu | 27 ++- .../contrib_ops/cuda/moe/moe_quantization.cc | 35 +++- .../contrib_ops/cuda/moe/qmoe_kernels.cu | 163 +++++++++++++++++- .../contrib_ops/cuda/moe/qmoe_kernels.h | 6 + .../cuda_kernels/softmax_topk_kernel_test.cc | 113 ++++++++++++ .../matmul_block_scaled_fp8_test.cc | 60 +++++++ 8 files changed, 419 insertions(+), 16 deletions(-) diff --git a/docs/contrib_ops/cuda/matmul_block_scaled_fp8_experiments.md b/docs/contrib_ops/cuda/matmul_block_scaled_fp8_experiments.md index f4b863ad315e2..d3079526b6976 100644 --- a/docs/contrib_ops/cuda/matmul_block_scaled_fp8_experiments.md +++ b/docs/contrib_ops/cuda/matmul_block_scaled_fp8_experiments.md @@ -1,5 +1,20 @@ # MatMulBlockQuantizedFp8Weight - CUDA Experiments +## Opt-in two-row per-column specialization + +`ORT_FP8_GEMV_PER_COLUMN_S2=1` specializes the existing MMA GEMV body for +two-row weight-only projections with one scale per output column +(`block_size == K`, no activation scale, and no pinned-residency dispatch). +It removes per-block scale indexing and activation-QDQ bookkeeping while +retaining the MMA accumulation, K-split reduction, optional bias, and split +outputs. Unsupported shapes and activation-QDQ inputs retain existing dispatch. +The default is `0`; other integer values are rejected. Set the option before +the first applicable invocation. + +FP16 and BF16 exact-output tests cover K=320/640/2560/6144, partial output +tiles, bias, and both normal and split outputs. Performance results from +experimental packages do not qualify model accuracy or arbitrary shapes. + This document records CUDA performance experiments for **MatMulBlockQuantizedFp8Weight** (`com.microsoft::MatMulBlockQuantizedFp8Weight`). It includes retained optimizations and their measured effect so future tuning diff --git a/docs/contrib_ops/cuda/qmoe_gemv_experiments.md b/docs/contrib_ops/cuda/qmoe_gemv_experiments.md index 6b08afc11c579..6ed2e42fbe548 100644 --- a/docs/contrib_ops/cuda/qmoe_gemv_experiments.md +++ b/docs/contrib_ops/cuda/qmoe_gemv_experiments.md @@ -1,5 +1,21 @@ # QMoE GEMV Profiling Experiments +## Small-decode fused routing and optional warp selection + +The packed-INT and FP4 GEMV paths can fuse stable softmax/top-K routing with +expert-map construction for one or two rows, avoiding separate routing/map +launches. Unsupported configurations keep the existing routing and map builders. + +`ORT_MOE_ROUTING_WARP_SELECT=1` additionally specializes normalized routing +with 512 experts and top-K=10. One warp per row holds 16 stable packed keys per +lane, selects ten maxima, and preserves the sequential normalization order. +A 128-thread block constructs the stable expert maps using the shared helper. +Other configurations retain the fused sort or generic fallback. + +The warp-selection option defaults to `0` and accepts only `0` or `1`. +Set it before the first invocation. Tests cover FP32/FP16/BF16, ties, +infinities, expert-map stability, and CUDA graph replay. + This file records QMoE INT4/INT8 GEMV profiling results so future kernel and dispatch changes can be compared against a stable baseline. > **Note**: These are **point-in-time** measurements captured on the specific GPU, driver, CUDA diff --git a/onnxruntime/contrib_ops/cuda/math/matmul_block_scaled_fp8.cu b/onnxruntime/contrib_ops/cuda/math/matmul_block_scaled_fp8.cu index 16ae3c0d5c949..db0771cf06021 100644 --- a/onnxruntime/contrib_ops/cuda/math/matmul_block_scaled_fp8.cu +++ b/onnxruntime/contrib_ops/cuda/math/matmul_block_scaled_fp8.cu @@ -527,7 +527,7 @@ __device__ __forceinline__ T* Fp8OutputColumn(T* output, int col, int& stride, return output + col; } -template +template __device__ __forceinline__ void Fp8MmaGemvBody(AType* __restrict__ output, const AType* __restrict__ input_a, const __nv_fp8_e4m3* __restrict__ input_b, @@ -542,7 +542,13 @@ __device__ __forceinline__ void Fp8MmaGemvBody(AType* __restrict__ output, const Fp8GemvOutputSplit& split) { using Mma = Fp8GemvMma; - const bool act_qdq = act_scale != nullptr; + // Keep the same MMA/KSplit reduction, but remove per-block scaling and QDQ + // bookkeeping for weight-only, two-row projections with one scale per column. + if constexpr (PerColumnS2) { + m = 2; + k_blocks = 1; + } + const bool act_qdq = !PerColumnS2 && act_scale != nullptr; const float a_scale = act_qdq ? *act_scale : 0.f; const float a_inv_scale = a_scale != 0.f ? 1.0f / a_scale : 0.f; @@ -577,7 +583,7 @@ __device__ __forceinline__ void Fp8MmaGemvBody(AType* __restrict__ output, #pragma unroll 2 for (int wi = warp; wi < windows; wi += KSplit) { const int k0 = wi << 6; - const int kb = k0 / block_size; + const int kb = PerColumnS2 ? 0 : k0 / block_size; if (kb != cur_kb) { if (cur_kb >= 0) { const float s_lo = lo_ok ? weight_scale[static_cast(col_lo) * k_blocks + cur_kb] : 0.f; @@ -742,6 +748,11 @@ __global__ void MatMulBlockScaledFp8MmaGemvKernel(ORT_FP8_MMA_GEMV_PARAMS) { Fp8MmaGemvBody(ORT_FP8_MMA_GEMV_ARGS); } +template +__global__ void MatMulBlockScaledFp8MmaGemvPerColumnS2Kernel(ORT_FP8_MMA_GEMV_PARAMS) { + Fp8MmaGemvBody(ORT_FP8_MMA_GEMV_ARGS); +} + template __global__ __launch_bounds__(32 * KSplit, 3) void MatMulBlockScaledFp8MmaGemvKernelPinned(ORT_FP8_MMA_GEMV_PARAMS) { Fp8MmaGemvBody(ORT_FP8_MMA_GEMV_ARGS); @@ -1038,6 +1049,10 @@ static Status LaunchMatMulBlockScaledFp8GemvImpl(void* y, const dim3 mma_blocks{static_cast((n + 15) / 16)}; const bool pin_residency = Fp8MmaGemvPinsResidency( n, k_split, mtiles, device_prop.multiProcessorCount, device_prop.major, device_prop.minor); + static const int per_column_s2 = + ParseEnvironmentVariableWithDefault("ORT_FP8_GEMV_PER_COLUMN_S2", 0); + ORT_ENFORCE(per_column_s2 == 0 || per_column_s2 == 1, + "ORT_FP8_GEMV_PER_COLUMN_S2 must be 0 or 1."); const auto launch_mma = [&]() { const dim3 mma_threads{32, KSplit}; const auto launch_variant = [&]() { @@ -1055,6 +1070,12 @@ static Status LaunchMatMulBlockScaledFp8GemvImpl(void* y, block_size, k_blocks, split); \ } \ } while (0) + if constexpr (MTiles == 1) { + if (per_column_s2 && m == 2 && block_size == k && act_scale == nullptr && !pin_residency) { + ORT_FP8_LAUNCH_MMA(MatMulBlockScaledFp8MmaGemvPerColumnS2Kernel); + return; + } + } if constexpr (KSplit == 16 && MTiles == 1) { if (pin_residency) { ORT_FP8_LAUNCH_MMA(MatMulBlockScaledFp8MmaGemvKernelPinned); diff --git a/onnxruntime/contrib_ops/cuda/moe/moe_quantization.cc b/onnxruntime/contrib_ops/cuda/moe/moe_quantization.cc index ae224961905aa..e33de83144323 100644 --- a/onnxruntime/contrib_ops/cuda/moe/moe_quantization.cc +++ b/onnxruntime/contrib_ops/cuda/moe/moe_quantization.cc @@ -1545,6 +1545,21 @@ Status QMoE::ComputeInternal(OpKernelContext* context) const { return fused_routing; }; + auto route_with_expert_maps = [&](int64_t rows, int experts, int* row_to_source, + int* sorted_experts, int64_t* expert_offsets) { + if (rows > 2) return false; + auto launch = [&](auto* type_ptr) { + using T = std::remove_pointer_t; + return TryLaunchSoftmaxTopKExpertMaps( + static_cast(router_probs->DataRaw()), expert_scales, expert_indices, + row_to_source, unpermuted_row_to_permuted_row, sorted_experts, expert_offsets, + static_cast(rows), experts, static_cast(k_), normalize_routing_weights_, stream); + }; + if (is_fp16) return launch(static_cast(nullptr)); + if (is_bf16) return launch(static_cast<__nv_bfloat16*>(nullptr)); + return launch(static_cast(nullptr)); + }; + if (use_fp8_fused) { Tensor* output = context->Output(0, input->Shape()); namespace ck = onnxruntime::llm::kernels::cutlass_kernels; @@ -2145,7 +2160,10 @@ Status QMoE::ComputeInternal(OpKernelContext* context) const { int* p_exp = p_exp_buf.get(); int64_t* p_efto = p_efto_buf.get(); - const auto fused_routing = route_tile(0, num_rows); + const bool fused_expert_maps = route_with_expert_maps(num_rows, num_experts, p_r2u, p_exp, p_efto); + const auto fused_routing = fused_expert_maps + ? ck::FusedRoutingParams{} + : route_tile(0, num_rows); ORT_ENFORCE(fused_routing.router_logits == nullptr, "QMoE packed INT GEMV requires materialized routing outputs."); #if !defined(BUILD_CUDA_EP_AS_PLUGIN) && !defined(ORT_MINIMAL_BUILD) @@ -2154,9 +2172,9 @@ Status QMoE::ComputeInternal(OpKernelContext* context) const { expert_indices, SafeInt(num_rows) * SafeInt(k_), stream)); } #endif - const bool expert_maps_built = ck::fusedBuildExpertMapsSortFirstToken( - expert_indices, p_r2u, unpermuted_row_to_permuted_row, p_exp, p_efto, - num_rows, num_experts, static_cast(k_), 0, num_experts, stream); + const bool expert_maps_built = fused_expert_maps || ck::fusedBuildExpertMapsSortFirstToken( + expert_indices, p_r2u, unpermuted_row_to_permuted_row, p_exp, p_efto, + num_rows, num_experts, static_cast(k_), 0, num_experts, stream); ORT_ENFORCE(expert_maps_built, "QMoE packed INT GEMV failed to build fused expert maps for a supported routing configuration."); @@ -2290,7 +2308,10 @@ Status QMoE::ComputeInternal(OpKernelContext* context) const { act_params.swiglu_fusion = swiglu_fusion; act_params.limit = swiglu_limit_; - const auto fused_routing = route_tile(0, num_rows); + const bool fused_expert_maps = route_with_expert_maps(num_rows, num_experts, p_r2u, p_exp, p_efto); + const auto fused_routing = fused_expert_maps + ? ck::FusedRoutingParams{} + : route_tile(0, num_rows); ORT_ENFORCE(fused_routing.router_logits == nullptr, "QMoE FP4 GEMV does not support fused routing."); #if !defined(BUILD_CUDA_EP_AS_PLUGIN) && !defined(ORT_MINIMAL_BUILD) @@ -2302,9 +2323,7 @@ Status QMoE::ComputeInternal(OpKernelContext* context) const { IAllocatorUniquePtr blocked_expert_counts; IAllocatorUniquePtr blocked_expert_counts_cumsum; IAllocatorUniquePtr blocked_row_to_unpermuted_row; - if (!TryLaunchSmallDecodeExpertMaps( - expert_indices, p_r2u, unpermuted_row_to_permuted_row, p_exp, p_efto, - static_cast(num_rows), num_experts, static_cast(k_), stream) && + if (!fused_expert_maps && !TryLaunchSmallDecodeExpertMaps(expert_indices, p_r2u, unpermuted_row_to_permuted_row, p_exp, p_efto, static_cast(num_rows), num_experts, static_cast(k_), stream) && !ck::fusedBuildExpertMapsSortFirstToken( expert_indices, p_r2u, unpermuted_row_to_permuted_row, p_exp, p_efto, num_rows, num_experts, static_cast(k_), 0, num_experts, stream)) { diff --git a/onnxruntime/contrib_ops/cuda/moe/qmoe_kernels.cu b/onnxruntime/contrib_ops/cuda/moe/qmoe_kernels.cu index 7225714263b10..acfe95c359be8 100644 --- a/onnxruntime/contrib_ops/cuda/moe/qmoe_kernels.cu +++ b/onnxruntime/contrib_ops/cuda/moe/qmoe_kernels.cu @@ -223,13 +223,10 @@ __global__ void SoftmaxTopKMergeKernel(const T* logits, float* topk_scales, int* } } -__global__ void SmallDecodeExpertMapsKernel( - const int* indices, int* row_to_source, int* source_to_row, int* sorted_experts, +__device__ __forceinline__ void BuildSmallDecodeExpertMaps( + const int* selected, int* row_to_source, int* source_to_row, int* sorted_experts, int64_t* expert_offsets, int rows, int experts, int k) { - __shared__ int selected[32]; const int expanded = rows * k; - if (threadIdx.x < expanded) selected[threadIdx.x] = indices[threadIdx.x]; - __syncthreads(); if (threadIdx.x < expanded) { const int original = threadIdx.x; const int expert = selected[original]; @@ -249,6 +246,162 @@ __global__ void SmallDecodeExpertMapsKernel( } } +// Two independent 128-thread sorts share a CTA so expert-map construction needs +// no second launch. The full-CTA barrier keeps both merge-sort groups in lockstep. +class DecodeRoutingMergeSort + : public cub::BlockMergeSortStrategy { + using Base = cub::BlockMergeSortStrategy; + + public: + __device__ explicit DecodeRoutingMergeSort(Base::TempStorage& storage, int tid) : Base(storage, tid) {} + __device__ void SyncImplementation() const { __syncthreads(); } +}; + +template +__global__ void FusedSoftmaxTopKExpertMapsKernel( + const T* logits, float* scales, int* indices, int* row_to_source, + int* source_to_row, int* sorted_experts, int64_t* expert_offsets, + int rows, int experts, int k) { + const int row = threadIdx.x / 128; + const int local = threadIdx.x % 128; + __shared__ DecodeRoutingMergeSort::TempStorage temp[2]; + __shared__ float maxima[2]; + __shared__ float probabilities[2][16]; + __shared__ int selected[32]; + uint64_t keys[4]; +#pragma unroll + for (int j = 0; j < 4; ++j) { + const int expert = local * 4 + j; + const bool valid = row < rows && expert < experts; + const float value = valid ? static_cast(logits[row * experts + expert]) + : onnxruntime::cuda::topk::kNegativeInfinity; + keys[j] = onnxruntime::cuda::topk::PackStableSortKey(value, valid ? expert : INT_MAX); + } + DecodeRoutingMergeSort(temp[row], local).Sort(keys, onnxruntime::cuda::topk::Greater()); + if (local == 0) maxima[row] = onnxruntime::cuda::topk::UnpackStableSortScore(keys[0]); + __syncthreads(); +#pragma unroll + for (int j = 0; j < 4; ++j) { + const int rank = local * 4 + j; + if (row < rows && rank < k) { + const int expert = onnxruntime::cuda::topk::UnpackStableSortIndex(keys[j]); + indices[row * k + rank] = expert; + selected[row * k + rank] = expert; + const float value = expf(onnxruntime::cuda::topk::UnpackStableSortScore(keys[j]) - maxima[row]); + probabilities[row][rank] = isfinite(value) ? value : 0.0f; + } + } + __syncthreads(); + // For normalized routing the full softmax denominator cancels. With at + // least one finite maximum, the selected exponential sum is >= 1. + if (local == 0 && row < rows) { + float sum = 0.0f; + for (int rank = 0; rank < k; ++rank) sum += probabilities[row][rank]; + const float denom = TopKNormalizeDenom(true, sum); + for (int rank = 0; rank < k; ++rank) scales[row * k + rank] = probabilities[row][rank] / denom; + } + BuildSmallDecodeExpertMaps(selected, row_to_source, source_to_row, sorted_experts, + expert_offsets, rows, experts, k); +} + +template +__global__ void WarpSelectSoftmaxTopKExpertMapsKernel( + const T* logits, float* scales, int* indices, int* row_to_source, + int* source_to_row, int* sorted_experts, int64_t* expert_offsets, int rows) { + constexpr int kExperts = 512; + constexpr int k = 10; + const int row = threadIdx.x / 32; + const int lane = threadIdx.x % 32; + __shared__ int selected[2 * k]; + if (row < rows) { + uint64_t keys[16]; +#pragma unroll + for (int i = 0; i < 16; ++i) { + const int expert = lane + i * 32; + keys[i] = onnxruntime::cuda::topk::PackStableSortKey( + static_cast(logits[row * kExperts + expert]), expert); + } + float probabilities[k]; + float maximum = 0.0f; + for (int rank = 0; rank < k; ++rank) { + uint64_t best = keys[0]; +#pragma unroll + for (int i = 1; i < 16; ++i) best = best > keys[i] ? best : keys[i]; +#pragma unroll + for (int offset = 16; offset > 0; offset >>= 1) { + const uint64_t other = __shfl_xor_sync(0xffffffff, static_cast(best), offset); + best = best > other ? best : other; + } +#pragma unroll + for (int i = 0; i < 16; ++i) { + if (keys[i] == best) keys[i] = onnxruntime::cuda::topk::kPaddingSortKey; + } + if (lane == 0) { + const float score = onnxruntime::cuda::topk::UnpackStableSortScore(best); + if (rank == 0) maximum = score; + const float value = expf(score - maximum); + probabilities[rank] = isfinite(value) ? value : 0.0f; + const int expert = onnxruntime::cuda::topk::UnpackStableSortIndex(best); + selected[row * k + rank] = expert; + indices[row * k + rank] = expert; + } + } + if (lane == 0) { + float sum = 0.0f; + for (int rank = 0; rank < k; ++rank) sum += probabilities[rank]; + const float denom = TopKNormalizeDenom(true, sum); + for (int rank = 0; rank < k; ++rank) scales[row * k + rank] = probabilities[rank] / denom; + } + } + __syncthreads(); + BuildSmallDecodeExpertMaps(selected, row_to_source, source_to_row, sorted_experts, + expert_offsets, rows, kExperts, k); +} + +template +bool TryLaunchSoftmaxTopKExpertMaps( + const T* logits, float* scales, int* indices, int* row_to_source, + int* source_to_row, int* sorted_experts, int64_t* expert_offsets, + int rows, int experts, int k, bool normalize, cudaStream_t stream) { + if (!normalize || rows < 1 || rows > 2 || experts < 1 || experts > 512 || + k < 1 || k > 16 || k > experts) { + return false; + } + static const int warp_select = + ParseEnvironmentVariableWithDefault("ORT_MOE_ROUTING_WARP_SELECT", 0); + ORT_ENFORCE(warp_select == 0 || warp_select == 1, + "ORT_MOE_ROUTING_WARP_SELECT must be 0 or 1."); + if (warp_select && experts == 512 && k == 10) { + WarpSelectSoftmaxTopKExpertMapsKernel<<<1, 128, 0, stream>>>( + logits, scales, indices, row_to_source, source_to_row, sorted_experts, expert_offsets, rows); + } else { + FusedSoftmaxTopKExpertMapsKernel<<<1, 256, 0, stream>>>( + logits, scales, indices, row_to_source, source_to_row, sorted_experts, + expert_offsets, rows, experts, k); + } + CUDA_CALL_THROW(cudaGetLastError()); + return true; +} + +#define INSTANTIATE_FUSED_ROUTING_MAPS(T) \ + template bool TryLaunchSoftmaxTopKExpertMaps( \ + const T*, float*, int*, int*, int*, int*, int64_t*, int, int, int, \ + bool, cudaStream_t); +INSTANTIATE_FUSED_ROUTING_MAPS(float) +INSTANTIATE_FUSED_ROUTING_MAPS(half) +INSTANTIATE_FUSED_ROUTING_MAPS(__nv_bfloat16) +#undef INSTANTIATE_FUSED_ROUTING_MAPS + +__global__ void SmallDecodeExpertMapsKernel( + const int* indices, int* row_to_source, int* source_to_row, int* sorted_experts, + int64_t* expert_offsets, int rows, int experts, int k) { + __shared__ int selected[32]; + if (threadIdx.x < rows * k) selected[threadIdx.x] = indices[threadIdx.x]; + __syncthreads(); + BuildSmallDecodeExpertMaps(selected, row_to_source, source_to_row, sorted_experts, + expert_offsets, rows, experts, k); +} + bool TryLaunchSmallDecodeExpertMaps( const int* indices, int* row_to_source, int* source_to_row, int* sorted_experts, int64_t* expert_offsets, int rows, int experts, int k, cudaStream_t stream) { diff --git a/onnxruntime/contrib_ops/cuda/moe/qmoe_kernels.h b/onnxruntime/contrib_ops/cuda/moe/qmoe_kernels.h index c28cc343f4ccf..33a401038d4b9 100644 --- a/onnxruntime/contrib_ops/cuda/moe/qmoe_kernels.h +++ b/onnxruntime/contrib_ops/cuda/moe/qmoe_kernels.h @@ -12,6 +12,12 @@ namespace onnxruntime { namespace contrib { namespace cuda { +template +bool TryLaunchSoftmaxTopKExpertMaps( + const T* logits, float* scales, int* indices, int* row_to_source, + int* source_to_row, int* sorted_experts, int64_t* expert_offsets, + int rows, int experts, int k, bool normalize, cudaStream_t stream); + bool TryLaunchSmallDecodeExpertMaps( const int* indices, int* row_to_source, int* source_to_row, int* sorted_experts, int64_t* expert_offsets, int rows, int experts, int k, cudaStream_t stream); diff --git a/onnxruntime/test/contrib_ops/cuda_kernels/softmax_topk_kernel_test.cc b/onnxruntime/test/contrib_ops/cuda_kernels/softmax_topk_kernel_test.cc index 345983b876b12..95e27c481a58b 100644 --- a/onnxruntime/test/contrib_ops/cuda_kernels/softmax_topk_kernel_test.cc +++ b/onnxruntime/test/contrib_ops/cuda_kernels/softmax_topk_kernel_test.cc @@ -268,6 +268,119 @@ TEST(CUDA_EP_Unittest, SoftmaxTopK_RejectsKGreaterThanExperts) { OnnxRuntimeException); } +template +void RunFusedRoutingMapsTest(int rows, int experts, int k, bool ties = false, bool nonfinite = false) { + auto values = MakeLogits(rows, experts); + if (ties) std::fill(values.begin(), values.end(), 0.0f); + if (nonfinite) { + std::fill(values.begin(), values.end(), -std::numeric_limits::infinity()); + if (rows == 2) values[experts] = std::numeric_limits::infinity(); + } + std::vector logits(values.size()); + std::transform(values.begin(), values.end(), logits.begin(), [](float v) { return FromFloat(v); }); + const int expanded = rows * k; + CudaBuffer d_logits(logits.size() * sizeof(T)); + CudaBuffer d_scales(expanded * sizeof(float)), d_reference_scales(expanded * sizeof(float)); + CudaBuffer d_indices(expanded * sizeof(int)), d_reference_indices(expanded * sizeof(int)); + CudaBuffer d_r2s(expanded * sizeof(int)), d_s2r(expanded * sizeof(int)); + CudaBuffer d_sorted(expanded * sizeof(int)), d_offsets((experts + 1) * sizeof(int64_t)); + d_logits.CopyFromHost(logits.data()); + cudaStream_t stream = nullptr; + CUDA_CALL_THROW(cudaStreamCreate(&stream)); + contrib::cuda::LaunchSoftmaxTopK(d_logits.As(), d_reference_scales.As(), + d_reference_indices.As(), rows, experts, k, true, stream); + CUDA_CALL_THROW(cudaStreamSynchronize(stream)); + cudaGraph_t graph = nullptr; + cudaGraphExec_t executable = nullptr; + CUDA_CALL_THROW(cudaStreamBeginCapture(stream, cudaStreamCaptureModeThreadLocal)); + const bool launched = contrib::cuda::TryLaunchSoftmaxTopKExpertMaps( + d_logits.As(), d_scales.As(), d_indices.As(), d_r2s.As(), + d_s2r.As(), d_sorted.As(), d_offsets.As(), rows, experts, k, true, stream); + CUDA_CALL_THROW(cudaStreamEndCapture(stream, &graph)); + ASSERT_TRUE(launched); + size_t node_count = 0; + CUDA_CALL_THROW(cudaGraphGetNodes(graph, nullptr, &node_count)); + EXPECT_EQ(node_count, 1u); + CUDA_CALL_THROW(cudaGraphInstantiate(&executable, graph, nullptr, nullptr, 0)); + std::vector actual_indices(expanded), reference_indices(expanded); + std::vector actual_scales(expanded), reference_scales(expanded); + std::vector r2s(expanded), s2r(expanded), sorted(expanded); + std::vector offsets(experts + 1); + d_reference_indices.CopyToHost(reference_indices.data()); + d_reference_scales.CopyToHost(reference_scales.data()); + std::vector order(expanded); + std::iota(order.begin(), order.end(), 0); + std::stable_sort(order.begin(), order.end(), [&](int a, int b) { + return reference_indices[a] < reference_indices[b]; + }); + for (int replay = 0; replay < 3; ++replay) { + CUDA_CALL_THROW(cudaGraphLaunch(executable, stream)); + CUDA_CALL_THROW(cudaStreamSynchronize(stream)); + d_indices.CopyToHost(actual_indices.data()); + d_scales.CopyToHost(actual_scales.data()); + d_r2s.CopyToHost(r2s.data()); + d_s2r.CopyToHost(s2r.data()); + d_sorted.CopyToHost(sorted.data()); + d_offsets.CopyToHost(offsets.data()); + EXPECT_EQ(actual_indices, reference_indices); + for (int i = 0; i < expanded; ++i) { + EXPECT_NEAR(actual_scales[i], reference_scales[i], 2e-6f); + const int original = order[i]; + const int source = original % k * rows + original / k; + EXPECT_EQ(r2s[i], source); + EXPECT_EQ(s2r[source], i); + EXPECT_EQ(sorted[i], reference_indices[original]); + } + for (int expert = 0; expert <= experts; ++expert) { + EXPECT_EQ(offsets[expert], std::count_if(reference_indices.begin(), reference_indices.end(), + [&](int selected) { return selected < expert; })); + } + } + CUDA_CALL_THROW(cudaGraphExecDestroy(executable)); + CUDA_CALL_THROW(cudaGraphDestroy(graph)); + CUDA_CALL_THROW(cudaStreamDestroy(stream)); +} + +TEST(CUDA_EP_Unittest, FusedSoftmaxTopKExpertMaps_NormalizedAndGraphReplay) { + for (int rows : {1, 2}) { + for (int experts : {1, 7, 32, 65, 256, 511, 512}) { + SCOPED_TRACE(::testing::Message() << "rows=" << rows << " experts=" << experts); + const int k = std::min(10, experts); + RunFusedRoutingMapsTest(rows, experts, k); + RunFusedRoutingMapsTest(rows, experts, k); + RunFusedRoutingMapsTest<__nv_bfloat16>(rows, experts, k); + } + RunFusedRoutingMapsTest<__nv_bfloat16>(rows, 512, 16, true); + RunFusedRoutingMapsTest(rows, 80, 16, false, true); + } +} + +TEST(CUDA_EP_Unittest, FusedSoftmaxTopKExpertMaps_UnsupportedUsesFallback) { + auto launch = [](int rows, int experts, int k, bool normalize) { + return contrib::cuda::TryLaunchSoftmaxTopKExpertMaps( + static_cast(nullptr), nullptr, nullptr, nullptr, nullptr, + nullptr, nullptr, rows, experts, k, normalize, nullptr); + }; + EXPECT_FALSE(launch(0, 512, 10, true)); + EXPECT_FALSE(launch(3, 512, 10, true)); + EXPECT_FALSE(launch(2, 513, 10, true)); + EXPECT_FALSE(launch(2, 512, 17, true)); + EXPECT_FALSE(launch(2, 8, 9, true)); + EXPECT_FALSE(launch(2, 512, 0, true)); + EXPECT_FALSE(launch(2, 512, 10, false)); +} + +TEST(CUDA_EP_Unittest, FusedSoftmaxTopKExpertMaps_WarpSelectSpecialized) { + for (int rows : {1, 2}) { + for (bool ties : {false, true}) { + RunFusedRoutingMapsTest(rows, 512, 10, ties); + RunFusedRoutingMapsTest(rows, 512, 10, ties); + RunFusedRoutingMapsTest<__nv_bfloat16>(rows, 512, 10, ties); + } + RunFusedRoutingMapsTest(rows, 512, 10, false, true); + } +} + } // namespace } // namespace test } // namespace onnxruntime diff --git a/onnxruntime/test/contrib_ops/matmul_block_scaled_fp8_test.cc b/onnxruntime/test/contrib_ops/matmul_block_scaled_fp8_test.cc index 0695e717ea6eb..cd235a45c6775 100644 --- a/onnxruntime/test/contrib_ops/matmul_block_scaled_fp8_test.cc +++ b/onnxruntime/test/contrib_ops/matmul_block_scaled_fp8_test.cc @@ -152,6 +152,66 @@ TEST(MatMulBlockQuantizedFp8WeightOpTest, SplitOutputsFp16) { TestFp8SplitOutputs(); } +template +void TestPerColumnS2ExactOutputs() { + if (!HasCudaEnvironment(800)) { + GTEST_SKIP() << "CUDA device is required for per-column FP8 matmul."; + } + constexpr int64_t m = 2, n = 33; + for (int64_t k : {320, 640, 2560, 6144}) { + std::vector a(m * k), bias(n), expected(m * n); + std::vector b(n * k); + std::vector scales(n); + for (int64_t i = 0; i < m * k; ++i) a[i] = T(float(i % 5 - 2) / 8.f); + for (int64_t col = 0; col < n; ++col) { + scales[col] = float(col % 3 + 1) / 8.f; + bias[col] = T(float(col % 7 - 3) / 32.f); + for (int64_t i = 0; i < k; ++i) b[col * k + i] = Float8E4M3FN(float((col + i) % 7 - 3) / 16.f); + for (int64_t row = 0; row < m; ++row) { + float sum = 0.f; + for (int64_t i = 0; i < k; ++i) { + sum += float(a[row * k + i]) * b[col * k + i].ToFloat(); + } + expected[row * n + col] = T(sum * scales[col] + float(bias[col])); + } + } + for (bool split : {false, true}) { + OpTester test("MatMulBlockQuantizedFp8Weight", 1, kMSDomain); + test.AddAttribute("block_size", k); + const std::vector widths = split ? std::vector{1, 7, 25} + : std::vector{n}; + if (split) test.AddAttribute>("split_sizes", widths); + test.AddInput("A", {m, k}, a); + test.AddInput("B", {n, k}, b); + test.AddInput("b_scale", {n, 1}, scales); + test.AddOptionalInputEdge(); + test.AddInput("bias", {n}, bias); + int64_t offset = 0; + for (size_t index = 0; index < widths.size(); ++index) { + std::vector values; + for (int64_t row = 0; row < m; ++row) { + values.insert(values.end(), expected.begin() + row * n + offset, + expected.begin() + row * n + offset + widths[index]); + } + test.AddOutput(("Y" + std::to_string(index)).c_str(), {m, widths[index]}, values); + offset += widths[index]; + } + test.SetOutputTolerance(0.f, 0.f); + std::vector> providers; + providers.push_back(DefaultCudaExecutionProvider()); + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &providers); + } + } +} + +TEST(MatMulBlockQuantizedFp8WeightOpTest, PerColumnS2ExactOutputsFp16) { + TestPerColumnS2ExactOutputs(); +} + +TEST(MatMulBlockQuantizedFp8WeightOpTest, PerColumnS2ExactOutputsBf16) { + TestPerColumnS2ExactOutputs(); +} + TEST(MatMulBlockQuantizedFp8WeightOpTest, SplitOutputsBFloat16) { TestFp8SplitOutputs(); } From 8d2fc230e790a3590ee6b98c56caec25f089feaa Mon Sep 17 00:00:00 2001 From: Akshay Sonawane Date: Thu, 22 Oct 2026 05:43:25 +0000 Subject: [PATCH 07/12] Fuse FP8 shared experts and add native warp routing reduction Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../cuda/matmul_block_scaled_fp8.md | 39 +++++ docs/contrib_ops/cuda/moe_qmoe.md | 28 ++++ .../cuda/math/matmul_block_scaled_fp8.cc | 120 ++++++++++++-- .../cuda/math/matmul_block_scaled_fp8.cu | 138 ++++++++++++++-- .../cuda/math/matmul_block_scaled_fp8.h | 17 +- .../contrib_ops/cuda/moe/qmoe_kernels.cu | 31 +++- .../core/graph/contrib_ops/contrib_defs.cc | 92 ++++++++++- .../cuda_kernels/softmax_topk_kernel_test.cc | 33 +++- .../matmul_block_scaled_fp8_test.cc | 147 ++++++++++++++++++ 9 files changed, 612 insertions(+), 33 deletions(-) diff --git a/docs/contrib_ops/cuda/matmul_block_scaled_fp8.md b/docs/contrib_ops/cuda/matmul_block_scaled_fp8.md index a36dfb4d4aa47..501af7ceec045 100644 --- a/docs/contrib_ops/cuda/matmul_block_scaled_fp8.md +++ b/docs/contrib_ops/cuda/matmul_block_scaled_fp8.md @@ -342,3 +342,42 @@ Python load locations before Python benchmarks (or reinstall the wheel): cp "$ORT_BUILD/libonnxruntime_providers_cuda.so" \ "$ORT_BUILD/onnxruntime/capi/libonnxruntime_providers_cuda.so" ``` + +### Shared-expert fusion (opt-in) + +With `shared_expert=1`, `A` is the BF16 up branch. Four appended BF16 inputs +are required: index 9 `shared_silu_gate` (same shape as `A`), index 10 +`shared_gate_activation` (same leading dimensions, arbitrary positive last +width `H`), index 11 `shared_gate_weight` (`[H, 1]`), and index 12 +`shared_residual` (same shape as `Y`). Indices 6-8 remain reserved for original +prefill weights; they must be absent in shared-expert mode. Bias, activation +quantization, split outputs and prefill-weight overrides are not supported +with this mode. Ordinary invocations retain their existing semantics. + +The operation preserves the unfused BF16 rounding boundaries: + +``` +prepared = bf16(A * bf16(shared_silu_gate * sigmoid_bf16(shared_silu_gate))) +down = bf16(prepared @ dequant(B).T) +gate = sigmoid_bf16(bf16(shared_gate_activation @ shared_gate_weight)) +Y = bf16(bf16(down * gate) + shared_residual) +``` + +`sigmoid_bf16` uses BF16 intermediates, including the negative-input +subtraction, matching `SigmoidGateMul(sigmoid_in_fp32=0)`. The projected +gate uses the same 256-thread CUB reduction as that operator. Input 5 +`prefill_numerics` still selects the original FP8 projection numerical path. + +One preparation kernel computes both SiLU/up and projected row gates. On SM90, +eligible one/two-row tensor-core GEMV invocations reuse the original MMA body +and absorb the gate/residual epilogue into its stores (two total launches +instead of three). Other architectures, disabled MMA, larger M, small or +unaligned K and dense-prefill paths retain the existing projection +dispatch and run a separate epilogue. No weights are transformed by the fusion. +The focused native tests compare bit-exactly against the three original CUDA +operators, including dense-prefill and generic fallbacks. + + +Optional CPU input 5 selects dense-prefill numerics. Optional inputs 6-8 and `prefill_weight_offsets` preserve the original projection weights/layout when a packed decode projection differs. They are mutually exclusive with shared-expert mode. + +Shared-expert preparation depends on the projected-gate helpers from companion PR #33470. diff --git a/docs/contrib_ops/cuda/moe_qmoe.md b/docs/contrib_ops/cuda/moe_qmoe.md index 0ad4bd6ad48de..5d0ca8f199754 100644 --- a/docs/contrib_ops/cuda/moe_qmoe.md +++ b/docs/contrib_ops/cuda/moe_qmoe.md @@ -1938,3 +1938,31 @@ The CUTLASS kernels are derived from TensorRT-LLM (CUTLASS 4.4.2, commit `BlockScaleParams`) - `Deep Gemm`, standalone FP4 GEMM, FP8 block-scale GEMM, fused gated GEMM directories (the relevant pieces are inlined into the MoE runner). + +## Hardware-reduction routing + +For normalized 512-expert, top-10 routing, `ORT_MOE_ROUTING_WARP_SELECT=1` +selects the warp-per-row register-selection implementation instead of the +default merge sort (`=0`). Experimental `=2` retains the same selection, +exponentials, normalization and expert maps but reduces each 64-bit stable key +with two native unsigned 32-bit warp maxima on SM80 and newer: first the score +word, then the index word among matching score lanes. Older architectures +retain the shuffle reduction. This preserves the full packed-key ordering, +including stable ties and nonfinite-score handling. Unsupported shapes retain +merge sort in all modes. Mode 2 remains opt-in; the default is unchanged. + +On the H200 Qwen INT2 S2 package with shared-expert and RMSNormRotary fusions, +forward and reverse fresh-process ABBA brackets (12 measured requests per +mode/context combined) gave 274.48 -> 275.22 decode TPS at 2K (+0.27%) and +254.54 -> 255.05 at 8K (+0.20%). Both bracket orders improved, but the small +gain is not a general performance guarantee. Paired 8K CUDA Graph captures +retain 946 kernels/cycle and 48 routing launches/cycle, with summed routing +duration 286.52 -> 264.44 us/cycle. Profiled kernel sums are not additive +end-to-end savings. All 76 requests including warmups/captures preserved exact +512-token outputs; full accuracy qualification remains deferred. + +All 13 focused CUDA routing tests pass in modes 0, 1 and 2. Tests cover exact +indices and stable maps, scale tolerances, one-node CUDA Graph replay, generic +fallbacks, all winners in one lane, signed-zero ties, large logits, NaNs and +infinities for FP32/FP16/BF16 one/two-row inputs. Invalid modes fail explicitly. +Seven INT2/mixed-width packed-decode integration tests also pass in mode 2. diff --git a/onnxruntime/contrib_ops/cuda/math/matmul_block_scaled_fp8.cc b/onnxruntime/contrib_ops/cuda/math/matmul_block_scaled_fp8.cc index f98f3a7232f6c..3bcb234d3e7c7 100644 --- a/onnxruntime/contrib_ops/cuda/math/matmul_block_scaled_fp8.cc +++ b/onnxruntime/contrib_ops/cuda/math/matmul_block_scaled_fp8.cc @@ -2,6 +2,7 @@ // Licensed under the MIT License. #include "contrib_ops/cuda/math/matmul_block_scaled_fp8.h" +#include "contrib_ops/cuda/bert/fused_tensor_ops.h" #include #include @@ -38,7 +39,9 @@ ONNX_OPERATOR_KERNEL_EX( (*KernelDefBuilder::Create()) .TypeConstraint("T", BuildKernelDefConstraints()) .TypeConstraint("T1", BuildKernelDefConstraints()) - .TypeConstraint("T2", BuildKernelDefConstraints()), + .TypeConstraint("T2", BuildKernelDefConstraints()) + .TypeConstraint("TI", BuildKernelDefConstraints()) + .InputMemoryType(OrtMemTypeCPUInput, 5), MatMulBlockQuantizedFp8Weight); #endif @@ -46,8 +49,14 @@ MatMulBlockQuantizedFp8Weight::MatMulBlockQuantizedFp8Weight(const OpKernelInfo& : CudaKernel(info), block_size_(info.GetAttrOrDefault("block_size", 128)), max_dequant_scratch_bytes_(DequantScratchLimitBytes()), - enable_deep_gemm_(ParseEnvironmentVariableWithDefault("ORT_FP8_MATMUL_DEEPGEMM", false)) { + enable_deep_gemm_(ParseEnvironmentVariableWithDefault("ORT_FP8_MATMUL_DEEPGEMM", false)), + shared_expert_(info.GetAttrOrDefault("shared_expert", 0) == 1) { + ORT_ENFORCE(info.GetAttrOrDefault("shared_expert", 0) == 0 || shared_expert_, + "shared_expert must be 0 or 1"); ORT_ENFORCE(block_size_ > 0, "block_size must be positive."); + static_cast(info.GetAttrs("prefill_weight_offsets", prefill_weight_offsets_)); + ORT_ENFORCE(prefill_weight_offsets_.size() <= 3, + "prefill_weight_offsets supports at most three original weight segments"); if (info.GetAttrs("split_sizes", split_sizes_).IsOK()) { ORT_ENFORCE(!split_sizes_.empty() && split_sizes_.size() <= Fp8GemvOutputSplit::kMaxOutputs, "split_sizes requires one to eight widths."); @@ -57,6 +66,8 @@ MatMulBlockQuantizedFp8Weight::MatMulBlockQuantizedFp8Weight(const OpKernelInfo& "split_sizes must contain positive widths with no sum overflow."); total += width; } + ORT_ENFORCE(!shared_expert_ || (split_sizes_.empty() && prefill_weight_offsets_.empty()), + "shared_expert does not support split_sizes or prefill_weight_offsets"); } } @@ -73,6 +84,14 @@ Status MatMulBlockQuantizedFp8Weight::ComputeImpl(OpKernelContext* context) cons const Tensor* b_scale = context->Input(2); const Tensor* a_scale = context->Input(3); // optional const Tensor* bias = context->Input(4); // optional + const Tensor* numerics = context->Input(5); + ORT_RETURN_IF_NOT(numerics == nullptr || + (numerics->IsDataType() && numerics->Shape().Size() == 1 && + (numerics->Data()[0] == 0 || numerics->Data()[0] == 1)), + "prefill_numerics must be an int32 scalar containing 0 or 1"); + const bool dense_prefill = numerics != nullptr && numerics->Data()[0] == 1; + ORT_RETURN_IF_NOT(!dense_prefill || a_scale == nullptr, + "dense prefill numerics requires weight-only FP8 projection"); const auto& a_shape = a->Shape(); ORT_ENFORCE(a_shape.NumDimensions() >= 1, "A must have rank at least 1."); @@ -107,8 +126,35 @@ Status MatMulBlockQuantizedFp8Weight::ComputeImpl(OpKernelContext* context) cons const int m_i = SafeInt(helper.M()); const int n_i = SafeInt(helper.N()); const int k_i = SafeInt(helper.K()); + const Tensor* silu_gate = context->Input(9); + const Tensor* gate_activation = context->Input(10); + const Tensor* gate_weight = context->Input(11); + const Tensor* residual = context->Input(12); + if (shared_expert_) { + ORT_RETURN_IF_NOT((std::is_same_v) && a_scale == nullptr && bias == nullptr && + silu_gate && gate_activation && gate_weight && residual && k_i > 0 && n_i > 0, + "shared_expert requires BF16, positive widths, all four fusion inputs, no bias or a_scale"); + ORT_RETURN_IF_NOT(silu_gate->Shape() == a_shape && + gate_activation->Shape().NumDimensions() == a_shape.NumDimensions() && + gate_activation->Shape()[a_rank - 1] > 0 && + gate_weight->Shape().NumDimensions() == 2 && + gate_weight->Shape()[0] == gate_activation->Shape()[a_rank - 1] && + gate_weight->Shape()[1] == 1 && residual->Shape() == helper.OutputShape(), + "shared_expert input shapes are incompatible"); + for (int64_t axis = 0; axis + 1 < a_rank; ++axis) { + ORT_RETURN_IF_NOT(gate_activation->Shape()[axis] == a_shape[axis], + "shared_expert leading dimensions must match"); + } + for (int index = 6; index <= 8; ++index) { + ORT_RETURN_IF_NOT(context->Input(index) == nullptr, + "shared_expert does not support original prefill weight inputs"); + } + } else { + ORT_RETURN_IF_NOT(!silu_gate && !gate_activation && !gate_weight && !residual, + "Shared expert inputs require shared_expert=1"); + } - const bool use_gemv = m_i > 0 && + const bool use_gemv = !dense_prefill && m_i > 0 && m_i <= MatMulBlockScaledFp8GemvMaxM(k_i, SafeInt(block_size_), GetDeviceProp()) && (k_i % 16 == 0) && (block_size_ % 16 == 0); const bool direct_split = !split_sizes_.empty() && use_gemv && @@ -146,7 +192,26 @@ Status MatMulBlockQuantizedFp8Weight::ComputeImpl(OpKernelContext* context) cons if (output_shape.Size() == 0) { return Status::OK(); } + IAllocatorUniquePtr shared_activation; + IAllocatorUniquePtr shared_gates; + Fp8SharedExpertEpilogue shared_epilogue; + bool shared_epilogue_done = false; + if (shared_expert_) { + shared_activation = GetScratchBuffer(SafeInt(m_i) * k_i, GetComputeStream(context)); + shared_gates = GetScratchBuffer(m_i, GetComputeStream(context)); + ORT_RETURN_IF_ERROR(LaunchPrepareSharedExpert( + Stream(context), reinterpret_cast(a->DataRaw()), + reinterpret_cast(silu_gate->DataRaw()), + reinterpret_cast(gate_activation->DataRaw()), + reinterpret_cast(gate_weight->DataRaw()), + reinterpret_cast(shared_activation.get()), shared_gates.get(), m_i, k_i, + SafeInt(gate_activation->Shape()[a_rank - 1]))); + shared_epilogue = {shared_gates.get(), residual->DataRaw()}; + } const auto finish_output = [&]() -> Status { + if (shared_expert_ && !shared_epilogue_done) { + ORT_RETURN_IF_ERROR(LaunchFp8SharedExpertEpilogue(y_data, shared_epilogue, m_i, n_i, Stream(context))); + } if (split.count != 0 && !direct_split) { size_t offset = 0; for (int i = 0; i < split.count; ++i) { @@ -186,7 +251,7 @@ Status MatMulBlockQuantizedFp8Weight::ComputeImpl(OpKernelContext* context) cons #if defined(USE_DEEP_GEMM) // Native FP8 changes intermediate rounding, so it is opt-in and requires the model's // activation scale. Keep the tuned small-M path and unsupported layouts on the fallback. - if (enable_deep_gemm_ && !use_gemv && a_scale != nullptr && block_size_ == 128 && + if (!dense_prefill && enable_deep_gemm_ && !use_gemv && a_scale != nullptr && block_size_ == 128 && GetDeviceProp().major == 9 && GetDeviceProp().minor == 0 && k_i % 128 == 0 && n_i % 64 == 0 && n_i >= 2048 && SafeInt(n_i) * k_i >= 8 * 1024 * 1024 && @@ -221,7 +286,7 @@ Status MatMulBlockQuantizedFp8Weight::ComputeImpl(OpKernelContext* context) cons } #endif - const void* a_ptr = a->DataRaw(); + const void* a_ptr = shared_expert_ ? shared_activation.get() : a->DataRaw(); IAllocatorUniquePtr a_dequant; if (a_scale != nullptr && !fuse_act_qdq) { a_dequant = GetScratchBuffer(SafeInt(m_i) * SafeInt(k_i), @@ -241,6 +306,14 @@ Status MatMulBlockQuantizedFp8Weight::ComputeImpl(OpKernelContext* context) cons // A fused warp-per-column kernel reads the FP8 weight directly, avoiding both the [N, K] // dequant scratch buffer and the cuBLAS GEMM (which is underutilized at M == 1). if (use_gemv) { + if (shared_expert_ && m_i <= 2 && GetDeviceProp().major == 9 && + MatMulBlockScaledFp8GemvSupportsSplitOutput(m_i, k_i, SafeInt(block_size_), GetDeviceProp())) { + ORT_RETURN_IF_ERROR(LaunchMatMulFp8SharedExpert( + y_data, a_ptr, b->DataRaw(), b_scale->Data(), m_i, n_i, k_i, + SafeInt(block_size_), shared_epilogue, GetDeviceProp(), Stream(context))); + shared_epilogue_done = true; + return finish_output(); + } ORT_RETURN_IF_ERROR(LaunchMatMulBlockScaledFp8Gemv( y_data, a_ptr, @@ -262,7 +335,8 @@ Status MatMulBlockQuantizedFp8Weight::ComputeImpl(OpKernelContext* context) cons // is tiled over N because a full [N, K] buffer is 2.37 GiB for a 248320 x 5120 LM head. const int64_t rows_per_scratch = static_cast(max_dequant_scratch_bytes_ / (static_cast(k) * sizeof(CudaT))); - const int64_t tile_rows = std::clamp(rows_per_scratch, 1, n); + // Dense MatMul consumes a contiguous KN weight, not a transposed NK operand or N tiles. + const int64_t tile_rows = dense_prefill ? n : std::clamp(rows_per_scratch, 1, n); IAllocatorUniquePtr b_dequant = GetScratchBuffer(SafeInt(tile_rows) * SafeInt(k), GetComputeStream(context)); @@ -283,19 +357,19 @@ Status MatMulBlockQuantizedFp8Weight::ComputeImpl(OpKernelContext* context) cons SafeInt(k), SafeInt(block_size_), std::is_same::value, - Stream(context))); + Stream(context), dense_prefill)); // The row-major [M, N] output is column-major [N, M] to cuBLAS, so an N tile is a row offset. CUBLAS_RETURN_IF_ERROR(cublasGemmHelper( GetCublasHandle(context), - CUBLAS_OP_T, // transB: dequantized weight is [N, K] row-major == K-major [K, N] + dense_prefill ? CUBLAS_OP_N : CUBLAS_OP_T, CUBLAS_OP_N, // transA rows, m_i, k_i, &alpha, b_dequant.get(), - helper.Ldb(transb), + dense_prefill ? n_i : helper.Ldb(transb), reinterpret_cast(a_ptr), helper.Lda(transa), &zero, @@ -305,6 +379,34 @@ Status MatMulBlockQuantizedFp8Weight::ComputeImpl(OpKernelContext* context) cons UseTF32())); } + if (dense_prefill) { + for (size_t segment = 0; segment < prefill_weight_offsets_.size(); ++segment) { + const auto* original = context->Input(6 + static_cast(segment)); + ORT_RETURN_IF_NOT(original != nullptr && original->IsDataType() && + original->Shape().NumDimensions() == 2 && + original->Shape()[0] == k, + "Original prefill weight must match activation type and shape [K, segmentN]"); + const int64_t width = original->Shape()[1]; + const int64_t offset = prefill_weight_offsets_[segment]; + ORT_RETURN_IF_NOT(offset >= 0 && width > 0 && offset <= n - width, + "Original prefill weight segment exceeds output width"); + auto original_output = GetScratchBuffer( + SafeInt(m_i) * SafeInt(width), GetComputeStream(context)); + const int width_i = SafeInt(width); + CUBLAS_RETURN_IF_ERROR(cublasGemmHelper( + GetCublasHandle(context), CUBLAS_OP_N, CUBLAS_OP_N, + width_i, m_i, k_i, &alpha, + reinterpret_cast(original->Data()), width_i, + reinterpret_cast(a_ptr), helper.Lda(transa), + &zero, original_output.get(), width_i, GetDeviceProp(), UseTF32())); + CUDA_RETURN_IF_ERROR(cudaMemcpy2DAsync( + static_cast(y_data) + offset, static_cast(n) * sizeof(CudaT), + original_output.get(), static_cast(width) * sizeof(CudaT), + static_cast(width) * sizeof(CudaT), static_cast(m_i), + cudaMemcpyDeviceToDevice, Stream(context))); + } + } + if (bias != nullptr) { ORT_RETURN_IF_ERROR(LaunchAddBiasBlockScaledFp8( y_data, diff --git a/onnxruntime/contrib_ops/cuda/math/matmul_block_scaled_fp8.cu b/onnxruntime/contrib_ops/cuda/math/matmul_block_scaled_fp8.cu index db0771cf06021..9c7182cfbe602 100644 --- a/onnxruntime/contrib_ops/cuda/math/matmul_block_scaled_fp8.cu +++ b/onnxruntime/contrib_ops/cuda/math/matmul_block_scaled_fp8.cu @@ -36,7 +36,8 @@ __global__ void DequantizeBlockScaledFp8Kernel(T* __restrict__ out, int n, int k, int k_blocks, - int block_size) { + int block_size, + bool transpose) { const long long total = static_cast(n) * k; const long long idx = static_cast(blockIdx.x) * blockDim.x + threadIdx.x; if (idx >= total) { @@ -46,7 +47,8 @@ __global__ void DequantizeBlockScaledFp8Kernel(T* __restrict__ out, const int col = static_cast(idx - static_cast(row) * k); const int blk = col / block_size; const float scale = weight_scale[row * k_blocks + blk]; - out[idx] = from_float(static_cast(b_fp8[idx]) * scale); + out[transpose ? static_cast(col) * n + row : idx] = + from_float(static_cast(b_fp8[idx]) * scale); } // Vectorized dequantization for K % 16 == 0. Each thread converts one 16-element K chunk of a @@ -527,7 +529,20 @@ __device__ __forceinline__ T* Fp8OutputColumn(T* output, int col, int& stride, return output + col; } -template +template +__device__ __forceinline__ T Fp8RoundOutput(float value, int row, int col, int n, + const Fp8SharedExpertEpilogue& epilogue) { + T rounded = from_float(value); + if constexpr (SharedExpert) { + rounded = from_float(to_float(rounded) * epilogue.row_gates[row]); + rounded = from_float(to_float(rounded) + + to_float(static_cast(epilogue.residual)[static_cast(row) * n + col])); + } + return rounded; +} + +template __device__ __forceinline__ void Fp8MmaGemvBody(AType* __restrict__ output, const AType* __restrict__ input_a, const __nv_fp8_e4m3* __restrict__ input_b, @@ -539,7 +554,8 @@ __device__ __forceinline__ void Fp8MmaGemvBody(AType* __restrict__ output, int k, int block_size, int k_blocks, - const Fp8GemvOutputSplit& split) { + const Fp8GemvOutputSplit& split, + const Fp8SharedExpertEpilogue& epilogue = {}) { using Mma = Fp8GemvMma; // Keep the same MMA/KSplit reduction, but remove per-block scaling and QDQ @@ -704,18 +720,22 @@ __device__ __forceinline__ void Fp8MmaGemvBody(AType* __restrict__ output, const int row = (t << 1) + (mt << 3); if (row < m) { if (lo_ok) { - output_lo[static_cast(row) * stride_lo] = from_float(acc[mt][0] + bias_lo); + output_lo[static_cast(row) * stride_lo] = + Fp8RoundOutput(acc[mt][0] + bias_lo, row, col_lo, n, epilogue); } if (hi_ok) { - output_hi[static_cast(row) * stride_hi] = from_float(acc[mt][2] + bias_hi); + output_hi[static_cast(row) * stride_hi] = + Fp8RoundOutput(acc[mt][2] + bias_hi, row, col_hi, n, epilogue); } } if (row + 1 < m) { if (lo_ok) { - output_lo[static_cast(row + 1) * stride_lo] = from_float(acc[mt][1] + bias_lo); + output_lo[static_cast(row + 1) * stride_lo] = + Fp8RoundOutput(acc[mt][1] + bias_lo, row + 1, col_lo, n, epilogue); } if (hi_ok) { - output_hi[static_cast(row + 1) * stride_hi] = from_float(acc[mt][3] + bias_hi); + output_hi[static_cast(row + 1) * stride_hi] = + Fp8RoundOutput(acc[mt][3] + bias_hi, row + 1, col_hi, n, epilogue); } } } @@ -762,6 +782,24 @@ __global__ __launch_bounds__(32 * KSplit, 3) void MatMulBlockScaledFp8MmaGemvKer #undef ORT_FP8_MMA_GEMV_PARAMS #undef ORT_FP8_SPLIT_PARAM +template +__global__ void MatMulFp8SharedExpertKernel( + __nv_bfloat16* output, const __nv_bfloat16* a, const __nv_fp8_e4m3* b, + const float* scales, int m, int n, int k, int block_size, int k_blocks, + Fp8SharedExpertEpilogue epilogue) { + Fp8MmaGemvBody( + output, a, b, scales, nullptr, nullptr, m, n, k, block_size, k_blocks, {}, epilogue); +} + +__global__ void Fp8SharedExpertEpilogueKernel(__nv_bfloat16* output, Fp8SharedExpertEpilogue epilogue, + int64_t count, int n) { + const int64_t index = static_cast(blockIdx.x) * blockDim.x + threadIdx.x; + if (index < count) { + output[index] = Fp8RoundOutput<__nv_bfloat16, true>( + __bfloat162float(output[index]), static_cast(index / n), static_cast(index % n), n, epilogue); + } +} + // Kill switch for A/B testing the tensor-core path against the FMA path in the same binary. bool Fp8GemvMmaEnabled() { static bool const enabled = onnxruntime::ParseEnvironmentVariableWithDefault("ORT_FP8_GEMV_MMA", true); @@ -786,6 +824,80 @@ int Fp8MmaGemvDispatchMaxM() { #endif // !DISABLE_FLOAT8_TYPES && defined(CUDA_VERSION) && CUDA_VERSION >= 11080 +int ApplyFp8MmaKSplitOverride(int k_split, int m, int n, int k); + +Status LaunchFp8SharedExpertEpilogue(void* y, const Fp8SharedExpertEpilogue& epilogue, + int m, int n, cudaStream_t stream) { +#if !defined(DISABLE_FLOAT8_TYPES) && defined(CUDA_VERSION) && CUDA_VERSION >= 11080 + const int64_t count = static_cast(m) * n; + ORT_RETURN_IF_NOT(m >= 0 && n > 0 && epilogue.row_gates && epilogue.residual, + "Shared expert epilogue requires valid dimensions, gates and residual"); + if (count == 0) return Status::OK(); + Fp8SharedExpertEpilogueKernel<<((count + 255) / 256), 256, 0, stream>>>( + static_cast<__nv_bfloat16*>(y), epilogue, count, n); + return CUDA_CALL(cudaGetLastError()); +#else + ORT_UNUSED_PARAMETER(y); + ORT_UNUSED_PARAMETER(epilogue); + ORT_UNUSED_PARAMETER(m); + ORT_UNUSED_PARAMETER(n); + ORT_UNUSED_PARAMETER(stream); + return ORT_MAKE_STATUS(ONNXRUNTIME, NOT_IMPLEMENTED, "Shared expert requires FP8 support."); +#endif +} + +Status LaunchMatMulFp8SharedExpert(void* y, const void* a, const void* weight, + const float* scales, int m, int n, int k, int block_size, + const Fp8SharedExpertEpilogue& epilogue, + const cudaDeviceProp& device_prop, cudaStream_t stream) { +#if !defined(DISABLE_FLOAT8_TYPES) && defined(CUDA_VERSION) && CUDA_VERSION >= 11080 + ORT_RETURN_IF_NOT(m >= 1 && m <= 2 && k >= 256 && k % 64 == 0 && block_size % 64 == 0 && + device_prop.major >= 8 && epilogue.row_gates && epilogue.residual, + "Shared expert fused GEMV requires SM80+, M <= 2 and aligned K/block size"); + const int selected = PickGenericFp8MmaKSplit(n, k / 64); + const int k_split = ApplyFp8MmaKSplitOverride(selected, m, n, k); + static const int per_column_s2 = + ParseEnvironmentVariableWithDefault("ORT_FP8_GEMV_PER_COLUMN_S2", 0); + ORT_RETURN_IF_NOT(per_column_s2 == 0 || per_column_s2 == 1, + "ORT_FP8_GEMV_PER_COLUMN_S2 must be 0 or 1"); + const auto launch = [&]() { + const auto variant = [&]() { + MatMulFp8SharedExpertKernel<<((n + 15) / 16), + dim3(32, KSplit), 0, stream>>>( + static_cast<__nv_bfloat16*>(y), static_cast(a), + static_cast(weight), scales, m, n, k, block_size, + (k + block_size - 1) / block_size, epilogue); + }; + if (per_column_s2 && m == 2 && block_size == k) { + variant.template operator()(); + } else { + variant.template operator()(); + } + }; + switch (k_split) { + case 4: launch.template operator()<4>(); break; + case 8: launch.template operator()<8>(); break; + case 16: launch.template operator()<16>(); break; + case 32: launch.template operator()<32>(); break; + default: return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Unsupported shared expert K split: ", k_split); + } + return CUDA_CALL(cudaGetLastError()); +#else + ORT_UNUSED_PARAMETER(y); + ORT_UNUSED_PARAMETER(a); + ORT_UNUSED_PARAMETER(weight); + ORT_UNUSED_PARAMETER(scales); + ORT_UNUSED_PARAMETER(m); + ORT_UNUSED_PARAMETER(n); + ORT_UNUSED_PARAMETER(k); + ORT_UNUSED_PARAMETER(block_size); + ORT_UNUSED_PARAMETER(epilogue); + ORT_UNUSED_PARAMETER(device_prop); + ORT_UNUSED_PARAMETER(stream); + return ORT_MAKE_STATUS(ONNXRUNTIME, NOT_IMPLEMENTED, "Shared expert requires FP8 support."); +#endif +} + Status LaunchDequantizeBlockScaledFp8(void* b_dequant, const void* b_fp8, const float* weight_scale, @@ -793,7 +905,8 @@ Status LaunchDequantizeBlockScaledFp8(void* b_dequant, int k, int block_size, bool is_bf16, - cudaStream_t stream) { + cudaStream_t stream, + bool transpose) { #if !defined(DISABLE_FLOAT8_TYPES) && defined(CUDA_VERSION) && CUDA_VERSION >= 11080 const long long total = static_cast(n) * k; if (total == 0) { @@ -804,7 +917,7 @@ Status LaunchDequantizeBlockScaledFp8(void* b_dequant, // Fast path: K is a multiple of 16, so each thread owns one aligned 16-element chunk and both the // FP8 load and the FP16/BF16 store are fully coalesced. This is the common prefill layout. - if (k % 16 == 0) { + if (!transpose && k % 16 == 0) { const int kv = k / 16; // 16-element chunks per row constexpr int kThreads = 256; const unsigned int grid_x = static_cast((kv + kThreads - 1) / kThreads); @@ -826,10 +939,10 @@ Status LaunchDequantizeBlockScaledFp8(void* b_dequant, const int blocks = static_cast((total + kThreads - 1) / kThreads); if (is_bf16) { DequantizeBlockScaledFp8Kernel<__nv_bfloat16><<>>( - reinterpret_cast<__nv_bfloat16*>(b_dequant), b, weight_scale, n, k, k_blocks, block_size); + reinterpret_cast<__nv_bfloat16*>(b_dequant), b, weight_scale, n, k, k_blocks, block_size, transpose); } else { DequantizeBlockScaledFp8Kernel<<>>( - reinterpret_cast(b_dequant), b, weight_scale, n, k, k_blocks, block_size); + reinterpret_cast(b_dequant), b, weight_scale, n, k, k_blocks, block_size, transpose); } return CUDA_CALL(cudaGetLastError()); #else @@ -840,6 +953,7 @@ Status LaunchDequantizeBlockScaledFp8(void* b_dequant, ORT_UNUSED_PARAMETER(k); ORT_UNUSED_PARAMETER(block_size); ORT_UNUSED_PARAMETER(is_bf16); + ORT_UNUSED_PARAMETER(transpose); ORT_UNUSED_PARAMETER(stream); return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "MatMulBlockQuantizedFp8Weight requires CUDA 11.8 or later."); #endif diff --git a/onnxruntime/contrib_ops/cuda/math/matmul_block_scaled_fp8.h b/onnxruntime/contrib_ops/cuda/math/matmul_block_scaled_fp8.h index b54a37da08a87..6d0e5bf338162 100644 --- a/onnxruntime/contrib_ops/cuda/math/matmul_block_scaled_fp8.h +++ b/onnxruntime/contrib_ops/cuda/math/matmul_block_scaled_fp8.h @@ -35,6 +35,8 @@ class MatMulBlockQuantizedFp8Weight final : public onnxruntime::cuda::CudaKernel std::vector split_sizes_; size_t max_dequant_scratch_bytes_; bool enable_deep_gemm_; + bool shared_expert_; + std::vector prefill_weight_offsets_; }; struct Fp8GemvOutputSplit { @@ -44,6 +46,18 @@ struct Fp8GemvOutputSplit { int count = 0; }; +struct Fp8SharedExpertEpilogue { + const float* row_gates = nullptr; + const void* residual = nullptr; +}; + +Status LaunchMatMulFp8SharedExpert(void* y, const void* a, const void* weight, + const float* scales, int m, int n, int k, int block_size, + const Fp8SharedExpertEpilogue& epilogue, + const cudaDeviceProp& device_prop, cudaStream_t stream); +Status LaunchFp8SharedExpertEpilogue(void* y, const Fp8SharedExpertEpilogue& epilogue, + int m, int n, cudaStream_t stream); + bool MatMulBlockScaledFp8GemvSupportsSplitOutput(int m, int k, int block_size, const cudaDeviceProp& device_prop); @@ -69,7 +83,8 @@ Status LaunchDequantizeBlockScaledFp8(void* b_dequant, int k, int block_size, bool is_bf16, - cudaStream_t stream); + cudaStream_t stream, + bool transpose = false); // Adds a per-column bias of shape [N] to a [M, N] row-major output in place. Status LaunchAddBiasBlockScaledFp8(void* y, diff --git a/onnxruntime/contrib_ops/cuda/moe/qmoe_kernels.cu b/onnxruntime/contrib_ops/cuda/moe/qmoe_kernels.cu index acfe95c359be8..89b7e410a4afb 100644 --- a/onnxruntime/contrib_ops/cuda/moe/qmoe_kernels.cu +++ b/onnxruntime/contrib_ops/cuda/moe/qmoe_kernels.cu @@ -304,7 +304,7 @@ __global__ void FusedSoftmaxTopKExpertMapsKernel( expert_offsets, rows, experts, k); } -template +template __global__ void WarpSelectSoftmaxTopKExpertMapsKernel( const T* logits, float* scales, int* indices, int* row_to_source, int* source_to_row, int* sorted_experts, int64_t* expert_offsets, int rows) { @@ -327,10 +327,22 @@ __global__ void WarpSelectSoftmaxTopKExpertMapsKernel( uint64_t best = keys[0]; #pragma unroll for (int i = 1; i < 16; ++i) best = best > keys[i] ? best : keys[i]; +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800 + if constexpr (HardwareReduce) { + // Reduce the packed key lexicographically, retaining stable expert-index ties. + const unsigned score = static_cast(best >> 32); + const unsigned maximum_score = __reduce_max_sync(0xffffffff, score); + const unsigned index = __reduce_max_sync( + 0xffffffff, score == maximum_score ? static_cast(best) : 0u); + best = (static_cast(maximum_score) << 32) | index; + } else +#endif + { #pragma unroll - for (int offset = 16; offset > 0; offset >>= 1) { - const uint64_t other = __shfl_xor_sync(0xffffffff, static_cast(best), offset); - best = best > other ? best : other; + for (int offset = 16; offset > 0; offset >>= 1) { + const uint64_t other = __shfl_xor_sync(0xffffffff, static_cast(best), offset); + best = best > other ? best : other; + } } #pragma unroll for (int i = 0; i < 16; ++i) { @@ -355,7 +367,7 @@ __global__ void WarpSelectSoftmaxTopKExpertMapsKernel( } __syncthreads(); BuildSmallDecodeExpertMaps(selected, row_to_source, source_to_row, sorted_experts, - expert_offsets, rows, kExperts, k); + expert_offsets, rows, kExperts, k); } template @@ -369,9 +381,12 @@ bool TryLaunchSoftmaxTopKExpertMaps( } static const int warp_select = ParseEnvironmentVariableWithDefault("ORT_MOE_ROUTING_WARP_SELECT", 0); - ORT_ENFORCE(warp_select == 0 || warp_select == 1, - "ORT_MOE_ROUTING_WARP_SELECT must be 0 or 1."); - if (warp_select && experts == 512 && k == 10) { + ORT_ENFORCE(warp_select >= 0 && warp_select <= 2, + "ORT_MOE_ROUTING_WARP_SELECT must be 0, 1 or 2."); + if (warp_select == 2 && experts == 512 && k == 10) { + WarpSelectSoftmaxTopKExpertMapsKernel<<<1, 128, 0, stream>>>( + logits, scales, indices, row_to_source, source_to_row, sorted_experts, expert_offsets, rows); + } else if (warp_select == 1 && experts == 512 && k == 10) { WarpSelectSoftmaxTopKExpertMapsKernel<<<1, 128, 0, stream>>>( logits, scales, indices, row_to_source, source_to_row, sorted_experts, expert_offsets, rows); } else { diff --git a/onnxruntime/core/graph/contrib_ops/contrib_defs.cc b/onnxruntime/core/graph/contrib_ops/contrib_defs.cc index 1acf856971369..cec17789ad310 100644 --- a/onnxruntime/core/graph/contrib_ops/contrib_defs.cc +++ b/onnxruntime/core/graph/contrib_ops/contrib_defs.cc @@ -3202,6 +3202,18 @@ The partition widths must be positive and sum to N.)DOC") AttributeProto::INT, static_cast(128)) .Attr("split_sizes", "Optional last-dimension output widths (one to eight positive sizes).", AttributeProto::INTS, OPTIONAL_VALUE) + .Attr("shared_expert", + "When one, BF16 A is the up branch and shared_silu_gate is the equal-shaped gate branch. " + "Multiply A by BF16(shared_silu_gate * BF16(sigmoid(shared_silu_gate))) before projection. " + "Round the projection to BF16, multiply by the BF16 sigmoid of the BF16 dot product " + "shared_gate_activation @ shared_gate_weight, round to BF16, then add shared_residual. " + "Sigmoid uses BF16 intermediates. Requires all shared inputs and excludes bias, a_scale, " + "split_sizes and original prefill weights. Default zero preserves ordinary projection.", + AttributeProto::INT, static_cast(0)) + .Attr("prefill_weight_offsets", + "Output-column offsets for optional original-type [K, segmentN] prefill weights. " + "Used only when prefill_numerics is one; decode retains FP8 weights.", + AttributeProto::INTS, std::vector{}) .Input(0, "A", "Row-major FP16/BF16 activation of shape [..., K].", "T") .Input(1, "B", "Row-major FP8 E4M3 weight of shape [N, K].", "T1") .Input(2, "b_scale", "Per-block FP32 weight scales of shape [N, ceil(K / block_size)].", "T2") @@ -3211,17 +3223,65 @@ The partition widths must be positive and sum to N.)DOC") "its FP16/BF16 precision.", "T2", OpSchema::Optional) .Input(4, "bias", "Optional bias of shape [N].", "T", OpSchema::Optional) + .Input(5, "prefill_numerics", + "Optional CUDA CPU-memory int32 control of shape [1]. Zero or omission preserves existing " + "dispatch; one selects dense BF16 dequantization/GEMM rounding during eager prefill. " + "Decode CUDA graph capture must bind an immutable zero.", + "TI", OpSchema::Optional) + .Input(6, "prefill_weight", "Optional original-type [K, segmentN] eager-prefill weight.", "T", OpSchema::Optional) + .Input(7, "prefill_weight2", "Optional second original-type eager-prefill weight segment.", "T", OpSchema::Optional) + .Input(8, "prefill_weight3", "Optional third original-type eager-prefill weight segment.", "T", OpSchema::Optional) + .Input(9, "shared_silu_gate", "BF16 gate branch matching A, for shared_expert=1.", "T", OpSchema::Optional) + .Input(10, "shared_gate_activation", "BF16 projected-gate activation with matching leading axes.", "T", OpSchema::Optional) + .Input(11, "shared_gate_weight", "BF16 [gate_input_width, 1] projected-gate weight.", "T", OpSchema::Optional) + .Input(12, "shared_residual", "BF16 residual matching the unsplit output.", "T", OpSchema::Optional) .Output(0, "Y", "Output(s) of shape [..., N] or [..., split_sizes[i]] in the activation type.", "T", OpSchema::Variadic) .TypeConstraint("T", {"tensor(float16)", "tensor(bfloat16)"}, "Constrain activation, bias and output to FP16 or BF16.") .TypeConstraint("T1", {"tensor(float8e4m3fn)"}, "Constrain weight to FP8 E4M3.") .TypeConstraint("T2", {"tensor(float)"}, "Constrain scales to FP32.") + .TypeConstraint("TI", {"tensor(int32)"}, "CPU prefill numerical-mode control.") .TypeAndShapeInferenceFunction([](ONNX_NAMESPACE::InferenceContext& ctx) { const auto* split = ctx.getAttribute("split_sizes"); + const auto shared = getAttribute(ctx, "shared_expert", 0); + if (shared != 0 && shared != 1) { + fail_shape_inference("shared_expert must be 0 or 1."); + } + if (shared) { + if (split != nullptr || (ctx.getNumInputs() > 3 && ctx.getInputType(3)) || + (ctx.getNumInputs() > 4 && ctx.getInputType(4)) || + (ctx.getAttribute("prefill_weight_offsets") && + ctx.getAttribute("prefill_weight_offsets")->ints_size() != 0)) { + fail_shape_inference("shared_expert excludes split_sizes, bias, a_scale and prefill weights."); + } + if (ctx.getInputType(0) && + ctx.getInputType(0)->tensor_type().elem_type() != ONNX_NAMESPACE::TensorProto::BFLOAT16) { + fail_type_inference("shared_expert requires BF16."); + } + for (size_t index = 9; index <= 12; ++index) { + if (ctx.getNumInputs() <= index || ctx.getInputType(index) == nullptr) { + fail_shape_inference("shared_expert requires all shared inputs."); + } + if (ctx.getInputType(index)->tensor_type().elem_type() != ONNX_NAMESPACE::TensorProto::BFLOAT16) { + fail_type_inference("shared_expert requires BF16 shared inputs."); + } + } + for (size_t index = 6; index <= 8 && index < ctx.getNumInputs(); ++index) { + if (ctx.getInputType(index)) { + fail_shape_inference("shared_expert excludes original prefill weights."); + } + } + } else { + for (size_t index = 9; index < ctx.getNumInputs(); ++index) { + if (ctx.getInputType(index)) { + fail_shape_inference("Shared expert inputs require shared_expert=1."); + } + } + } if ((split == nullptr && ctx.getNumOutputs() != 1) || (split != nullptr && (split->ints_size() < 1 || split->ints_size() > 8 || - static_cast(split->ints_size()) != ctx.getNumOutputs()))) { + static_cast(split->ints_size()) != ctx.getNumOutputs()))) { fail_shape_inference("Output count must match split_sizes, or be one when split_sizes is absent."); } int64_t split_total = 0; @@ -3253,6 +3313,36 @@ The partition widths must be positive and sum to N.)DOC") *output_shape.add_dim() = a_shape.dim(i); } *output_shape.add_dim() = b_shape.dim(0); + if (shared) { + const auto check_shape = [&](size_t input, const ONNX_NAMESPACE::TensorShapeProto& expected) { + if (!hasInputShape(ctx, input)) return; + const auto& actual = getInputShape(ctx, input); + if (actual.dim_size() != expected.dim_size()) { + fail_shape_inference("shared_expert input ranks are incompatible."); + } + for (int axis = 0; axis < actual.dim_size(); ++axis) { + if (actual.dim(axis).has_dim_value() && expected.dim(axis).has_dim_value() && + actual.dim(axis).dim_value() != expected.dim(axis).dim_value()) { + fail_shape_inference("shared_expert input shapes are incompatible."); + } + } + }; + check_shape(9, a_shape); + check_shape(12, output_shape); + if (hasInputShape(ctx, 10)) { + auto gate_shape = a_shape; + const auto& activation = getInputShape(ctx, 10); + if (activation.dim_size() != a_shape.dim_size()) { + fail_shape_inference("shared_expert gate activation rank must match A."); + } + *gate_shape.mutable_dim(gate_shape.dim_size() - 1) = activation.dim(activation.dim_size() - 1); + check_shape(10, gate_shape); + ONNX_NAMESPACE::TensorShapeProto weight_shape; + *weight_shape.add_dim() = activation.dim(activation.dim_size() - 1); + weight_shape.add_dim()->set_dim_value(1); + check_shape(11, weight_shape); + } + } if (split != nullptr && b_shape.dim(0).has_dim_value() && split_total != b_shape.dim(0).dim_value()) { fail_shape_inference("split_sizes must sum to the weight's N dimension."); diff --git a/onnxruntime/test/contrib_ops/cuda_kernels/softmax_topk_kernel_test.cc b/onnxruntime/test/contrib_ops/cuda_kernels/softmax_topk_kernel_test.cc index 95e27c481a58b..9d633a7066cce 100644 --- a/onnxruntime/test/contrib_ops/cuda_kernels/softmax_topk_kernel_test.cc +++ b/onnxruntime/test/contrib_ops/cuda_kernels/softmax_topk_kernel_test.cc @@ -269,8 +269,10 @@ TEST(CUDA_EP_Unittest, SoftmaxTopK_RejectsKGreaterThanExperts) { } template -void RunFusedRoutingMapsTest(int rows, int experts, int k, bool ties = false, bool nonfinite = false) { - auto values = MakeLogits(rows, experts); +void RunFusedRoutingMapsTest(int rows, int experts, int k, bool ties = false, bool nonfinite = false, + const std::vector* custom_values = nullptr) { + auto values = custom_values ? *custom_values : MakeLogits(rows, experts); + ASSERT_EQ(values.size(), static_cast(rows) * experts); if (ties) std::fill(values.begin(), values.end(), 0.0f); if (nonfinite) { std::fill(values.begin(), values.end(), -std::numeric_limits::infinity()); @@ -381,6 +383,33 @@ TEST(CUDA_EP_Unittest, FusedSoftmaxTopKExpertMaps_WarpSelectSpecialized) { } } +TEST(CUDA_EP_Unittest, FusedSoftmaxTopKExpertMaps_WarpSelectAdversarialKeys) { + for (int rows : {1, 2}) { + for (int pattern = 0; pattern < 4; ++pattern) { + SCOPED_TRACE(::testing::Message() << "rows=" << rows << " pattern=" << pattern); + auto values = MakeLogits(rows, 512); + for (int row = 0; row < rows; ++row) { + for (int expert = 0; expert < 512; ++expert) { + float& value = values[row * 512 + expert]; + if (pattern == 0) { + value = expert % 32 == row ? 32.0f - expert / 32 : -32.0f; + } else if (pattern == 1) { + value = expert % 2 ? -0.0f : 0.0f; + } else if (pattern == 2) { + value = expert % 3 ? -std::numeric_limits::infinity() + : std::numeric_limits::quiet_NaN(); + } else { + value = (expert + row) % 7 == 0 ? 1000.0f : -1000.0f; + } + } + } + RunFusedRoutingMapsTest(rows, 512, 10, false, false, &values); + RunFusedRoutingMapsTest(rows, 512, 10, false, false, &values); + RunFusedRoutingMapsTest<__nv_bfloat16>(rows, 512, 10, false, false, &values); + } + } +} + } // namespace } // namespace test } // namespace onnxruntime diff --git a/onnxruntime/test/contrib_ops/matmul_block_scaled_fp8_test.cc b/onnxruntime/test/contrib_ops/matmul_block_scaled_fp8_test.cc index cd235a45c6775..b7d94c361e274 100644 --- a/onnxruntime/test/contrib_ops/matmul_block_scaled_fp8_test.cc +++ b/onnxruntime/test/contrib_ops/matmul_block_scaled_fp8_test.cc @@ -148,6 +148,153 @@ void TestFp8SplitOutputs() { } } +TEST(MatMulBlockQuantizedFp8WeightOpTest, SharedExpertNumericsAndFallbacks) { + if (!HasCudaEnvironment(800)) GTEST_SKIP() << "SM80+ CUDA device required."; + const auto round = [](float x) { return static_cast(BFloat16(x)); }; + for (const int64_t m : {0, 1, 2, 3, 65}) { + for (const int64_t k : {64, 256, 640}) { + for (const int32_t prefill : {0, 1}) { + SCOPED_TRACE("M=" + std::to_string(m) + " K=" + std::to_string(k) + + " prefill=" + std::to_string(prefill)); + constexpr int64_t n = 17; + constexpr int64_t gate_width = 32; + const int64_t block_size = k == 256 ? 128 : k; + const int64_t k_blocks = k / block_size; + const std::vector input_shape = m == 3 ? std::vector{1, m, k} : std::vector{m, k}; + const std::vector output_shape = m == 3 ? std::vector{1, m, n} : std::vector{m, n}; + const std::vector activation_shape = m == 3 ? std::vector{1, m, gate_width} : std::vector{m, gate_width}; + std::vector up(m * k), gate(m * k), prepared(m * k), residual(m * n), expected(m * n); + for (int64_t row = 0; row < m; ++row) { + for (int64_t col = 0; col < k; ++col) { + const float g = static_cast(col % 3 - 1); + const float u = static_cast((col + row) % 5 - 2) / 16.f; + up[row * k + col] = BFloat16(u); + gate[row * k + col] = BFloat16(g); + const float sig = g > 0 ? round(1.f / round(1.f + round(std::exp(-g)))) + : round(1.f - round(1.f / round(1.f + round(std::exp(g))))); + prepared[row * k + col] = BFloat16(round(u * round(g * sig))); + } + for (int64_t col = 0; col < n; ++col) { + residual[row * n + col] = BFloat16(static_cast(col % 5 - 2) / 8.f); + } + } + const auto run = [](OpTester& op) { + std::vector> providers; + providers.push_back(DefaultCudaExecutionProvider()); + op.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &providers); + }; + OpTester silu("SigmoidGateMul", 1, kMSDomain); + silu.AddAttribute("silu_gate", 1); + silu.AddAttribute("sigmoid_in_fp32", 0); + silu.AddInput("up", input_shape, up); + silu.AddInput("gate", input_shape, gate); + silu.AddOptionalInputEdge(); + silu.AddInput("prefill", {1}, {prefill}); + silu.AddOutput("prepared", input_shape, prepared); + silu.SetOutputTolerance(0.f, 0.f); + run(silu); + + std::vector projected(m * n); + OpTester projection("MatMulBlockQuantizedFp8Weight", 1, kMSDomain); + projection.AddAttribute("block_size", block_size); + projection.AddInput("A", input_shape, prepared); + projection.AddInput("B", {n, k}, MakeConstRowWeight(std::vector(n, 1.f), k)); + projection.AddInput("scale", {n, k_blocks}, std::vector(n * k_blocks, .125f)); + projection.AddOptionalInputEdge(); + projection.AddOptionalInputEdge(); + projection.AddInput("prefill", {1}, {prefill}); + projection.AddOutput("down", output_shape, projected); + projection.SetCustomOutputVerifier([&](const std::vector& fetches, const std::string&) { + ASSERT_EQ(fetches.size(), 1u); + const auto& output = fetches[0].Get(); + projected.assign(output.Data(), output.Data() + m * n); + }); + run(projection); + + std::vector gate_activation(m * gate_width); + const std::vector gate_weight(gate_width, BFloat16(.125f)); + for (int64_t i = 0; i < m * gate_width; ++i) { + gate_activation[i] = BFloat16(static_cast((i / gate_width) % 3 - 1)); + } + OpTester post("SigmoidGateMul", 1, kMSDomain); + post.AddAttribute("broadcast_gate", 1); + post.AddAttribute("sigmoid_in_fp32", 0); + post.AddInput("down", output_shape, projected); + post.AddInput("activation", activation_shape, gate_activation); + post.AddInput("residual", output_shape, residual); + post.AddInput("prefill", {1}, {prefill}); + post.AddInput("weight", {gate_width, 1}, gate_weight); + post.AddOutput("Y", output_shape, expected); + post.SetCustomOutputVerifier([&](const std::vector& fetches, const std::string&) { + ASSERT_EQ(fetches.size(), 1u); + const auto& output = fetches[0].Get(); + expected.assign(output.Data(), output.Data() + m * n); + }); + run(post); + + OpTester test("MatMulBlockQuantizedFp8Weight", 1, kMSDomain); + test.AddAttribute("block_size", block_size); + test.AddAttribute("shared_expert", 1); + test.AddInput("up", input_shape, up); + test.AddInput("B", {n, k}, MakeConstRowWeight(std::vector(n, 1.f), k)); + test.AddInput("scale", {n, k_blocks}, std::vector(n * k_blocks, .125f)); + test.AddOptionalInputEdge(); + test.AddOptionalInputEdge(); + test.AddInput("prefill", {1}, {prefill}); + for (int i = 0; i < 3; ++i) test.AddOptionalInputEdge(); + test.AddInput("gate", input_shape, gate); + test.AddInput("gate_activation", activation_shape, gate_activation); + test.AddInput("gate_weight", {gate_width, 1}, gate_weight); + test.AddInput("residual", output_shape, residual); + test.AddOutput("Y", output_shape, expected); + test.SetOutputTolerance(0.f, 0.f); + run(test); + } + } + } +} + +TEST(MatMulBlockQuantizedFp8WeightOpTest, SharedExpertRejectsMissingInputs) { + OpTester test("MatMulBlockQuantizedFp8Weight", 1, kMSDomain); + test.AddAttribute("shared_expert", 1); + test.AddInput("A", {1, 256}, std::vector(256)); + test.AddInput("B", {1, 256}, MakeConstRowWeight({1.f}, 256)); + test.AddInput("scale", {1, 2}, {1.f, 1.f}); + test.AddOutput("Y", {1, 1}, {BFloat16(0.f)}); + test.Run(OpTester::ExpectResult::kExpectFailure, "shared_expert requires all shared inputs"); +} + +TEST(MatMulBlockQuantizedFp8WeightOpTest, SharedExpertRejectsInvalidContracts) { + for (int kind = 0; kind < 7; ++kind) { + SCOPED_TRACE(kind); + OpTester test("MatMulBlockQuantizedFp8Weight", 1, kMSDomain); + test.AddAttribute("block_size", 256); + test.AddAttribute("shared_expert", kind == 5 ? 2 : kind == 6 ? 0 + : 1); + if (kind == 4) test.AddAttribute>("split_sizes", {8}); + test.AddInput("A", {2, 256}, std::vector(512)); + test.AddInput("B", {8, 256}, MakeConstRowWeight(std::vector(8, 1.f), 256)); + test.AddInput("scale", {8, 1}, std::vector(8, 1.f)); + if (kind == 3) { + test.AddInput("a_scale", {1}, {1.f}); + } else { + test.AddOptionalInputEdge(); + } + test.AddOptionalInputEdge(); + test.AddInput("prefill", {1}, {0}); + for (int i = 0; i < 3; ++i) test.AddOptionalInputEdge(); + test.AddInput("gate", {kind == 0 ? 3 : 2, 256}, std::vector((kind == 0 ? 3 : 2) * 256)); + test.AddInput("activation", {2, 8}, std::vector(16)); + test.AddInput("gate_weight", {kind == 1 ? 9 : 8, 1}, std::vector(kind == 1 ? 9 : 8)); + test.AddInput("residual", {kind == 2 ? 3 : 2, 8}, std::vector((kind == 2 ? 3 : 2) * 8)); + test.AddOutput("Y", {2, 8}, std::vector(16)); + const std::string error = kind <= 2 ? "shared_expert input shapes are incompatible" : kind <= 4 ? "shared_expert excludes" + : kind == 5 ? "shared_expert must be 0 or 1" + : "Shared expert inputs require shared_expert=1"; + test.Run(OpTester::ExpectResult::kExpectFailure, error); + } +} + TEST(MatMulBlockQuantizedFp8WeightOpTest, SplitOutputsFp16) { TestFp8SplitOutputs(); } From 2c6795d22892798acc55e673f91e2c4a654762df Mon Sep 17 00:00:00 2001 From: Akshay Sonawane Date: Thu, 22 Oct 2026 17:32:10 +0000 Subject: [PATCH 08/12] Apply lintrunner formatting to PR changes Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../cuda/math/matmul_block_scaled_fp8.cu | 33 ++++++++++++------- .../cuda/math/matmul_block_scaled_fp8.h | 8 ++--- .../contrib_ops/cuda/moe/moe_quantization.cc | 4 +-- .../contrib_ops/cuda/moe/qmoe_kernels.cu | 2 +- .../core/graph/contrib_ops/contrib_defs.cc | 2 +- 5 files changed, 29 insertions(+), 20 deletions(-) diff --git a/onnxruntime/contrib_ops/cuda/math/matmul_block_scaled_fp8.cu b/onnxruntime/contrib_ops/cuda/math/matmul_block_scaled_fp8.cu index 9c7182cfbe602..32bfcf239fb41 100644 --- a/onnxruntime/contrib_ops/cuda/math/matmul_block_scaled_fp8.cu +++ b/onnxruntime/contrib_ops/cuda/math/matmul_block_scaled_fp8.cu @@ -531,7 +531,7 @@ __device__ __forceinline__ T* Fp8OutputColumn(T* output, int col, int& stride, template __device__ __forceinline__ T Fp8RoundOutput(float value, int row, int col, int n, - const Fp8SharedExpertEpilogue& epilogue) { + const Fp8SharedExpertEpilogue& epilogue) { T rounded = from_float(value); if constexpr (SharedExpert) { rounded = from_float(to_float(rounded) * epilogue.row_gates[row]); @@ -792,7 +792,7 @@ __global__ void MatMulFp8SharedExpertKernel( } __global__ void Fp8SharedExpertEpilogueKernel(__nv_bfloat16* output, Fp8SharedExpertEpilogue epilogue, - int64_t count, int n) { + int64_t count, int n) { const int64_t index = static_cast(blockIdx.x) * blockDim.x + threadIdx.x; if (index < count) { output[index] = Fp8RoundOutput<__nv_bfloat16, true>( @@ -827,7 +827,7 @@ int Fp8MmaGemvDispatchMaxM() { int ApplyFp8MmaKSplitOverride(int k_split, int m, int n, int k); Status LaunchFp8SharedExpertEpilogue(void* y, const Fp8SharedExpertEpilogue& epilogue, - int m, int n, cudaStream_t stream) { + int m, int n, cudaStream_t stream) { #if !defined(DISABLE_FLOAT8_TYPES) && defined(CUDA_VERSION) && CUDA_VERSION >= 11080 const int64_t count = static_cast(m) * n; ORT_RETURN_IF_NOT(m >= 0 && n > 0 && epilogue.row_gates && epilogue.residual, @@ -847,9 +847,9 @@ Status LaunchFp8SharedExpertEpilogue(void* y, const Fp8SharedExpertEpilogue& epi } Status LaunchMatMulFp8SharedExpert(void* y, const void* a, const void* weight, - const float* scales, int m, int n, int k, int block_size, - const Fp8SharedExpertEpilogue& epilogue, - const cudaDeviceProp& device_prop, cudaStream_t stream) { + const float* scales, int m, int n, int k, int block_size, + const Fp8SharedExpertEpilogue& epilogue, + const cudaDeviceProp& device_prop, cudaStream_t stream) { #if !defined(DISABLE_FLOAT8_TYPES) && defined(CUDA_VERSION) && CUDA_VERSION >= 11080 ORT_RETURN_IF_NOT(m >= 1 && m <= 2 && k >= 256 && k % 64 == 0 && block_size % 64 == 0 && device_prop.major >= 8 && epilogue.row_gates && epilogue.residual, @@ -863,7 +863,7 @@ Status LaunchMatMulFp8SharedExpert(void* y, const void* a, const void* weight, const auto launch = [&]() { const auto variant = [&]() { MatMulFp8SharedExpertKernel<<((n + 15) / 16), - dim3(32, KSplit), 0, stream>>>( + dim3(32, KSplit), 0, stream>>>( static_cast<__nv_bfloat16*>(y), static_cast(a), static_cast(weight), scales, m, n, k, block_size, (k + block_size - 1) / block_size, epilogue); @@ -875,11 +875,20 @@ Status LaunchMatMulFp8SharedExpert(void* y, const void* a, const void* weight, } }; switch (k_split) { - case 4: launch.template operator()<4>(); break; - case 8: launch.template operator()<8>(); break; - case 16: launch.template operator()<16>(); break; - case 32: launch.template operator()<32>(); break; - default: return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Unsupported shared expert K split: ", k_split); + case 4: + launch.template operator()<4>(); + break; + case 8: + launch.template operator()<8>(); + break; + case 16: + launch.template operator()<16>(); + break; + case 32: + launch.template operator()<32>(); + break; + default: + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Unsupported shared expert K split: ", k_split); } return CUDA_CALL(cudaGetLastError()); #else diff --git a/onnxruntime/contrib_ops/cuda/math/matmul_block_scaled_fp8.h b/onnxruntime/contrib_ops/cuda/math/matmul_block_scaled_fp8.h index 6d0e5bf338162..155132fa04da0 100644 --- a/onnxruntime/contrib_ops/cuda/math/matmul_block_scaled_fp8.h +++ b/onnxruntime/contrib_ops/cuda/math/matmul_block_scaled_fp8.h @@ -52,11 +52,11 @@ struct Fp8SharedExpertEpilogue { }; Status LaunchMatMulFp8SharedExpert(void* y, const void* a, const void* weight, - const float* scales, int m, int n, int k, int block_size, - const Fp8SharedExpertEpilogue& epilogue, - const cudaDeviceProp& device_prop, cudaStream_t stream); + const float* scales, int m, int n, int k, int block_size, + const Fp8SharedExpertEpilogue& epilogue, + const cudaDeviceProp& device_prop, cudaStream_t stream); Status LaunchFp8SharedExpertEpilogue(void* y, const Fp8SharedExpertEpilogue& epilogue, - int m, int n, cudaStream_t stream); + int m, int n, cudaStream_t stream); bool MatMulBlockScaledFp8GemvSupportsSplitOutput(int m, int k, int block_size, const cudaDeviceProp& device_prop); diff --git a/onnxruntime/contrib_ops/cuda/moe/moe_quantization.cc b/onnxruntime/contrib_ops/cuda/moe/moe_quantization.cc index db198230f53e5..727161e6175c4 100644 --- a/onnxruntime/contrib_ops/cuda/moe/moe_quantization.cc +++ b/onnxruntime/contrib_ops/cuda/moe/moe_quantization.cc @@ -2423,8 +2423,8 @@ Status QMoE::ComputeInternal(OpKernelContext* context) const { #endif if (!fused_expert_maps && !TryLaunchSmallDecodeExpertMaps(expert_indices, p_r2u, unpermuted_row_to_permuted_row, - p_exp, p_efto, static_cast(num_rows), num_experts, - static_cast(k_), stream) && + p_exp, p_efto, static_cast(num_rows), num_experts, + static_cast(k_), stream) && !ck::fusedBuildExpertMapsSortFirstToken( expert_indices, p_r2u, unpermuted_row_to_permuted_row, p_exp, p_efto, num_rows, num_experts, static_cast(k_), 0, num_experts, stream)) { diff --git a/onnxruntime/contrib_ops/cuda/moe/qmoe_kernels.cu b/onnxruntime/contrib_ops/cuda/moe/qmoe_kernels.cu index bf845513c5f91..0135a11152252 100644 --- a/onnxruntime/contrib_ops/cuda/moe/qmoe_kernels.cu +++ b/onnxruntime/contrib_ops/cuda/moe/qmoe_kernels.cu @@ -368,7 +368,7 @@ __global__ void WarpSelectSoftmaxTopKExpertMapsKernel( } __syncthreads(); BuildSmallDecodeExpertMaps(selected, row_to_source, source_to_row, sorted_experts, - expert_offsets, rows, kExperts, k); + expert_offsets, rows, kExperts, k); } template diff --git a/onnxruntime/core/graph/contrib_ops/contrib_defs.cc b/onnxruntime/core/graph/contrib_ops/contrib_defs.cc index cec17789ad310..6a3c0208f4b20 100644 --- a/onnxruntime/core/graph/contrib_ops/contrib_defs.cc +++ b/onnxruntime/core/graph/contrib_ops/contrib_defs.cc @@ -3281,7 +3281,7 @@ The partition widths must be positive and sum to N.)DOC") } if ((split == nullptr && ctx.getNumOutputs() != 1) || (split != nullptr && (split->ints_size() < 1 || split->ints_size() > 8 || - static_cast(split->ints_size()) != ctx.getNumOutputs()))) { + static_cast(split->ints_size()) != ctx.getNumOutputs()))) { fail_shape_inference("Output count must match split_sizes, or be one when split_sizes is absent."); } int64_t split_total = 0; From 691a4724062d51b4f4d5cc83a8c5ce6ab76b60c3 Mon Sep 17 00:00:00 2001 From: Tianlei Wu Date: Sat, 24 Oct 2026 22:47:01 +0000 Subject: [PATCH 09/12] Address PR review: restore unrelated formatting, guard contrib-op QDQ tests, validate shared_expert attrs, sync QMoE docs Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/ContribOperators.md | 2 +- .../cuda/math/matmul_block_scaled_fp8.cc | 4 ++-- .../graph/contrib_ops/quantization_defs.cc | 24 ++++++++++--------- .../test/optimizer/qdq_transformer_test.cc | 4 +++- 4 files changed, 19 insertions(+), 15 deletions(-) diff --git a/docs/ContribOperators.md b/docs/ContribOperators.md index 996ce6ad2a623..60e3e26efb090 100644 --- a/docs/ContribOperators.md +++ b/docs/ContribOperators.md @@ -6514,7 +6514,7 @@ This version of the operator has been available since version 1 of the 'com.micr
use_sparse_mixer : int
Whether to use sparse mixer
weights_prepacked : int
-
Expert-weight layout selector with quantization-specific meanings. For quant_type='int', -1 and 1 select execution-provider-specific prepacked int4/int8 fc1/fc2 layouts. 0 selects raw [E,N,K/pack] tensors as produced by quantize_matmul_{4,8}bits. For quant_type='nvfp4' on CUDA, -1 (default) and 0 select legacy N-packed [E,K,N/2] bytes; 1 selects K-packed row-major [E,N,K/2] bytes while retaining logical [E,K,N/2] dimensions. NVFP4 block scales and global scales retain their original layouts and values. Ignored for other quantization types. Supported values are -1, 0, and 1; defaults to -1.
+
Expert-weight layout selector with quantization-specific meanings. For quant_type='int', -1 and 1 select execution-provider-specific prepacked int4/int8 fc1/fc2 layouts. 0 selects raw [E,N,K/pack] tensors as produced by quantize_matmul_{4,8}bits. Uniform INT2 on CUDA also accepts explicit 1 for SM80 CUTLASS W2_A16 bytes, retaining logical [E,N,K/4] dimensions and ordinary block scales. It requires supported packed GEMV/grouped-GEMM execution; raw fallback is unavailable. Mixed-width CUDA INT execution requires 0; INT2 does not accept -1. For quant_type='nvfp4' on CUDA, -1 (default) and 0 select legacy N-packed [E,K,N/2] bytes; 1 selects K-packed row-major [E,N,K/2] bytes while retaining logical [E,K,N/2] dimensions. NVFP4 block scales and global scales retain their original layouts and values. Ignored for other quantization types. Supported values are -1, 0, and 1; defaults to -1.
#### Inputs (6 - 21) diff --git a/onnxruntime/contrib_ops/cuda/math/matmul_block_scaled_fp8.cc b/onnxruntime/contrib_ops/cuda/math/matmul_block_scaled_fp8.cc index 3bcb234d3e7c7..35341df0bdbbb 100644 --- a/onnxruntime/contrib_ops/cuda/math/matmul_block_scaled_fp8.cc +++ b/onnxruntime/contrib_ops/cuda/math/matmul_block_scaled_fp8.cc @@ -66,9 +66,9 @@ MatMulBlockQuantizedFp8Weight::MatMulBlockQuantizedFp8Weight(const OpKernelInfo& "split_sizes must contain positive widths with no sum overflow."); total += width; } - ORT_ENFORCE(!shared_expert_ || (split_sizes_.empty() && prefill_weight_offsets_.empty()), - "shared_expert does not support split_sizes or prefill_weight_offsets"); } + ORT_ENFORCE(!shared_expert_ || (split_sizes_.empty() && prefill_weight_offsets_.empty()), + "shared_expert does not support split_sizes or prefill_weight_offsets"); } template diff --git a/onnxruntime/core/graph/contrib_ops/quantization_defs.cc b/onnxruntime/core/graph/contrib_ops/quantization_defs.cc index 46bfebf9ec693..9a3facb52a4ca 100644 --- a/onnxruntime/core/graph/contrib_ops/quantization_defs.cc +++ b/onnxruntime/core/graph/contrib_ops/quantization_defs.cc @@ -210,7 +210,9 @@ ONNX_MS_OPERATOR_SET_SCHEMA(DequantizeLinear, 1, "T1", OpSchema::Optional) .Output(0, "y", "N-D full precision output tensor. It has same shape as input 'x'.", "T2") - .TypeConstraint("T1", {"tensor(int8)", "tensor(uint8)", "tensor(int16)", "tensor(uint16)", "tensor(int32)", "tensor(int4)", "tensor(uint4)", "tensor(float8e4m3fn)"}, + .TypeConstraint("T1", {"tensor(int8)", "tensor(uint8)", "tensor(int16)", + "tensor(uint16)", "tensor(int32)", "tensor(int4)", + "tensor(uint4)", "tensor(float8e4m3fn)"}, "Constrain 'x' and 'x_zero_point' to 8-bit integer tensors, " "16-bit integer tensors, 32-bit signed integer tensors, or " "FP8 E4M3FN tensors.") @@ -867,16 +869,16 @@ ONNX_MS_OPERATOR_SET_SCHEMA( {"tensor(uint8)", "tensor(int8)"}, "Constrain input and output types to 8 bit signed and unsigned tensors.") .TypeAndShapeInferenceFunction([](InferenceContext& ctx) { - propagateElemTypeFromInputToOutput(ctx, 1, 0); - if (hasNInputShapes(ctx, 9)) { - std::vector shapes; - shapes.push_back(&ctx.getInputType(0)->tensor_type().shape()); - shapes.push_back(&ctx.getInputType(1)->tensor_type().shape()); - shapes.push_back(&ctx.getInputType(4)->tensor_type().shape()); - multidirectionalBroadcastShapeInference( - shapes, *ctx.getOutputType(0)->mutable_tensor_type()->mutable_shape()); - } -})); + propagateElemTypeFromInputToOutput(ctx, 1, 0); + if (hasNInputShapes(ctx, 9)) { + std::vector shapes; + shapes.push_back(&ctx.getInputType(0)->tensor_type().shape()); + shapes.push_back(&ctx.getInputType(1)->tensor_type().shape()); + shapes.push_back(&ctx.getInputType(4)->tensor_type().shape()); + multidirectionalBroadcastShapeInference( + shapes, *ctx.getOutputType(0)->mutable_tensor_type()->mutable_shape()); + } + })); ONNX_MS_OPERATOR_SET_SCHEMA( QGemm, 1, diff --git a/onnxruntime/test/optimizer/qdq_transformer_test.cc b/onnxruntime/test/optimizer/qdq_transformer_test.cc index eaa18ce41673a..dddeda71e2ca6 100644 --- a/onnxruntime/test/optimizer/qdq_transformer_test.cc +++ b/onnxruntime/test/optimizer/qdq_transformer_test.cc @@ -4651,6 +4651,7 @@ TEST(QDQTransformerTests, QDQPropagation_DQForward_ConstantInput_NoPropagation) } } +#if !defined(DISABLE_CONTRIB_OPS) TEST(QDQTransformerTests, QDQPropagation_MSIndependentScaleOutput_NoPropagation) { for (bool explicit_output_dtype : {false, true}) { auto build_test_case = [=](ModelTestBuilder& builder) { @@ -4705,7 +4706,8 @@ TEST(QDQTransformerTests, QDQPropagation_MSFp8Bf16_NoPropagation) { std::make_unique(), TransformerLevel::Level1, 2, nullptr, check_graph)); } -#endif +#endif // !defined(DISABLE_FLOAT8_TYPES) +#endif // !defined(DISABLE_CONTRIB_OPS) TEST(QDQTransformerTests, QDQPropagation_StopAtOtherQDQ) { auto test_case = [&](const std::vector& input_shape, bool same_scale, bool same_zp, From a82104deadf824fe76761e1a059bcdc908bbfcf0 Mon Sep 17 00:00:00 2001 From: Tianlei Wu Date: Sat, 24 Oct 2026 23:23:12 +0000 Subject: [PATCH 10/12] Make FP8 shared-expert path self-contained Implement the shared-expert preparation kernel locally instead of including contrib_ops/cuda/bert/fused_tensor_ops.h, which does not exist on main, and compute the SigmoidGateMul reference values on the host in the test. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../cuda/math/matmul_block_scaled_fp8.cc | 12 ++-- .../cuda/math/matmul_block_scaled_fp8.cu | 66 +++++++++++++++++++ .../cuda/math/matmul_block_scaled_fp8.h | 3 + .../matmul_block_scaled_fp8_test.cc | 35 +++------- 4 files changed, 82 insertions(+), 34 deletions(-) diff --git a/onnxruntime/contrib_ops/cuda/math/matmul_block_scaled_fp8.cc b/onnxruntime/contrib_ops/cuda/math/matmul_block_scaled_fp8.cc index 35341df0bdbbb..f69c2eac4d560 100644 --- a/onnxruntime/contrib_ops/cuda/math/matmul_block_scaled_fp8.cc +++ b/onnxruntime/contrib_ops/cuda/math/matmul_block_scaled_fp8.cc @@ -2,7 +2,6 @@ // Licensed under the MIT License. #include "contrib_ops/cuda/math/matmul_block_scaled_fp8.h" -#include "contrib_ops/cuda/bert/fused_tensor_ops.h" #include #include @@ -199,13 +198,10 @@ Status MatMulBlockQuantizedFp8Weight::ComputeImpl(OpKernelContext* context) cons if (shared_expert_) { shared_activation = GetScratchBuffer(SafeInt(m_i) * k_i, GetComputeStream(context)); shared_gates = GetScratchBuffer(m_i, GetComputeStream(context)); - ORT_RETURN_IF_ERROR(LaunchPrepareSharedExpert( - Stream(context), reinterpret_cast(a->DataRaw()), - reinterpret_cast(silu_gate->DataRaw()), - reinterpret_cast(gate_activation->DataRaw()), - reinterpret_cast(gate_weight->DataRaw()), - reinterpret_cast(shared_activation.get()), shared_gates.get(), m_i, k_i, - SafeInt(gate_activation->Shape()[a_rank - 1]))); + ORT_RETURN_IF_ERROR(LaunchFp8PrepareSharedExpert( + shared_activation.get(), shared_gates.get(), a->DataRaw(), silu_gate->DataRaw(), + gate_activation->DataRaw(), gate_weight->DataRaw(), m_i, k_i, + SafeInt(gate_activation->Shape()[a_rank - 1]), Stream(context))); shared_epilogue = {shared_gates.get(), residual->DataRaw()}; } const auto finish_output = [&]() -> Status { diff --git a/onnxruntime/contrib_ops/cuda/math/matmul_block_scaled_fp8.cu b/onnxruntime/contrib_ops/cuda/math/matmul_block_scaled_fp8.cu index 32bfcf239fb41..70a9e88dfff6c 100644 --- a/onnxruntime/contrib_ops/cuda/math/matmul_block_scaled_fp8.cu +++ b/onnxruntime/contrib_ops/cuda/math/matmul_block_scaled_fp8.cu @@ -6,6 +6,7 @@ #include #include +#include #include "core/platform/env_var_utils.h" #include "core/providers/cuda/cu_inc/common.cuh" @@ -800,6 +801,44 @@ __global__ void Fp8SharedExpertEpilogueKernel(__nv_bfloat16* output, Fp8SharedEx } } +constexpr int kSharedExpertPrepareThreads = 256; + +// Matches CUDA Sigmoid's BF16 intermediates, including its negative-input subtraction. +__device__ __forceinline__ float SharedExpertBf16Sigmoid(float value) { + using namespace onnxruntime::cuda; + const BFloat16 a(value); + return static_cast(a > BFloat16(0.0f) + ? BFloat16(1.0f) / (BFloat16(1.0f) + _Exp(-a)) + : BFloat16(1.0f) - BFloat16(1.0f) / (BFloat16(1.0f) + _Exp(a))); +} + +// One block per row: computes the projected row gate sigmoid(dot(gate_activation, gate_weight)) and +// the SiLU-gated activation up * silu(silu_gate). +__global__ void PrepareSharedExpertKernel( + const __nv_bfloat16* up, const __nv_bfloat16* silu_gate, const __nv_bfloat16* activation, + const __nv_bfloat16* weight, __nv_bfloat16* prepared, float* row_gates, int width, int input_width) { + using Reduce = cub::BlockReduce; + __shared__ typename Reduce::TempStorage reduction; + __shared__ float gate; + const int64_t row = blockIdx.x; + float dot = 0.0f; + for (int k = threadIdx.x; k < input_width; k += kSharedExpertPrepareThreads) { + dot = fmaf(__bfloat162float(activation[row * input_width + k]), __bfloat162float(weight[k]), dot); + } + const float total = Reduce(reduction).Sum(dot); + if (threadIdx.x == 0) { + gate = SharedExpertBf16Sigmoid(__bfloat162float(__float2bfloat16_rn(total))); + row_gates[row] = gate; + } + __syncthreads(); + for (int column = threadIdx.x; column < width; column += kSharedExpertPrepareThreads) { + const int64_t index = row * width + column; + const float value = __bfloat162float(silu_gate[index]); + const auto silu = __float2bfloat16_rn(value * SharedExpertBf16Sigmoid(value)); + prepared[index] = __float2bfloat16_rn(__bfloat162float(up[index]) * __bfloat162float(silu)); + } +} + // Kill switch for A/B testing the tensor-core path against the FMA path in the same binary. bool Fp8GemvMmaEnabled() { static bool const enabled = onnxruntime::ParseEnvironmentVariableWithDefault("ORT_FP8_GEMV_MMA", true); @@ -846,6 +885,33 @@ Status LaunchFp8SharedExpertEpilogue(void* y, const Fp8SharedExpertEpilogue& epi #endif } +Status LaunchFp8PrepareSharedExpert(void* prepared, float* row_gates, const void* up, const void* silu_gate, + const void* gate_activation, const void* gate_weight, + int m, int k, int gate_width, cudaStream_t stream) { +#if !defined(DISABLE_FLOAT8_TYPES) && defined(CUDA_VERSION) && CUDA_VERSION >= 11080 + ORT_RETURN_IF_NOT(m >= 0 && k > 0 && gate_width > 0, + "Shared expert preparation requires nonnegative rows and positive widths"); + if (m == 0) return Status::OK(); + PrepareSharedExpertKernel<<>>( + static_cast(up), static_cast(silu_gate), + static_cast(gate_activation), static_cast(gate_weight), + static_cast<__nv_bfloat16*>(prepared), row_gates, k, gate_width); + return CUDA_CALL(cudaGetLastError()); +#else + ORT_UNUSED_PARAMETER(prepared); + ORT_UNUSED_PARAMETER(row_gates); + ORT_UNUSED_PARAMETER(up); + ORT_UNUSED_PARAMETER(silu_gate); + ORT_UNUSED_PARAMETER(gate_activation); + ORT_UNUSED_PARAMETER(gate_weight); + ORT_UNUSED_PARAMETER(m); + ORT_UNUSED_PARAMETER(k); + ORT_UNUSED_PARAMETER(gate_width); + ORT_UNUSED_PARAMETER(stream); + return ORT_MAKE_STATUS(ONNXRUNTIME, NOT_IMPLEMENTED, "Shared expert requires FP8 support."); +#endif +} + Status LaunchMatMulFp8SharedExpert(void* y, const void* a, const void* weight, const float* scales, int m, int n, int k, int block_size, const Fp8SharedExpertEpilogue& epilogue, diff --git a/onnxruntime/contrib_ops/cuda/math/matmul_block_scaled_fp8.h b/onnxruntime/contrib_ops/cuda/math/matmul_block_scaled_fp8.h index 155132fa04da0..f9e69a9d12684 100644 --- a/onnxruntime/contrib_ops/cuda/math/matmul_block_scaled_fp8.h +++ b/onnxruntime/contrib_ops/cuda/math/matmul_block_scaled_fp8.h @@ -55,6 +55,9 @@ Status LaunchMatMulFp8SharedExpert(void* y, const void* a, const void* weight, const float* scales, int m, int n, int k, int block_size, const Fp8SharedExpertEpilogue& epilogue, const cudaDeviceProp& device_prop, cudaStream_t stream); +Status LaunchFp8PrepareSharedExpert(void* prepared, float* row_gates, const void* up, const void* silu_gate, + const void* gate_activation, const void* gate_weight, + int m, int k, int gate_width, cudaStream_t stream); Status LaunchFp8SharedExpertEpilogue(void* y, const Fp8SharedExpertEpilogue& epilogue, int m, int n, cudaStream_t stream); diff --git a/onnxruntime/test/contrib_ops/matmul_block_scaled_fp8_test.cc b/onnxruntime/test/contrib_ops/matmul_block_scaled_fp8_test.cc index b7d94c361e274..ae3e165fc89cc 100644 --- a/onnxruntime/test/contrib_ops/matmul_block_scaled_fp8_test.cc +++ b/onnxruntime/test/contrib_ops/matmul_block_scaled_fp8_test.cc @@ -183,17 +183,6 @@ TEST(MatMulBlockQuantizedFp8WeightOpTest, SharedExpertNumericsAndFallbacks) { providers.push_back(DefaultCudaExecutionProvider()); op.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &providers); }; - OpTester silu("SigmoidGateMul", 1, kMSDomain); - silu.AddAttribute("silu_gate", 1); - silu.AddAttribute("sigmoid_in_fp32", 0); - silu.AddInput("up", input_shape, up); - silu.AddInput("gate", input_shape, gate); - silu.AddOptionalInputEdge(); - silu.AddInput("prefill", {1}, {prefill}); - silu.AddOutput("prepared", input_shape, prepared); - silu.SetOutputTolerance(0.f, 0.f); - run(silu); - std::vector projected(m * n); OpTester projection("MatMulBlockQuantizedFp8Weight", 1, kMSDomain); projection.AddAttribute("block_size", block_size); @@ -216,21 +205,15 @@ TEST(MatMulBlockQuantizedFp8WeightOpTest, SharedExpertNumericsAndFallbacks) { for (int64_t i = 0; i < m * gate_width; ++i) { gate_activation[i] = BFloat16(static_cast((i / gate_width) % 3 - 1)); } - OpTester post("SigmoidGateMul", 1, kMSDomain); - post.AddAttribute("broadcast_gate", 1); - post.AddAttribute("sigmoid_in_fp32", 0); - post.AddInput("down", output_shape, projected); - post.AddInput("activation", activation_shape, gate_activation); - post.AddInput("residual", output_shape, residual); - post.AddInput("prefill", {1}, {prefill}); - post.AddInput("weight", {gate_width, 1}, gate_weight); - post.AddOutput("Y", output_shape, expected); - post.SetCustomOutputVerifier([&](const std::vector& fetches, const std::string&) { - ASSERT_EQ(fetches.size(), 1u); - const auto& output = fetches[0].Get(); - expected.assign(output.Data(), output.Data() + m * n); - }); - run(post); + for (int64_t row = 0; row < m; ++row) { + const float logit = round(4.f * static_cast(row % 3 - 1)); + const float gate_value = logit > 0 ? round(1.f / round(1.f + round(std::exp(-logit)))) + : round(1.f - round(1.f / round(1.f + round(std::exp(logit))))); + for (int64_t col = 0; col < n; ++col) { + const float gated = round(static_cast(projected[row * n + col]) * gate_value); + expected[row * n + col] = BFloat16(round(gated + static_cast(residual[row * n + col]))); + } + } OpTester test("MatMulBlockQuantizedFp8Weight", 1, kMSDomain); test.AddAttribute("block_size", block_size); From a44fa6ace9e260d5642b48d6471e5fba76302add Mon Sep 17 00:00:00 2001 From: Tianlei Wu Date: Mon, 26 Oct 2026 07:03:03 +0000 Subject: [PATCH 11/12] Address PR review: fix QDQ/DML guards, INT2 GEMV row bound, NVFP4 K-lanes, routing NaNs, and test coverage Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- cmake/onnxruntime_unittests.cmake | 18 ++ docs/OperatorKernels.md | 2 +- docs/contrib_ops/cuda/moe_qmoe.md | 5 +- .../contrib_ops/cuda/llm/moe_gemm/moe_gemv.cu | 7 +- .../cuda/llm/moe_gemm/moe_gemv_fp4.cu | 22 +- .../cuda/math/matmul_block_scaled_fp8.cc | 4 + .../contrib_ops/cuda/moe/qmoe_kernels.cu | 17 +- .../quantize_dequantize_linear.cc | 2 +- .../onnx_transpose_optimization.cc | 16 +- .../src/Operators/DmlOperatorElementWise.cpp | 26 +++ .../src/Operators/OperatorRegistration.cpp | 5 +- .../cuda_kernels/qmoe_fp8_compaction_test.cc | 43 +++- .../cuda_kernels/softmax_topk_kernel_test.cc | 26 ++- .../contrib_ops/dequantize_fp8_bf16_test.cc | 98 +++++---- .../matmul_block_scaled_fp8_test.cc | 193 ++++++++++++++++ onnxruntime/test/contrib_ops/moe_test.cc | 208 ++++++++++++++++++ .../test/contrib_ops/quantize_ops_test.cc | 103 +++++---- .../test/optimizer/qdq_transformer_test.cc | 78 ++++--- 18 files changed, 730 insertions(+), 143 deletions(-) diff --git a/cmake/onnxruntime_unittests.cmake b/cmake/onnxruntime_unittests.cmake index e852baf50b5d7..735b464eaf41e 100644 --- a/cmake/onnxruntime_unittests.cmake +++ b/cmake/onnxruntime_unittests.cmake @@ -1723,6 +1723,24 @@ block() ENVIRONMENT "GTEST_FILTER=${fp8_decode_filter};ORT_QMOE_FP8_GEMV_NATIVE_DECODE=${native_decode}" RUN_SERIAL TRUE) endforeach() + + if (NOT onnxruntime_MINIMAL_BUILD AND NOT onnxruntime_REDUCED_OPS_BUILD) + set(moe_routing_warp_select_filter "CUDA_EP_Unittest.FusedSoftmaxTopKExpertMaps*") + if (onnxruntime_BUILD_CUDA_EP_AS_PLUGIN) + set(moe_routing_warp_select_outer_filter "${moe_routing_warp_select_filter}") + else() + set(moe_routing_warp_select_outer_filter "CUDA_EP_Unittest.All") + endif() + foreach(warp_select 0 1 2) + add_test(NAME onnxruntime_moe_routing_warp_select_${warp_select} + COMMAND $ + "--gtest_filter=${moe_routing_warp_select_outer_filter}" + WORKING_DIRECTORY $) + set_tests_properties(onnxruntime_moe_routing_warp_select_${warp_select} PROPERTIES + ENVIRONMENT "GTEST_FILTER=${moe_routing_warp_select_filter};ORT_MOE_ROUTING_WARP_SELECT=${warp_select}" + RUN_SERIAL TRUE) + endforeach() + endif() endif() if (onnxruntime_USE_CUDA AND onnxruntime_ENABLE_CUDA_EP_INTERNAL_TESTS AND diff --git a/docs/OperatorKernels.md b/docs/OperatorKernels.md index 447b62b6c291d..20069b3b1dfdc 100644 --- a/docs/OperatorKernels.md +++ b/docs/OperatorKernels.md @@ -1631,7 +1631,7 @@ The **OpSet Version** column uses the following notation: |BiasGelu|*in* A:**T**
*in* B:**T**
*out* C:**T**|1+|**T** = tensor(float), tensor(float16)| |BiasSplitGelu|*in* X:**T**
*in* bias:**T**
*out* Y:**T**|1+|**T** = tensor(float), tensor(float16)| |ConvTransposeWithDynamicPads|*in* X:**T**
*in* W:**T**
*in* Pads:**tensor(int64)**
*in* B:**T**
*out* Y:**T**|1+|**T** = tensor(float), tensor(float16)| -|DequantizeLinear|*in* x:**T1**
*in* x_scale:**TS**
*in* x_zero_point:**T1**
*out* y:**T2**|1+|**T1** = tensor(int32), tensor(int8), tensor(uint8)
**T2** = tensor(float), tensor(float16)| +|DequantizeLinear|*in* x:**T1**
*in* x_scale:**TS**
*in* x_zero_point:**T1**
*out* y:**T2**|1+|**T1** = tensor(int32), tensor(int8), tensor(uint8)
**T2** = tensor(float), tensor(float16)
**TS** = tensor(float), tensor(float16)| |DynamicQuantizeMatMul|*in* A:**T1**
*in* B:**T2**
*in* b_scale:**T1**
*in* b_zero_point:**T2**
*in* bias:**T1**
*out* Y:**T1**|1+|**T1** = tensor(float)
**T2** = tensor(int8), tensor(uint8)| |EmbedLayerNormalization|*in* input_ids:**T1**
*in* segment_ids:**T1**
*in* word_embedding:**T**
*in* position_embedding:**T**
*in* segment_embedding:**T**
*in* gamma:**T**
*in* beta:**T**
*in* mask:**T1**
*in* position_ids:**T1**
*out* output:**T**
*out* mask_index:**T1**
*out* embedding_sum:**T**|1+|**T** = tensor(float), tensor(float16)| |FastGelu|*in* X:**T**
*in* bias:**T**
*out* Y:**T**|1+|**T** = tensor(float), tensor(float16)| diff --git a/docs/contrib_ops/cuda/moe_qmoe.md b/docs/contrib_ops/cuda/moe_qmoe.md index f0c0ae60085e9..6930e74ef2101 100644 --- a/docs/contrib_ops/cuda/moe_qmoe.md +++ b/docs/contrib_ops/cuda/moe_qmoe.md @@ -861,9 +861,8 @@ prefill and decode kernels with no extra conversion. ### Large-expert INT2 execution and single-copy packed weights Compatible INT2 execution supports up to 512 experts and top-10 routing by -default. `ORT_QMOE_INT2_LARGE_EXPERTS=0` restores the previous 256-expert -eligibility. Packed INT2 GEMV accepts up to 20 expanded rows, covering one-token -decode and two-token verification with top-10 routing. Existing GPU, alignment, +default. Packed INT2 GEMV accepts up to 80 expanded rows, covering one-token +decode and multi-token verification with top-10 routing. Existing GPU, alignment, block-size, symmetric-weight and fused-SwiGLU checks remain in force. Larger calls use supported packed grouped GEMM; raw inputs retain the bounded dense fallback for unsupported configurations. diff --git a/onnxruntime/contrib_ops/cuda/llm/moe_gemm/moe_gemv.cu b/onnxruntime/contrib_ops/cuda/llm/moe_gemm/moe_gemv.cu index 3c9208762c4ba..f061b6646f1e5 100644 --- a/onnxruntime/contrib_ops/cuda/llm/moe_gemm/moe_gemv.cu +++ b/onnxruntime/contrib_ops/cuda/llm/moe_gemm/moe_gemv.cu @@ -1112,12 +1112,7 @@ bool is_moe_gemv_shape_supported(int sm, int64_t expanded_num_rows, int64_t n, i if (group_size > 0 && k % group_size != 0) { return false; } - const int64_t max_expanded_rows = - weight_bits == 2 && - onnxruntime::ParseEnvironmentVariableWithDefault("ORT_QMOE_INT2_LARGE_EXPERTS", 1) == 1 - ? 20 - : kMaxProfiledExpandedRows; - if (expanded_num_rows <= 0 || expanded_num_rows > max_expanded_rows) { + if (expanded_num_rows <= 0 || expanded_num_rows > kMaxProfiledExpandedRows) { return false; } if (n < kMinProfiledProblemDim || k < kMinProfiledProblemDim) { diff --git a/onnxruntime/contrib_ops/cuda/llm/moe_gemm/moe_gemv_fp4.cu b/onnxruntime/contrib_ops/cuda/llm/moe_gemm/moe_gemv_fp4.cu index 3dfaa3496b423..5574066f89038 100644 --- a/onnxruntime/contrib_ops/cuda/llm/moe_gemm/moe_gemv_fp4.cu +++ b/onnxruntime/contrib_ops/cuda/llm/moe_gemm/moe_gemv_fp4.cu @@ -953,20 +953,28 @@ bool try_launch_moe_gemv_nvfp4_fused_finalize( return false; } const dim3 grid(static_cast(num_rows), static_cast((n + 7) / 8)); - const dim3 block(64, static_cast(experts_per_token)); const int64_t weight_stride = static_cast(n) * k / 2; const int64_t scale_stride = static_cast(n) * (k / 16); const cutlass_kernels::ActivationParams activation; - if (bias != nullptr) { - MoeGemvFp4RawKPackedKernel<<>>( + const auto launch = [&]() { + const dim3 block(8 * KLanes, static_cast(experts_per_token)); + MoeGemvFp4RawKPackedKernel<<>>( act, weight, block_scales, global_scales, bias, out, nullptr, permuted_row_to_expert, num_experts, weight_stride, scale_stride, n, k, activation, nullptr, num_rows, unpermuted_row_to_permuted_row, routing_scales, experts_per_token); + }; + if (k <= 1024) { + if (bias != nullptr) { + launch.template operator()<4, true>(); + } else { + launch.template operator()<4, false>(); + } } else { - MoeGemvFp4RawKPackedKernel<<>>( - act, weight, block_scales, global_scales, bias, out, nullptr, permuted_row_to_expert, - num_experts, weight_stride, scale_stride, n, k, activation, nullptr, num_rows, - unpermuted_row_to_permuted_row, routing_scales, experts_per_token); + if (bias != nullptr) { + launch.template operator()<8, true>(); + } else { + launch.template operator()<8, false>(); + } } CUDA_CALL_THROW(cudaGetLastError()); return true; diff --git a/onnxruntime/contrib_ops/cuda/math/matmul_block_scaled_fp8.cc b/onnxruntime/contrib_ops/cuda/math/matmul_block_scaled_fp8.cc index f69c2eac4d560..97620b3dee306 100644 --- a/onnxruntime/contrib_ops/cuda/math/matmul_block_scaled_fp8.cc +++ b/onnxruntime/contrib_ops/cuda/math/matmul_block_scaled_fp8.cc @@ -401,6 +401,10 @@ Status MatMulBlockQuantizedFp8Weight::ComputeImpl(OpKernelContext* context) cons static_cast(width) * sizeof(CudaT), static_cast(m_i), cudaMemcpyDeviceToDevice, Stream(context))); } + for (size_t segment = prefill_weight_offsets_.size(); segment < 3; ++segment) { + ORT_RETURN_IF_NOT(context->Input(6 + static_cast(segment)) == nullptr, + "Original prefill weight inputs require matching prefill_weight_offsets"); + } } if (bias != nullptr) { diff --git a/onnxruntime/contrib_ops/cuda/moe/qmoe_kernels.cu b/onnxruntime/contrib_ops/cuda/moe/qmoe_kernels.cu index 72071cba8ab5a..94b664a0079af 100644 --- a/onnxruntime/contrib_ops/cuda/moe/qmoe_kernels.cu +++ b/onnxruntime/contrib_ops/cuda/moe/qmoe_kernels.cu @@ -267,18 +267,23 @@ __global__ void FusedSoftmaxTopKExpertMapsKernel( const int local = threadIdx.x % 128; __shared__ DecodeRoutingMergeSort::TempStorage temp[2]; __shared__ float maxima[2]; + __shared__ bool row_has_nan[2]; __shared__ float probabilities[2][16]; __shared__ int selected[32]; + if (local == 0) row_has_nan[row] = false; uint64_t keys[4]; + bool has_nan = false; #pragma unroll for (int j = 0; j < 4; ++j) { const int expert = local * 4 + j; const bool valid = row < rows && expert < experts; const float value = valid ? static_cast(logits[row * experts + expert]) : onnxruntime::cuda::topk::kNegativeInfinity; + has_nan |= valid && isnan(value); keys[j] = onnxruntime::cuda::topk::PackStableSortKey(value, valid ? expert : INT_MAX); } DecodeRoutingMergeSort(temp[row], local).Sort(keys, onnxruntime::cuda::topk::Greater()); + if (has_nan) row_has_nan[row] = true; if (local == 0) maxima[row] = onnxruntime::cuda::topk::UnpackStableSortScore(keys[0]); __syncthreads(); #pragma unroll @@ -288,7 +293,8 @@ __global__ void FusedSoftmaxTopKExpertMapsKernel( const int expert = onnxruntime::cuda::topk::UnpackStableSortIndex(keys[j]); indices[row * k + rank] = expert; selected[row * k + rank] = expert; - const float value = expf(onnxruntime::cuda::topk::UnpackStableSortScore(keys[j]) - maxima[row]); + const float value = + row_has_nan[row] ? 0.0f : expf(onnxruntime::cuda::topk::UnpackStableSortScore(keys[j]) - maxima[row]); probabilities[row][rank] = isfinite(value) ? value : 0.0f; } } @@ -316,12 +322,15 @@ __global__ void WarpSelectSoftmaxTopKExpertMapsKernel( __shared__ int selected[2 * k]; if (row < rows) { uint64_t keys[16]; + bool has_nan = false; #pragma unroll for (int i = 0; i < 16; ++i) { const int expert = lane + i * 32; - keys[i] = onnxruntime::cuda::topk::PackStableSortKey( - static_cast(logits[row * kExperts + expert]), expert); + const float value = static_cast(logits[row * kExperts + expert]); + has_nan |= isnan(value); + keys[i] = onnxruntime::cuda::topk::PackStableSortKey(value, expert); } + const bool row_has_nan = __any_sync(0xffffffff, has_nan); float probabilities[k]; float maximum = 0.0f; for (int rank = 0; rank < k; ++rank) { @@ -352,7 +361,7 @@ __global__ void WarpSelectSoftmaxTopKExpertMapsKernel( if (lane == 0) { const float score = onnxruntime::cuda::topk::UnpackStableSortScore(best); if (rank == 0) maximum = score; - const float value = expf(score - maximum); + const float value = row_has_nan ? 0.0f : expf(score - maximum); probabilities[rank] = isfinite(value) ? value : 0.0f; const int expert = onnxruntime::cuda::topk::UnpackStableSortIndex(best); selected[row * k + rank] = expert; diff --git a/onnxruntime/contrib_ops/cuda/quantization/quantize_dequantize_linear.cc b/onnxruntime/contrib_ops/cuda/quantization/quantize_dequantize_linear.cc index 1d9e9794de2d4..1e7f0f93aa73d 100644 --- a/onnxruntime/contrib_ops/cuda/quantization/quantize_dequantize_linear.cc +++ b/onnxruntime/contrib_ops/cuda/quantization/quantize_dequantize_linear.cc @@ -81,7 +81,7 @@ class DequantizeLinearFp8Bf16 final : public CudaKernel { zero_point == nullptr ? nullptr : zero_point->Data(), output->MutableData(), static_cast(count), static_cast(axis_stride), static_cast(scale_count)); - return Status::OK(); + return CUDA_CALL(cudaGetLastError()); } private: diff --git a/onnxruntime/core/optimizer/transpose_optimization/onnx_transpose_optimization.cc b/onnxruntime/core/optimizer/transpose_optimization/onnx_transpose_optimization.cc index 946f9aa0e993a..501e4c656d109 100755 --- a/onnxruntime/core/optimizer/transpose_optimization/onnx_transpose_optimization.cc +++ b/onnxruntime/core/optimizer/transpose_optimization/onnx_transpose_optimization.cc @@ -529,10 +529,24 @@ static bool MakeQDQNodeUnit(api::GraphRef& graph, const api::NodeRef& dq_node) { const auto scale_input = dq_inputs[1]; std::optional zp_input; - if (dq_inputs.size() > 2) { + if (dq_inputs.size() > 2 && !dq_inputs[2].empty()) { zp_input = dq_inputs[2]; } + if (dq_domain == onnxruntime::kMSDomain) { + const api::DataType input_dtype = graph.GetValueInfo(dq_inputs[0])->DType(); + const api::DataType scale_dtype = graph.GetValueInfo(scale_input)->DType(); + const api::DataType output_dtype = graph.GetValueInfo(dq_node.Outputs()[0])->DType(); + // The MS quantizer cannot invert FP8, BF16, or independently typed scale/output dequantization. + if (dq_node.GetAttributeInt("output_dtype").has_value() || + input_dtype == api::DataType::UNDEFINED || scale_dtype == api::DataType::UNDEFINED || + output_dtype == api::DataType::UNDEFINED || scale_dtype != output_dtype || + (scale_dtype != api::DataType::FLOAT && scale_dtype != api::DataType::FLOAT16) || + input_dtype == api::DataType::FLOAT8E4M3FN) { + return false; + } + } + std::optional dq_quant_info = GetQuantizationInfo(graph, dq_node); if (!dq_quant_info || !IsSupportedQuantizationMode(dq_quant_info->mode)) { return false; // Can't get the quantization mode/axis or is a quantization mode that is not supported. diff --git a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/DmlOperatorElementWise.cpp b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/DmlOperatorElementWise.cpp index 94129987905e5..e93cd722b34e3 100644 --- a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/DmlOperatorElementWise.cpp +++ b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/DmlOperatorElementWise.cpp @@ -996,6 +996,32 @@ DML_OP_DEFINE_CREATION_FUNCTION(Min, DmlOperatorElementwiseBinaryLo DML_OP_DEFINE_CREATION_FUNCTION(Max, DmlOperatorElementwiseBinaryLoop); DML_OP_DEFINE_CREATION_FUNCTION(Mean, DmlOperatorElementwiseMean); +void CALLBACK QueryDequantizeLinear(IMLOperatorSupportQueryContextPrivate* context, /*out*/ bool* isSupported) +{ + *isSupported = false; + + if (!context->IsInputValid(1) || !context->IsOutputValid(0)) + { + return; + } + + MLOperatorEdgeDescription scaleEdgeDesc = {}; + if (FAILED(context->GetInputEdgeDescription(1, &scaleEdgeDesc)) || + scaleEdgeDesc.edgeType != MLOperatorEdgeType::Tensor) + { + return; + } + + MLOperatorEdgeDescription outputEdgeDesc = {}; + if (FAILED(context->GetOutputEdgeDescription(0, &outputEdgeDesc)) || + outputEdgeDesc.edgeType != MLOperatorEdgeType::Tensor) + { + return; + } + + *isSupported = (scaleEdgeDesc.tensorDataType == outputEdgeDesc.tensorDataType); +} + // Operators with extra attributes: DML_OP_DEFINE_CREATION_FUNCTION(Clip7, DmlOperatorElementwiseClip7); DML_OP_DEFINE_CREATION_FUNCTION(Clip11, DmlOperatorElementwiseClip11); diff --git a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/OperatorRegistration.cpp b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/OperatorRegistration.cpp index c9d1cfad2a646..7f42ea4382ca1 100644 --- a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/OperatorRegistration.cpp +++ b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/OperatorRegistration.cpp @@ -555,12 +555,14 @@ DML_OP_EXTERN_QUERY_FUNCTION(QLinearSigmoid); DML_OP_EXTERN_QUERY_FUNCTION(QAttention); DML_OP_EXTERN_QUERY_FUNCTION(Attention); DML_OP_EXTERN_QUERY_FUNCTION(MatMulNBits); +DML_OP_EXTERN_QUERY_FUNCTION(DequantizeLinear); constexpr static std::array typeNameListDefault = {"T"}; constexpr static std::array typeNameListDefaultV = {"V"}; constexpr static std::array typeNameListAttention = {"T", "M"}; constexpr static std::array typeNameListRotaryEmbedding = {"T", "M"}; constexpr static std::array typeNameListTwo = { "T1", "T2" }; +constexpr static std::array typeNameListDequantizeLinearMs = { "T1", "T2", "TS" }; constexpr static std::array typeNameListLayerNorm = { "T", "U" }; constexpr static std::array typeNameListLayerNormContrib = { "T", "U", "V" }; constexpr static std::array typeNameListThree = { "T1", "T2", "T3" }; @@ -611,6 +613,7 @@ constexpr static std::array supportedTypeListQuanti constexpr static std::array supportedTypeListQuantizeLinear21 = { SupportedTensorDataTypes::Float16to32, SupportedTensorDataTypes::UInt4 | SupportedTensorDataTypes::Int4 | SupportedTensorDataTypes::UInt8 | SupportedTensorDataTypes::Int8 }; constexpr static std::array supportedTypeListDequantizeLinear = { SupportedTensorDataTypes::UInt8 | SupportedTensorDataTypes::Int8 | SupportedTensorDataTypes::Int32 }; constexpr static std::array supportedTypeListDequantizeLinear19 = { SupportedTensorDataTypes::UInt8 | SupportedTensorDataTypes::Int8 | SupportedTensorDataTypes::Int32, SupportedTensorDataTypes::Float16to32 }; +constexpr static std::array supportedTypeListDequantizeLinearMs = { SupportedTensorDataTypes::UInt8 | SupportedTensorDataTypes::Int8 | SupportedTensorDataTypes::Int32, SupportedTensorDataTypes::Float16to32, SupportedTensorDataTypes::Float16to32 }; constexpr static std::array supportedTypeListDequantizeLinear21 = { SupportedTensorDataTypes::UInt4 | SupportedTensorDataTypes::Int4 | SupportedTensorDataTypes::UInt8 | SupportedTensorDataTypes::Int8, SupportedTensorDataTypes::Float16to32 }; constexpr static std::array supportedTypeListIsNan = { SupportedTensorDataTypes::Float16to32, SupportedTensorDataTypes::Bool }; constexpr static std::array supportedTypeListIsInf = { SupportedTensorDataTypes::Float32, SupportedTensorDataTypes::Bool }; @@ -942,7 +945,7 @@ constexpr static OperatorRegistrationInformation operatorRegistrationInformation {REG_INFO( 19, DequantizeLinear, typeNameListTwo, supportedTypeListDequantizeLinear19, DmlGraphSupport::Supported)}, {REG_INFO_VER( 21, DequantizeLinear, typeNameListTwo, supportedTypeListDequantizeLinear21, DmlGraphSupport::Supported)}, {REG_INFO_MS( 1, QuantizeLinear, typeNameListTwo, supportedTypeListQuantizeLinear19, DmlGraphSupport::Supported)}, - {REG_INFO_MS( 1, DequantizeLinear, typeNameListTwo, supportedTypeListDequantizeLinear19, DmlGraphSupport::Supported)}, + {REG_INFO_MS( 1, DequantizeLinear, typeNameListDequantizeLinearMs, supportedTypeListDequantizeLinearMs, DmlGraphSupport::Supported, requiredConstantCpuInputs(), std::nullopt, QueryDequantizeLinear)}, {REG_INFO( 9, Sign, typeNameListDefault, supportedTypeListFloat16to32Ints8to64, DmlGraphSupport::Supported)}, {REG_INFO( 13, Sign, typeNameListDefault, supportedTypeListFloat16to32Ints8to64, DmlGraphSupport::Supported)}, {REG_INFO( 9, IsNaN, typeNameListTwo, supportedTypeListIsNan, DmlGraphSupport::Supported)}, diff --git a/onnxruntime/test/contrib_ops/cuda_kernels/qmoe_fp8_compaction_test.cc b/onnxruntime/test/contrib_ops/cuda_kernels/qmoe_fp8_compaction_test.cc index 86b0bb6f83ab8..d1a8c1ee343a2 100644 --- a/onnxruntime/test/contrib_ops/cuda_kernels/qmoe_fp8_compaction_test.cc +++ b/onnxruntime/test/contrib_ops/cuda_kernels/qmoe_fp8_compaction_test.cc @@ -300,7 +300,7 @@ void TestNvfp4FusedFinalize(int num_rows, int topk, int k, bool use_bias) { TEST(CUDA_EP_Unittest, QMoENvfp4FusedFinalizePreservesRoutingAndRounding) { for (int rows : {1, 2}) { for (int topk : {1, 4, 10, 16}) { - for (int k : {144, 640}) { + for (int k : {144, 640, 1152}) { for (bool use_bias : {false, true}) { TestNvfp4FusedFinalize(rows, topk, k, use_bias); TestNvfp4FusedFinalize<__nv_bfloat16>(rows, topk, k, use_bias); @@ -308,6 +308,47 @@ TEST(CUDA_EP_Unittest, QMoENvfp4FusedFinalizePreservesRoutingAndRounding) { } } } + + if (!SupportsSm80OrLater()) { + GTEST_SKIP() << "NVFP4 fused finalization requires SM80 or newer"; + } + // Cancellation-sensitive parity check for K <= 1024 (4 K-lanes): + // lane 0 accumulates offsets 0, 64, and 128 in order: (100139008 + 1) + (-100139008) == 0. + constexpr int k = 144, n = 2, experts = 1, num_rows = 1, topk = 1; + std::vector<__nv_bfloat16> activations(k, __nv_bfloat16(0.0f)); + activations[0] = __nv_bfloat16(100139008.0f); + activations[64] = __nv_bfloat16(1.0f); + activations[128] = __nv_bfloat16(-100139008.0f); + // E2M1 code 2 is +1.0, so 0x22 packs two +1.0 weights; E4M3 code 0x38 is 1.0. + const std::vector weights(n * k / 2, 0x22), scales(n * k / 16, 0x38); + const std::vector globals{1.0f}, routing{1.0f}; + const std::vector mapping{0}, expert_ids{0}; + CudaBuffer a(activations.size() * sizeof(__nv_bfloat16)), w(weights.size()), s(scales.size()); + CudaBuffer g(sizeof(float)), m(sizeof(int)), e(sizeof(int)), r(sizeof(float)); + CudaBuffer reference(n * sizeof(__nv_bfloat16)), output(n * sizeof(__nv_bfloat16)); + a.Upload(activations.data()); + w.Upload(weights.data()); + s.Upload(scales.data()); + g.Upload(globals.data()); + m.Upload(mapping.data()); + e.Upload(expert_ids.data()); + r.Upload(routing.data()); + gemv::launch_moe_gemv_fp4_symmetric<__nv_bfloat16>( + a.As<__nv_bfloat16>(), w.As(), nullptr, s.As(), g.As(), + nullptr, reference.As<__nv_bfloat16>(), nullptr, e.As(), + experts, 1, n, k, 16, 90, gemv::MoeGemvConfig::kDefault, false, true, nullptr, true); + ASSERT_TRUE(gemv::try_launch_moe_gemv_nvfp4_fused_finalize<__nv_bfloat16>( + a.As<__nv_bfloat16>(), w.As(), s.As(), g.As(), + nullptr, output.As<__nv_bfloat16>(), m.As(), e.As(), r.As(), + experts, num_rows, topk, n, k, nullptr)); + CUDA_CALL_THROW(cudaGetLastError()); + std::vector<__nv_bfloat16> ref_host(n), out_host(n); + reference.Download(ref_host.data()); + output.Download(out_host.data()); + for (int col = 0; col < n; ++col) { + EXPECT_EQ(static_cast(ref_host[col]), 0.0f); + EXPECT_EQ(static_cast(out_host[col]), static_cast(ref_host[col])); + } } #endif diff --git a/onnxruntime/test/contrib_ops/cuda_kernels/softmax_topk_kernel_test.cc b/onnxruntime/test/contrib_ops/cuda_kernels/softmax_topk_kernel_test.cc index 9d633a7066cce..6b02a5a11cb8b 100644 --- a/onnxruntime/test/contrib_ops/cuda_kernels/softmax_topk_kernel_test.cc +++ b/onnxruntime/test/contrib_ops/cuda_kernels/softmax_topk_kernel_test.cc @@ -57,6 +57,13 @@ float ToFloat(T value) { template T FromFloat(float value) { + if (std::isnan(value) && std::signbit(value)) { + if constexpr (std::is_same_v) { + return __ushort_as_half(0xfe00u); + } else if constexpr (std::is_same_v) { + return __ushort_as_bfloat16(0xffc0u); + } + } return static_cast(value); } @@ -354,6 +361,15 @@ TEST(CUDA_EP_Unittest, FusedSoftmaxTopKExpertMaps_NormalizedAndGraphReplay) { } RunFusedRoutingMapsTest<__nv_bfloat16>(rows, 512, 16, true); RunFusedRoutingMapsTest(rows, 80, 16, false, true); + for (int experts : {65, 512}) { + std::vector mixed_neg_nan(static_cast(rows) * experts, 0.0f); + for (int row = 0; row < rows; ++row) { + mixed_neg_nan[row * experts + experts - 1 - row] = -std::numeric_limits::quiet_NaN(); + } + RunFusedRoutingMapsTest(rows, experts, 10, false, false, &mixed_neg_nan); + RunFusedRoutingMapsTest(rows, experts, 10, false, false, &mixed_neg_nan); + RunFusedRoutingMapsTest<__nv_bfloat16>(rows, experts, 10, false, false, &mixed_neg_nan); + } } } @@ -385,7 +401,7 @@ TEST(CUDA_EP_Unittest, FusedSoftmaxTopKExpertMaps_WarpSelectSpecialized) { TEST(CUDA_EP_Unittest, FusedSoftmaxTopKExpertMaps_WarpSelectAdversarialKeys) { for (int rows : {1, 2}) { - for (int pattern = 0; pattern < 4; ++pattern) { + for (int pattern = 0; pattern < 6; ++pattern) { SCOPED_TRACE(::testing::Message() << "rows=" << rows << " pattern=" << pattern); auto values = MakeLogits(rows, 512); for (int row = 0; row < rows; ++row) { @@ -398,8 +414,14 @@ TEST(CUDA_EP_Unittest, FusedSoftmaxTopKExpertMaps_WarpSelectAdversarialKeys) { } else if (pattern == 2) { value = expert % 3 ? -std::numeric_limits::infinity() : std::numeric_limits::quiet_NaN(); - } else { + } else if (pattern == 3) { value = (expert + row) % 7 == 0 ? 1000.0f : -1000.0f; + } else if (pattern == 4) { + value = expert == 511 - row ? -std::numeric_limits::quiet_NaN() : 0.0f; + } else { + if (expert == 257 + row) { + value = -std::numeric_limits::quiet_NaN(); + } } } } diff --git a/onnxruntime/test/contrib_ops/dequantize_fp8_bf16_test.cc b/onnxruntime/test/contrib_ops/dequantize_fp8_bf16_test.cc index 2face506a20a3..88ea81515f3c1 100644 --- a/onnxruntime/test/contrib_ops/dequantize_fp8_bf16_test.cc +++ b/onnxruntime/test/contrib_ops/dequantize_fp8_bf16_test.cc @@ -96,48 +96,62 @@ TEST(DequantizeLinearTest, Fp8ToBf16EmptyInput) { } TEST(DequantizeLinearTest, Fp8ToBf16OptimizedReshapeTranspose) { - auto provider = DefaultCudaExecutionProvider(); - if (provider == nullptr) GTEST_SKIP() << "CUDA execution provider is not available"; - std::unordered_map versions{{kOnnxDomain, 19}, {kMSDomain, 1}}; - Model model("Fp8Bf16Optimized", false, ModelMetaData(), PathString(), - IOnnxRuntimeOpSchemaRegistryList(), versions, {}, DefaultLoggingManager().DefaultLogger()); - auto& graph = model.MainGraph(); - ModelTestBuilder builder(graph); - std::vector values; - for (float value : {1.0f, 2.0f, -1.0f, -2.0f}) values.emplace_back(value); - auto* input = builder.MakeInput({4}, values); - auto* scale = builder.MakeScalarInitializer(2.0f); - auto* dequantized = builder.MakeIntermediate(); - builder.AddNode("DequantizeLinear", {input, scale}, {dequantized}, kMSDomain) - .AddAttribute("output_dtype", static_cast(ONNX_NAMESPACE::TensorProto::BFLOAT16)); - auto* reshaped = builder.MakeIntermediate(); - builder.AddNode("Reshape", {dequantized, builder.Make1DInitializer({2, 2})}, {reshaped}); - builder.AddNode("Transpose", {reshaped}, {builder.MakeOutput()}) - .AddAttribute("perm", std::vector{1, 0}); - builder.SetGraphOutputs(); - ASSERT_STATUS_OK(graph.Resolve()); - std::string serialized; - ASSERT_TRUE(model.ToProto().SerializeToString(&serialized)); - SessionOptions options; - options.graph_optimization_level = TransformerLevel::Level3; - InferenceSessionWrapper session(options, GetEnvironment()); - ASSERT_STATUS_OK(session.RegisterExecutionProvider(std::move(provider))); - std::istringstream stream(serialized); - ASSERT_STATUS_OK(session.Load(stream)); - ASSERT_STATUS_OK(session.Initialize()); - for (const auto& node : session.GetGraph().Nodes()) { - EXPECT_NE(node.OpType(), "QuantizeLinear"); - } - std::vector fetches; - ASSERT_STATUS_OK(session.Run(RunOptions{}, builder.feeds_, builder.output_names_, &fetches)); - ASSERT_EQ(fetches.size(), 1u); - const auto& output = fetches[0].Get(); - ASSERT_EQ(output.Shape(), TensorShape({2, 2})); - const auto actual = output.DataAsSpan(); - const auto expected = ToBFloat16({2, -2, 4, -4}); - ASSERT_EQ(actual.size(), expected.size()); - for (size_t i = 0; i < actual.size(); ++i) { - EXPECT_EQ(actual[i], expected[i]); + for (bool with_reshape : {true, false}) { + for (bool with_zero_point : {false, true}) { + SCOPED_TRACE(MakeString("with_reshape=", with_reshape, " with_zero_point=", with_zero_point)); + auto provider = DefaultCudaExecutionProvider(); + if (provider == nullptr) GTEST_SKIP() << "CUDA execution provider is not available"; + std::unordered_map versions{{kOnnxDomain, 19}, {kMSDomain, 1}}; + Model model("Fp8Bf16Optimized", false, ModelMetaData(), PathString(), + IOnnxRuntimeOpSchemaRegistryList(), versions, {}, DefaultLoggingManager().DefaultLogger()); + auto& graph = model.MainGraph(); + ModelTestBuilder builder(graph); + std::vector values; + for (float value : {1.0f, 2.0f, -1.0f, -2.0f}) values.emplace_back(value); + const std::vector input_shape = with_reshape ? std::vector{4} : std::vector{2, 2}; + auto* input = builder.MakeInput(input_shape, values); + auto* scale = builder.MakeScalarInitializer(2.0f); + std::vector dq_inputs{input, scale}; + if (with_zero_point) { + dq_inputs.push_back(builder.MakeScalarInitializer(Float8E4M3FN(1.0f))); + } + auto* dequantized = builder.MakeIntermediate(); + builder.AddNode("DequantizeLinear", dq_inputs, {dequantized}, kMSDomain) + .AddAttribute("output_dtype", static_cast(ONNX_NAMESPACE::TensorProto::BFLOAT16)); + NodeArg* transpose_input = dequantized; + if (with_reshape) { + auto* reshaped = builder.MakeIntermediate(); + builder.AddNode("Reshape", {dequantized, builder.Make1DInitializer({2, 2})}, {reshaped}); + transpose_input = reshaped; + } + builder.AddNode("Transpose", {transpose_input}, {builder.MakeOutput()}) + .AddAttribute("perm", std::vector{1, 0}); + builder.SetGraphOutputs(); + ASSERT_STATUS_OK(graph.Resolve()); + std::string serialized; + ASSERT_TRUE(model.ToProto().SerializeToString(&serialized)); + SessionOptions options; + options.graph_optimization_level = TransformerLevel::Level3; + InferenceSessionWrapper session(options, GetEnvironment()); + ASSERT_STATUS_OK(session.RegisterExecutionProvider(std::move(provider))); + std::istringstream stream(serialized); + ASSERT_STATUS_OK(session.Load(stream)); + ASSERT_STATUS_OK(session.Initialize()); + for (const auto& node : session.GetGraph().Nodes()) { + EXPECT_NE(node.OpType(), "QuantizeLinear"); + } + std::vector fetches; + ASSERT_STATUS_OK(session.Run(RunOptions{}, builder.feeds_, builder.output_names_, &fetches)); + ASSERT_EQ(fetches.size(), 1u); + const auto& output = fetches[0].Get(); + ASSERT_EQ(output.Shape(), TensorShape({2, 2})); + const auto actual = output.DataAsSpan(); + const auto expected = with_zero_point ? ToBFloat16({0, -4, 2, -6}) : ToBFloat16({2, -2, 4, -4}); + ASSERT_EQ(actual.size(), expected.size()); + for (size_t i = 0; i < actual.size(); ++i) { + EXPECT_EQ(actual[i], expected[i]); + } + } } } diff --git a/onnxruntime/test/contrib_ops/matmul_block_scaled_fp8_test.cc b/onnxruntime/test/contrib_ops/matmul_block_scaled_fp8_test.cc index ae3e165fc89cc..231e7aaed4636 100644 --- a/onnxruntime/test/contrib_ops/matmul_block_scaled_fp8_test.cc +++ b/onnxruntime/test/contrib_ops/matmul_block_scaled_fp8_test.cc @@ -335,11 +335,204 @@ void TestPerColumnS2ExactOutputs() { } TEST(MatMulBlockQuantizedFp8WeightOpTest, PerColumnS2ExactOutputsFp16) { + constexpr const char* kChildProcessVariable = "ORT_FP8_GEMV_S2_FP16_TEST_CHILD"; + const bool is_child_process = !Env::Default().GetEnvironmentVar(kChildProcessVariable).empty(); + if (!HasCudaEnvironment(800)) { + GTEST_SKIP() << "CUDA device is required for per-column FP8 matmul."; + } TestPerColumnS2ExactOutputs(); + if (!is_child_process) { + ScopedEnvironmentVariables scoped_env_vars{EnvVarMap{ + {"ORT_FP8_GEMV_MMA", "1"}, + {"ORT_FP8_GEMV_MAX_M", "32"}, + {"ORT_FP8_GEMV_PER_COLUMN_S2", "1"}, + {kChildProcessVariable, "1"}, + }}; + const std::string command = + "\"" + CurrentExecutablePath() + + "\" --gtest_filter=MatMulBlockQuantizedFp8WeightOpTest.PerColumnS2ExactOutputsFp16 --gtest_color=no"; + ASSERT_EQ(std::system(command.c_str()), 0); + } } TEST(MatMulBlockQuantizedFp8WeightOpTest, PerColumnS2ExactOutputsBf16) { + constexpr const char* kChildProcessVariable = "ORT_FP8_GEMV_S2_BF16_TEST_CHILD"; + const bool is_child_process = !Env::Default().GetEnvironmentVar(kChildProcessVariable).empty(); + if (!HasCudaEnvironment(800)) { + GTEST_SKIP() << "CUDA device is required for per-column FP8 matmul."; + } TestPerColumnS2ExactOutputs(); + if (!is_child_process) { + ScopedEnvironmentVariables scoped_env_vars{EnvVarMap{ + {"ORT_FP8_GEMV_MMA", "1"}, + {"ORT_FP8_GEMV_MAX_M", "32"}, + {"ORT_FP8_GEMV_PER_COLUMN_S2", "1"}, + {kChildProcessVariable, "1"}, + }}; + const std::string command = + "\"" + CurrentExecutablePath() + + "\" --gtest_filter=MatMulBlockQuantizedFp8WeightOpTest.PerColumnS2ExactOutputsBf16 --gtest_color=no"; + ASSERT_EQ(std::system(command.c_str()), 0); + } +} + +template +void TestPrefillWeightOverrides() { + if (!HasCudaEnvironment(800)) { + GTEST_SKIP() << "CUDA device is required for prefill weight override tests."; + } + constexpr int64_t m = 3, n = 10, k = 16, block_size = 16; + const std::vector offsets{1, 4, 8}; + const std::vector seg_widths{2, 3, 2}; + + std::vector a(m * k), bias(n); + std::vector b(n * k); + std::vector b_scale(n, 0.5f); + for (int64_t r = 0; r < m; ++r) { + for (int64_t i = 0; i < k; ++i) { + a[r * k + i] = T(static_cast((r + i) % 5 - 2) * 0.25f); + } + } + for (int64_t col = 0; col < n; ++col) { + bias[col] = T(static_cast(col - 4) * 0.125f); + for (int64_t i = 0; i < k; ++i) { + b[col * k + i] = Float8E4M3FN(static_cast((col + i) % 3 - 1)); + } + } + + // Build three original [K, segmentN] weight segments whose values differ from the FP8 reconstruction. + std::vector seg0(k * seg_widths[0]), seg1(k * seg_widths[1]), seg2(k * seg_widths[2]); + auto fill_segment = [&](std::vector& seg, int64_t width, int seed) { + for (int64_t i = 0; i < k; ++i) { + for (int64_t c = 0; c < width; ++c) { + seg[i * width + c] = T(static_cast((i * 3 + c + seed) % 7 - 3) * 0.5f); + } + } + }; + fill_segment(seg0, seg_widths[0], 1); + fill_segment(seg1, seg_widths[1], 2); + fill_segment(seg2, seg_widths[2], 3); + + auto segment_weight = [&](int64_t col, int64_t i, bool use_override) -> float { + if (use_override) { + if (col >= offsets[0] && col < offsets[0] + seg_widths[0]) { + return static_cast(seg0[i * seg_widths[0] + (col - offsets[0])]); + } + if (col >= offsets[1] && col < offsets[1] + seg_widths[1]) { + return static_cast(seg1[i * seg_widths[1] + (col - offsets[1])]); + } + if (col >= offsets[2] && col < offsets[2] + seg_widths[2]) { + return static_cast(seg2[i * seg_widths[2] + (col - offsets[2])]); + } + } + return b[col * k + i].ToFloat() * b_scale[col]; + }; + + for (int32_t prefill : {0, 1}) { + for (bool with_bias : {false, true}) { + for (bool split : {false, true}) { + SCOPED_TRACE(MakeString("prefill=", prefill, " bias=", with_bias, " split=", split)); + const std::vector widths = split ? std::vector{3, 4, 3} : std::vector{n}; + OpTester test("MatMulBlockQuantizedFp8Weight", 1, kMSDomain); + test.AddAttribute("block_size", block_size); + test.AddAttribute>("prefill_weight_offsets", offsets); + if (split) { + test.AddAttribute>("split_sizes", widths); + } + test.AddInput("A", {m, k}, a); + test.AddInput("B", {n, k}, b); + test.AddInput("b_scale", {n, 1}, b_scale); + test.AddOptionalInputEdge(); + if (with_bias) { + test.AddInput("bias", {n}, bias); + } else { + test.AddOptionalInputEdge(); + } + test.AddInput("prefill_numerics", {1}, {prefill}); + test.AddInput("prefill_weight", {k, seg_widths[0]}, seg0); + test.AddInput("prefill_weight2", {k, seg_widths[1]}, seg1); + test.AddInput("prefill_weight3", {k, seg_widths[2]}, seg2); + + int64_t start = 0; + for (size_t out_idx = 0; out_idx < widths.size(); ++out_idx) { + std::vector expected; + for (int64_t r = 0; r < m; ++r) { + for (int64_t col = start; col < start + widths[out_idx]; ++col) { + float sum = with_bias ? static_cast(bias[col]) : 0.0f; + for (int64_t i = 0; i < k; ++i) { + sum += static_cast(a[r * k + i]) * segment_weight(col, i, prefill == 1); + } + expected.emplace_back(sum); + } + } + test.AddOutput(("Y" + std::to_string(out_idx)).c_str(), {m, widths[out_idx]}, expected); + start += widths[out_idx]; + } + test.SetOutputTolerance(0.0f, 0.0f); + std::vector> providers; + providers.push_back(DefaultCudaExecutionProvider()); + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &providers); + } + } + } +} + +TEST(MatMulBlockQuantizedFp8WeightOpTest, PrefillWeightOverridesFp16AndBf16) { + TestPrefillWeightOverrides(); + TestPrefillWeightOverrides(); +} + +TEST(MatMulBlockQuantizedFp8WeightOpTest, RejectsInvalidPrefillWeightSegments) { + if (!HasCudaEnvironment(800)) { + GTEST_SKIP() << "CUDA device is required for prefill weight validation tests."; + } + constexpr int64_t m = 2, n = 8, k = 16; + const std::vector a(m * k, BFloat16(1.0f)); + const std::vector b(n * k, Float8E4M3FN(1.0f)); + const std::vector scales(n, 1.0f); + const std::vector y(m * n, BFloat16(0.0f)); + + for (int kind = 0; kind < 7; ++kind) { + SCOPED_TRACE(kind); + OpTester test("MatMulBlockQuantizedFp8Weight", 1, kMSDomain); + test.AddAttribute("block_size", k); + if (kind == 0) { + test.AddAttribute>("prefill_weight_offsets", {0, 1, 2, 3}); + } else if (kind == 1) { + test.AddAttribute>("prefill_weight_offsets", {1, 4}); + } else if (kind == 2 || kind == 3) { + test.AddAttribute>("prefill_weight_offsets", {1}); + } else if (kind == 4) { + test.AddAttribute>("prefill_weight_offsets", {-1}); + } else if (kind == 5) { + test.AddAttribute>("prefill_weight_offsets", {7}); + } + test.AddInput("A", {m, k}, a); + test.AddInput("B", {n, k}, b); + test.AddInput("b_scale", {n, 1}, scales); + test.AddOptionalInputEdge(); + test.AddOptionalInputEdge(); + test.AddInput("prefill_numerics", {1}, {1}); + if (kind == 1) { + test.AddInput("prefill_weight", {k, 2}, std::vector(k * 2)); + test.AddOptionalInputEdge(); + } else if (kind == 2) { + test.AddInput("prefill_weight", {k + 1, 2}, std::vector((k + 1) * 2)); + } else if (kind == 3) { + test.AddInput("prefill_weight", {k * 2}, std::vector(k * 2)); + } else if (kind >= 4) { + test.AddInput("prefill_weight", {k, 2}, std::vector(k * 2)); + } + test.AddOutput("Y", {m, n}, y); + const std::string expected_error = + kind == 0 ? "prefill_weight_offsets supports at most three original weight segments" + : kind <= 3 ? "Original prefill weight must match activation type and shape [K, segmentN]" + : kind <= 5 ? "Original prefill weight segment exceeds output width" + : "Original prefill weight inputs require matching prefill_weight_offsets"; + std::vector> providers; + providers.push_back(DefaultCudaExecutionProvider()); + test.Run(OpTester::ExpectResult::kExpectFailure, expected_error, {}, nullptr, &providers); + } } TEST(MatMulBlockQuantizedFp8WeightOpTest, SplitOutputsBFloat16) { diff --git a/onnxruntime/test/contrib_ops/moe_test.cc b/onnxruntime/test/contrib_ops/moe_test.cc index 244cf0bb790e9..74e4275232756 100644 --- a/onnxruntime/test/contrib_ops/moe_test.cc +++ b/onnxruntime/test/contrib_ops/moe_test.cc @@ -2254,6 +2254,53 @@ TEST(MoETest, QMoETest_Int2CudaPrepackedRejectsRawFallback) { "requires a packed execution path"); } +TEST(MoETest, QMoETest_Int2CudaPrepackedExpandedRows21To80) { + if (!HasCudaEnvironment(800)) { + GTEST_SKIP() << "CUDA device with compute capability 8.0 or newer is required."; + } + ScopedEnvironmentVariables scoped_env_vars{ + {{"ORT_DISABLE_MOE_GEMV", std::nullopt}, {"ORT_ENABLE_QMOE_INT2_PREFILL", "0"}}}; + constexpr int64_t num_rows = 24; + constexpr int64_t num_experts = 1; + constexpr int64_t hidden_size = 512; + constexpr int64_t inter_size = 512; + constexpr int64_t block_size = 64; + constexpr int64_t pack_size = 4; + constexpr int64_t fc1_rows = 2 * inter_size; + + // All-0xAA bytes represent zero weights in both raw INT2 and SM80 CUTLASS W2_A16 prepacked layouts. + const std::vector fc1_weights(static_cast(num_experts * fc1_rows * hidden_size / pack_size), 0xAA); + const std::vector fc2_weights(static_cast(num_experts * hidden_size * inter_size / pack_size), 0xAA); + const std::vector fc1_scales(static_cast(num_experts * fc1_rows * (hidden_size / block_size)), 1.0f); + const std::vector fc2_scales(static_cast(num_experts * hidden_size * (inter_size / block_size)), 1.0f); + const std::vector input(static_cast(num_rows * hidden_size), 0.25f); + const std::vector router_probs(static_cast(num_rows * num_experts), 1.0f); + const std::vector expected(static_cast(num_rows * hidden_size), 0.0f); + + OpTester tester("QMoE", 1, onnxruntime::kMSDomain); + tester.AddAttribute("k", 1); + tester.AddAttribute("activation_type", "swiglu"); + tester.AddAttribute("swiglu_fusion", 1); + tester.AddAttribute("normalize_routing_weights", 1); + tester.AddAttribute("expert_weight_bits", 2); + tester.AddAttribute("weights_prepacked", 1); + tester.AddAttribute("block_size", block_size); + tester.AddInput("input", {num_rows, hidden_size}, ToFloat16(input)); + tester.AddInput("router_probs", {num_rows, num_experts}, ToFloat16(router_probs)); + tester.AddInput("fc1_experts_weights", {num_experts, fc1_rows, hidden_size / pack_size}, fc1_weights, true); + tester.AddInput("fc1_scales", {num_experts, fc1_rows, hidden_size / block_size}, ToFloat16(fc1_scales), true); + tester.AddOptionalInputEdge(); + tester.AddInput("fc2_experts_weights", {num_experts, hidden_size, inter_size / pack_size}, fc2_weights, true); + tester.AddInput("fc2_scales", {num_experts, hidden_size, inter_size / block_size}, ToFloat16(fc2_scales), true); + tester.AddOutput("output", {num_rows, hidden_size}, ToFloat16(expected)); + + SessionOptions session_options; + ASSERT_STATUS_OK(session_options.config_options.AddConfigEntry(kOrtSessionOptionsDisableCPUEPFallback, "1")); + std::vector> execution_providers; + execution_providers.push_back(DefaultCudaExecutionProvider()); + tester.Run(session_options, OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &execution_providers); +} + // Runs the same multi-row, multi-expert, k=2 INT2 SwiGLU model once with the packed GEMV // path selected and once with it disabled (forcing the already-validated dense fallback from // #32743), so a stride/permutation/top-k bug in the new packed pipeline shows up as a numeric @@ -2603,6 +2650,167 @@ TEST(MoETest, QMoETest_Int2CudaCachedScalesDecodeThenPrefill) { EXPECT_NEAR(prefill[column], first_decode[column], 0.02f) << "mismatch at column " << column; } } + +TEST(MoETest, QMoETest_Int2CudaSharedGpuInitializerCache) { + if (!HasCudaEnvironment(800)) { + GTEST_SKIP() << "CUDA device with compute capability 8.0 or newer is required."; + } + + constexpr int64_t num_experts = 1; + constexpr int64_t hidden_size = 512; + constexpr int64_t inter_size = 512; + constexpr int64_t pack_size = 4; + constexpr int64_t fc1_rows = 2 * inter_size; + const std::string model_bytes = BuildQMoEInt2DynamicRowsModel(); + + auto make_identity_weights = [](int64_t rows, int64_t columns) { + std::vector weights(static_cast(rows * columns / pack_size), 0xAA); + for (int64_t row = 0; row < rows; ++row) { + const int64_t column = row % columns; + const size_t byte_index = static_cast(row * (columns / pack_size) + column / pack_size); + weights[byte_index] |= static_cast(1u << ((column % pack_size) * 2)); + } + return weights; + }; + const auto identity_fc1 = make_identity_weights(fc1_rows, hidden_size); + const auto identity_fc2 = make_identity_weights(hidden_size, inter_size); + const std::vector zero_fc1(identity_fc1.size(), 0xAA); + const std::vector zero_fc2(identity_fc2.size(), 0xAA); + + auto cuda_ep_probe = DefaultCudaExecutionProvider(); + ASSERT_NE(cuda_ep_probe, nullptr); + const auto allocators = cuda_ep_probe->CreatePreferredAllocators(); + ASSERT_FALSE(allocators.empty()); + AllocatorPtr gpu_alloc = allocators[0]; + auto data_transfer = cuda_ep_probe->GetDataTransfer(); + ASSERT_NE(data_transfer, nullptr); + + struct GpuBuffer { + AllocatorPtr alloc; + IDataTransfer* transfer = nullptr; + TensorShape shape; + void* ptr = nullptr; + GpuBuffer(AllocatorPtr allocator, IDataTransfer* dt, TensorShape tensor_shape) + : alloc(std::move(allocator)), transfer(dt), shape(std::move(tensor_shape)) { + ptr = alloc->Alloc(static_cast(shape.Size())); + ORT_ENFORCE(ptr != nullptr); + } + ~GpuBuffer() { + if (ptr != nullptr) { + alloc->Free(ptr); + } + } + void Upload(const std::vector& host) const { + ORT_ENFORCE(host.size() == static_cast(shape.Size())); + Tensor src_tensor(DataTypeImpl::GetType(), shape, + const_cast(host.data()), OrtMemoryInfo()); + Tensor dst_tensor(DataTypeImpl::GetType(), shape, ptr, alloc->Info()); + ORT_ENFORCE(transfer->CopyTensor(src_tensor, dst_tensor).IsOK()); + } + }; + + const TensorShape fc1_shape{num_experts, fc1_rows, hidden_size / pack_size}; + const TensorShape fc2_shape{num_experts, hidden_size, inter_size / pack_size}; + GpuBuffer shared_fc1(gpu_alloc, data_transfer.get(), fc1_shape); + GpuBuffer shared_fc2(gpu_alloc, data_transfer.get(), fc2_shape); + GpuBuffer distinct_fc1(gpu_alloc, data_transfer.get(), fc1_shape); + GpuBuffer distinct_fc2(gpu_alloc, data_transfer.get(), fc2_shape); + shared_fc1.Upload(identity_fc1); + shared_fc2.Upload(identity_fc2); + distinct_fc1.Upload(zero_fc1); + distinct_fc2.Upload(zero_fc2); + + auto make_weight_value = [&](void* ptr, const TensorShape& shape) { + OrtValue val; + Tensor::InitOrtValue(DataTypeImpl::GetType(), shape, ptr, gpu_alloc->Info(), val); + return val; + }; + + const OrtValue shared_fc1_val = make_weight_value(shared_fc1.ptr, fc1_shape); + const OrtValue shared_fc2_val = make_weight_value(shared_fc2.ptr, fc2_shape); + const OrtValue distinct_fc1_val = make_weight_value(distinct_fc1.ptr, fc1_shape); + const OrtValue distinct_fc2_val = make_weight_value(distinct_fc2.ptr, fc2_shape); + + auto create_session = [&](const OrtValue& fc1_val, const OrtValue& fc2_val) { + SessionOptions session_options; + ORT_ENFORCE(session_options.config_options.AddConfigEntry(kOrtSessionOptionsDisableCPUEPFallback, "1").IsOK()); + ORT_ENFORCE(session_options.AddInitializer("fc1_weights", &fc1_val).IsOK()); + ORT_ENFORCE(session_options.AddInitializer("fc2_weights", &fc2_val).IsOK()); + auto session = std::make_unique(session_options, GetEnvironment()); + ORT_ENFORCE(session->RegisterExecutionProvider(DefaultCudaExecutionProvider()).IsOK()); + ORT_ENFORCE(session->Load(model_bytes.data(), static_cast(model_bytes.size())).IsOK()); + ORT_ENFORCE(session->Initialize().IsOK()); + return session; + }; + + auto run_session = [&](InferenceSessionWrapper& session) { + std::vector input(static_cast(hidden_size), MLFloat16(0.5f)); + std::vector router_probs{MLFloat16(1.0f)}; + OrtValue input_value; + OrtValue router_value; + Tensor::InitOrtValue(DataTypeImpl::GetType(), TensorShape({1, hidden_size}), + input.data(), OrtMemoryInfo(), input_value); + Tensor::InitOrtValue(DataTypeImpl::GetType(), TensorShape({1, 1}), + router_probs.data(), OrtMemoryInfo(), router_value); + NameMLValMap feeds{{"input", input_value}, {"router_probs", router_value}}; + const std::vector output_names{"output"}; + std::vector fetches; + ORT_ENFORCE(session.Run(feeds, output_names, &fetches).IsOK()); + const Tensor& output = fetches[0].Get(); + const MLFloat16* output_data = output.Data(); + std::vector result(static_cast(output.Shape().Size())); + std::transform(output_data, output_data + result.size(), result.begin(), + [](MLFloat16 value) { return value.ToFloat(); }); + return result; + }; + + const std::vector all_zeros(static_cast(hidden_size), 0.0f); + + { + ScopedEnvironmentVariables scoped_env_vars{{{"ORT_QMOE_INT2_SHARE_PACKED", "1"}}}; + auto session1 = create_session(shared_fc1_val, shared_fc2_val); + const auto out1 = run_session(*session1); + EXPECT_NE(out1, all_zeros); + + // Overwrite backing GPU memory after session1 packed it. A cache hit in session2 must reuse + // session1's packed buffer rather than re-reading the mutated source buffer. + shared_fc1.Upload(zero_fc1); + shared_fc2.Upload(zero_fc2); + + auto session2 = create_session(shared_fc1_val, shared_fc2_val); + EXPECT_EQ(run_session(*session2), out1); + + // Distinct GPU initializers at different addresses must not be conflated with the cached entry. + auto session_distinct = create_session(distinct_fc1_val, distinct_fc2_val); + EXPECT_EQ(run_session(*session_distinct), all_zeros); + + // Destroy the first session and verify the surviving session still runs on the shared allocation. + session1.reset(); + EXPECT_EQ(run_session(*session2), out1); + + // Release the last owner so the weak cache entry expires, then recreate: it must re-pack from + // the current (zeroed) GPU buffer. + session2.reset(); + auto session3 = create_session(shared_fc1_val, shared_fc2_val); + EXPECT_EQ(run_session(*session3), all_zeros); + } + + // Verify the cache opt-out (ORT_QMOE_INT2_SHARE_PACKED=0) forces each session to pack independently. + { + ScopedEnvironmentVariables scoped_env_vars{{{"ORT_QMOE_INT2_SHARE_PACKED", "0"}}}; + shared_fc1.Upload(identity_fc1); + shared_fc2.Upload(identity_fc2); + auto session_optout_1 = create_session(shared_fc1_val, shared_fc2_val); + const auto out_optout_1 = run_session(*session_optout_1); + EXPECT_NE(out_optout_1, all_zeros); + + shared_fc1.Upload(zero_fc1); + shared_fc2.Upload(zero_fc2); + auto session_optout_2 = create_session(shared_fc1_val, shared_fc2_val); + EXPECT_EQ(run_session(*session_optout_2), all_zeros); + EXPECT_EQ(run_session(*session_optout_1), out_optout_1); + } +} #endif TEST(MoETest, QMoETest_MixedWidthCudaPackedPrefillDefaultEnabled) { diff --git a/onnxruntime/test/contrib_ops/quantize_ops_test.cc b/onnxruntime/test/contrib_ops/quantize_ops_test.cc index 3baf97a70c86a..d5fb7fc31b4ef 100644 --- a/onnxruntime/test/contrib_ops/quantize_ops_test.cc +++ b/onnxruntime/test/contrib_ops/quantize_ops_test.cc @@ -16,47 +16,68 @@ namespace onnxruntime { namespace test { TEST(DequantizeLinearOpTest, CpuScaleTypeConstraints) { - for (int scale_type : {ONNX_NAMESPACE::TensorProto::FLOAT, ONNX_NAMESPACE::TensorProto::FLOAT16, - ONNX_NAMESPACE::TensorProto::BFLOAT16}) { - std::unordered_map versions{{kOnnxDomain, 19}, {kMSDomain, 1}}; - Model model("DequantizeLinearCpuScales", false, ModelMetaData(), PathString(), - IOnnxRuntimeOpSchemaRegistryList(), versions, {}, DefaultLoggingManager().DefaultLogger()); - auto& graph = model.MainGraph(); - ModelTestBuilder builder(graph); - auto* input = builder.MakeInput({2}, {-2, 4}); - NodeArg* scale; - if (scale_type == ONNX_NAMESPACE::TensorProto::FLOAT) { - scale = builder.MakeScalarInitializer(0.5f); - } else if (scale_type == ONNX_NAMESPACE::TensorProto::FLOAT16) { - scale = builder.MakeScalarInitializer(MLFloat16(0.5f)); - } else { - scale = builder.MakeScalarInitializer(BFloat16(0.5f)); - } - builder.AddNode("DequantizeLinear", {input, scale}, {builder.MakeOutput()}, kMSDomain) - .AddAttribute("output_dtype", static_cast(ONNX_NAMESPACE::TensorProto::FLOAT)); - builder.SetGraphOutputs(); - ASSERT_STATUS_OK(graph.Resolve()); - std::string serialized; - ASSERT_TRUE(model.ToProto().SerializeToString(&serialized)); - SessionOptions options; - options.graph_optimization_level = TransformerLevel::Default; - InferenceSessionWrapper session(options, GetEnvironment()); - ASSERT_STATUS_OK(session.RegisterExecutionProvider(DefaultCpuExecutionProvider())); - std::istringstream stream(serialized); - ASSERT_STATUS_OK(session.Load(stream)); - const auto status = session.Initialize(); - if (scale_type == ONNX_NAMESPACE::TensorProto::BFLOAT16) { - EXPECT_EQ(status.Code(), common::NOT_IMPLEMENTED) << status.ErrorMessage(); - EXPECT_NE(status.ErrorMessage().find("DequantizeLinear"), std::string::npos); - } else { - ASSERT_STATUS_OK(status); - std::vector fetches; - ASSERT_STATUS_OK(session.Run(RunOptions{}, builder.feeds_, builder.output_names_, &fetches)); - ASSERT_EQ(fetches.size(), 1u); - const auto actual = fetches[0].Get().DataAsSpan(); - ASSERT_EQ(actual.size(), 2u); - EXPECT_EQ(actual[0], -1.0f); - EXPECT_EQ(actual[1], 2.0f); + for (bool include_accelerators : {false, true}) { + for (int scale_type : {ONNX_NAMESPACE::TensorProto::FLOAT, ONNX_NAMESPACE::TensorProto::FLOAT16, + ONNX_NAMESPACE::TensorProto::BFLOAT16}) { + std::unordered_map versions{{kOnnxDomain, 19}, {kMSDomain, 1}}; + Model model("DequantizeLinearCpuScales", false, ModelMetaData(), PathString(), + IOnnxRuntimeOpSchemaRegistryList(), versions, {}, DefaultLoggingManager().DefaultLogger()); + auto& graph = model.MainGraph(); + ModelTestBuilder builder(graph); + auto* input = builder.MakeInput({2}, {-2, 4}); + NodeArg* scale; + if (scale_type == ONNX_NAMESPACE::TensorProto::FLOAT) { + scale = builder.MakeScalarInitializer(0.5f); + } else if (scale_type == ONNX_NAMESPACE::TensorProto::FLOAT16) { + scale = builder.MakeScalarInitializer(MLFloat16(0.5f)); + } else { + scale = builder.MakeScalarInitializer(BFloat16(0.5f)); + } + builder.AddNode("DequantizeLinear", {input, scale}, {builder.MakeOutput()}, kMSDomain) + .AddAttribute("output_dtype", static_cast(ONNX_NAMESPACE::TensorProto::FLOAT)); + builder.SetGraphOutputs(); + ASSERT_STATUS_OK(graph.Resolve()); + std::string serialized; + ASSERT_TRUE(model.ToProto().SerializeToString(&serialized)); + SessionOptions options; + options.graph_optimization_level = TransformerLevel::Default; + InferenceSessionWrapper session(options, GetEnvironment()); + if (include_accelerators) { + if (auto dml = DefaultDmlExecutionProvider(); dml != nullptr) { + ASSERT_STATUS_OK(session.RegisterExecutionProvider(std::move(dml))); + } + if (auto cuda = DefaultCudaExecutionProvider(); cuda != nullptr) { + ASSERT_STATUS_OK(session.RegisterExecutionProvider(std::move(cuda))); + } + } + ASSERT_STATUS_OK(session.RegisterExecutionProvider(DefaultCpuExecutionProvider())); + std::istringstream stream(serialized); + ASSERT_STATUS_OK(session.Load(stream)); + const auto status = session.Initialize(); + if (scale_type == ONNX_NAMESPACE::TensorProto::BFLOAT16) { + if (!include_accelerators) { + EXPECT_EQ(status.Code(), common::NOT_IMPLEMENTED) << status.ErrorMessage(); + } else { + EXPECT_FALSE(status.IsOK()); + } + EXPECT_NE(status.ErrorMessage().find("DequantizeLinear"), std::string::npos); + } else { + ASSERT_STATUS_OK(status); + if (scale_type == ONNX_NAMESPACE::TensorProto::FLOAT16) { + for (const auto& node : session.GetGraph().Nodes()) { + if (node.OpType() == "DequantizeLinear") { + EXPECT_EQ(node.GetExecutionProviderType(), kCpuExecutionProvider); + } + } + } + std::vector fetches; + ASSERT_STATUS_OK(session.Run(RunOptions{}, builder.feeds_, builder.output_names_, &fetches)); + ASSERT_EQ(fetches.size(), 1u); + const auto actual = fetches[0].Get().DataAsSpan(); + ASSERT_EQ(actual.size(), 2u); + EXPECT_EQ(actual[0], -1.0f); + EXPECT_EQ(actual[1], 2.0f); + } } } } diff --git a/onnxruntime/test/optimizer/qdq_transformer_test.cc b/onnxruntime/test/optimizer/qdq_transformer_test.cc index dddeda71e2ca6..e44783e824002 100644 --- a/onnxruntime/test/optimizer/qdq_transformer_test.cc +++ b/onnxruntime/test/optimizer/qdq_transformer_test.cc @@ -4654,16 +4654,53 @@ TEST(QDQTransformerTests, QDQPropagation_DQForward_ConstantInput_NoPropagation) #if !defined(DISABLE_CONTRIB_OPS) TEST(QDQTransformerTests, QDQPropagation_MSIndependentScaleOutput_NoPropagation) { for (bool explicit_output_dtype : {false, true}) { + for (bool with_zero_point : {false, true}) { + auto build_test_case = [=](ModelTestBuilder& builder) { + auto* input = builder.MakeInput({4}, {-2, -1, 1, 2}); + NodeArg* scale = explicit_output_dtype + ? builder.MakeScalarInitializer(0.5f) + : builder.MakeScalarInitializer(BFloat16(0.5f)); + std::vector dq_inputs{input, scale}; + if (with_zero_point) { + dq_inputs.push_back(builder.MakeScalarInitializer(int8_t(0))); + } + auto* dequantized = builder.MakeIntermediate(); + auto& dq = builder.AddNode("DequantizeLinear", dq_inputs, {dequantized}, kMSDomain); + if (explicit_output_dtype) { + dq.AddAttribute("output_dtype", static_cast(ONNX_NAMESPACE::TensorProto::FLOAT16)); + } + auto* reshaped = builder.MakeIntermediate(); + builder.AddNode("Reshape", {dequantized, builder.Make1DInitializer({2, 2})}, {reshaped}); + builder.AddNode("Transpose", {reshaped}, {builder.MakeOutput()}) + .AddAttribute("perm", std::vector{1, 0}); + }; + auto check_graph = [](Graph& graph) -> Status { + const auto counts = CountOpsInGraph(graph); + TEST_RETURN_IF_NOT(counts.count("com.microsoft.QuantizeLinear") == 0); + TEST_RETURN_IF_NOT(counts.at("com.microsoft.DequantizeLinear") == 1); + TEST_RETURN_IF_NOT(counts.at("Reshape") == 1 && counts.at("Transpose") == 1); + return graph.Resolve(); + }; + ASSERT_STATUS_OK(TestGraphTransformer(build_test_case, 19, DefaultLoggingManager().DefaultLogger(), + std::make_unique(), + TransformerLevel::Level1, 2, nullptr, check_graph)); + } + } +} + +#if !defined(DISABLE_FLOAT8_TYPES) +TEST(QDQTransformerTests, QDQPropagation_MSFp8Bf16_NoPropagation) { + for (bool with_zero_point : {false, true}) { auto build_test_case = [=](ModelTestBuilder& builder) { - auto* input = builder.MakeInput({4}, {-2, -1, 1, 2}); - NodeArg* scale = explicit_output_dtype - ? builder.MakeScalarInitializer(0.5f) - : builder.MakeScalarInitializer(BFloat16(0.5f)); - auto* dequantized = builder.MakeIntermediate(); - auto& dq = builder.AddNode("DequantizeLinear", {input, scale}, {dequantized}, kMSDomain); - if (explicit_output_dtype) { - dq.AddAttribute("output_dtype", static_cast(ONNX_NAMESPACE::TensorProto::FLOAT16)); + auto* input = builder.MakeInput({4}, std::vector(4, Float8E4M3FN(1.0f))); + auto* scale = builder.MakeScalarInitializer(0.5f); + std::vector dq_inputs{input, scale}; + if (with_zero_point) { + dq_inputs.push_back(builder.MakeScalarInitializer(Float8E4M3FN(0.0f))); } + auto* dequantized = builder.MakeIntermediate(); + builder.AddNode("DequantizeLinear", dq_inputs, {dequantized}, kMSDomain) + .AddAttribute("output_dtype", static_cast(ONNX_NAMESPACE::TensorProto::BFLOAT16)); auto* reshaped = builder.MakeIntermediate(); builder.AddNode("Reshape", {dequantized, builder.Make1DInitializer({2, 2})}, {reshaped}); builder.AddNode("Transpose", {reshaped}, {builder.MakeOutput()}) @@ -4681,31 +4718,6 @@ TEST(QDQTransformerTests, QDQPropagation_MSIndependentScaleOutput_NoPropagation) TransformerLevel::Level1, 2, nullptr, check_graph)); } } - -#if !defined(DISABLE_FLOAT8_TYPES) -TEST(QDQTransformerTests, QDQPropagation_MSFp8Bf16_NoPropagation) { - auto build_test_case = [](ModelTestBuilder& builder) { - auto* input = builder.MakeInput({4}, std::vector(4, Float8E4M3FN(1.0f))); - auto* scale = builder.MakeScalarInitializer(0.5f); - auto* dequantized = builder.MakeIntermediate(); - builder.AddNode("DequantizeLinear", {input, scale}, {dequantized}, kMSDomain) - .AddAttribute("output_dtype", static_cast(ONNX_NAMESPACE::TensorProto::BFLOAT16)); - auto* reshaped = builder.MakeIntermediate(); - builder.AddNode("Reshape", {dequantized, builder.Make1DInitializer({2, 2})}, {reshaped}); - builder.AddNode("Transpose", {reshaped}, {builder.MakeOutput()}) - .AddAttribute("perm", std::vector{1, 0}); - }; - auto check_graph = [](Graph& graph) -> Status { - const auto counts = CountOpsInGraph(graph); - TEST_RETURN_IF_NOT(counts.count("com.microsoft.QuantizeLinear") == 0); - TEST_RETURN_IF_NOT(counts.at("com.microsoft.DequantizeLinear") == 1); - TEST_RETURN_IF_NOT(counts.at("Reshape") == 1 && counts.at("Transpose") == 1); - return graph.Resolve(); - }; - ASSERT_STATUS_OK(TestGraphTransformer(build_test_case, 19, DefaultLoggingManager().DefaultLogger(), - std::make_unique(), - TransformerLevel::Level1, 2, nullptr, check_graph)); -} #endif // !defined(DISABLE_FLOAT8_TYPES) #endif // !defined(DISABLE_CONTRIB_OPS) From fac1638cc9523e07675f292eb4c68763bf1abc67 Mon Sep 17 00:00:00 2001 From: Tianlei Wu Date: Mon, 26 Oct 2026 22:02:33 +0000 Subject: [PATCH 12/12] Fix MoE routing NaN race and CUDA plugin test linkage Reduce NaN flags per warp with one shared-memory writer and cover multiple NaNs confined to either routing row. Compile tactic-cache internal tests with the CUDA plugin adapter definitions, and correct stale shared-expert documentation. --- cmake/onnxruntime_unittests.cmake | 14 ++++++++------ docs/contrib_ops/cuda/matmul_block_scaled_fp8.md | 11 ++++++----- onnxruntime/contrib_ops/cuda/moe/qmoe_kernels.cu | 10 ++++++---- .../cuda_kernels/gemm_tactic_cache_test.cc | 4 ++++ .../cuda_kernels/softmax_topk_kernel_test.cc | 16 ++++++++++++++++ 5 files changed, 40 insertions(+), 15 deletions(-) diff --git a/cmake/onnxruntime_unittests.cmake b/cmake/onnxruntime_unittests.cmake index 735b464eaf41e..fc2ed61e7ce22 100644 --- a/cmake/onnxruntime_unittests.cmake +++ b/cmake/onnxruntime_unittests.cmake @@ -1138,12 +1138,14 @@ if (onnxruntime_ENABLE_CUDA_EP_INTERNAL_TESTS AND onnxruntime_BUILD_CUDA_EP_AS_P if (TARGET onnxruntime_providers_cuda_plugin) if (NOT onnxruntime_DISABLE_CONTRIB_OPS) - set_property(SOURCE "${TEST_SRC_DIR}/contrib_ops/cuda_kernels/moe_gemm_int2_test.cc" - APPEND PROPERTY INCLUDE_DIRECTORIES - $) - set_property(SOURCE "${TEST_SRC_DIR}/contrib_ops/cuda_kernels/moe_gemm_int2_test.cc" - APPEND PROPERTY COMPILE_DEFINITIONS - BUILD_CUDA_EP_AS_PLUGIN ORT_API_MANUAL_INIT ORT_USE_EP_API_ADAPTERS=1) + foreach(cuda_plugin_internal_test gemm_tactic_cache_test.cc moe_gemm_int2_test.cc) + set_property(SOURCE "${TEST_SRC_DIR}/contrib_ops/cuda_kernels/${cuda_plugin_internal_test}" + APPEND PROPERTY INCLUDE_DIRECTORIES + $) + set_property(SOURCE "${TEST_SRC_DIR}/contrib_ops/cuda_kernels/${cuda_plugin_internal_test}" + APPEND PROPERTY COMPILE_DEFINITIONS + BUILD_CUDA_EP_AS_PLUGIN ORT_API_MANUAL_INIT ORT_USE_EP_API_ADAPTERS=1) + endforeach() endif() set(onnxruntime_providers_cuda_plugin_ut_impl_src "${ONNXRUNTIME_ROOT}/core/providers/cuda/cuda_allocator.cc" diff --git a/docs/contrib_ops/cuda/matmul_block_scaled_fp8.md b/docs/contrib_ops/cuda/matmul_block_scaled_fp8.md index 501af7ceec045..cbe93a4e44f57 100644 --- a/docs/contrib_ops/cuda/matmul_block_scaled_fp8.md +++ b/docs/contrib_ops/cuda/matmul_block_scaled_fp8.md @@ -364,8 +364,8 @@ Y = bf16(bf16(down * gate) + shared_residual) ``` `sigmoid_bf16` uses BF16 intermediates, including the negative-input -subtraction, matching `SigmoidGateMul(sigmoid_in_fp32=0)`. The projected -gate uses the same 256-thread CUB reduction as that operator. Input 5 +subtraction, matching the CUDA `Sigmoid` operator. The projected gate uses +a local 256-thread CUB reduction. Input 5 `prefill_numerics` still selects the original FP8 projection numerical path. One preparation kernel computes both SiLU/up and projected row gates. On SM90, @@ -374,10 +374,11 @@ and absorb the gate/residual epilogue into its stores (two total launches instead of three). Other architectures, disabled MMA, larger M, small or unaligned K and dense-prefill paths retain the existing projection dispatch and run a separate epilogue. No weights are transformed by the fusion. -The focused native tests compare bit-exactly against the three original CUDA -operators, including dense-prefill and generic fallbacks. +The focused native tests compare bit-exactly against the unfused FP8 projection +and a reference for the BF16 activation and epilogue, including dense-prefill +and generic fallbacks. Optional CPU input 5 selects dense-prefill numerics. Optional inputs 6-8 and `prefill_weight_offsets` preserve the original projection weights/layout when a packed decode projection differs. They are mutually exclusive with shared-expert mode. -Shared-expert preparation depends on the projected-gate helpers from companion PR #33470. +Shared-expert preparation is self-contained and requires no companion operator PR. diff --git a/onnxruntime/contrib_ops/cuda/moe/qmoe_kernels.cu b/onnxruntime/contrib_ops/cuda/moe/qmoe_kernels.cu index 94b664a0079af..559ffe194ffca 100644 --- a/onnxruntime/contrib_ops/cuda/moe/qmoe_kernels.cu +++ b/onnxruntime/contrib_ops/cuda/moe/qmoe_kernels.cu @@ -267,10 +267,9 @@ __global__ void FusedSoftmaxTopKExpertMapsKernel( const int local = threadIdx.x % 128; __shared__ DecodeRoutingMergeSort::TempStorage temp[2]; __shared__ float maxima[2]; - __shared__ bool row_has_nan[2]; + __shared__ bool warp_has_nan[8]; __shared__ float probabilities[2][16]; __shared__ int selected[32]; - if (local == 0) row_has_nan[row] = false; uint64_t keys[4]; bool has_nan = false; #pragma unroll @@ -282,10 +281,13 @@ __global__ void FusedSoftmaxTopKExpertMapsKernel( has_nan |= valid && isnan(value); keys[j] = onnxruntime::cuda::topk::PackStableSortKey(value, valid ? expert : INT_MAX); } + const bool any_nan = __any_sync(0xffffffff, has_nan); + if (local % 32 == 0) warp_has_nan[threadIdx.x / 32] = any_nan; DecodeRoutingMergeSort(temp[row], local).Sort(keys, onnxruntime::cuda::topk::Greater()); - if (has_nan) row_has_nan[row] = true; if (local == 0) maxima[row] = onnxruntime::cuda::topk::UnpackStableSortScore(keys[0]); __syncthreads(); + const bool row_has_nan = warp_has_nan[row * 4] || warp_has_nan[row * 4 + 1] || + warp_has_nan[row * 4 + 2] || warp_has_nan[row * 4 + 3]; #pragma unroll for (int j = 0; j < 4; ++j) { const int rank = local * 4 + j; @@ -294,7 +296,7 @@ __global__ void FusedSoftmaxTopKExpertMapsKernel( indices[row * k + rank] = expert; selected[row * k + rank] = expert; const float value = - row_has_nan[row] ? 0.0f : expf(onnxruntime::cuda::topk::UnpackStableSortScore(keys[j]) - maxima[row]); + row_has_nan ? 0.0f : expf(onnxruntime::cuda::topk::UnpackStableSortScore(keys[j]) - maxima[row]); probabilities[row][rank] = isfinite(value) ? value : 0.0f; } } diff --git a/onnxruntime/test/contrib_ops/cuda_kernels/gemm_tactic_cache_test.cc b/onnxruntime/test/contrib_ops/cuda_kernels/gemm_tactic_cache_test.cc index 9f807df4eefa5..b5afe99a0a853 100644 --- a/onnxruntime/test/contrib_ops/cuda_kernels/gemm_tactic_cache_test.cc +++ b/onnxruntime/test/contrib_ops/cuda_kernels/gemm_tactic_cache_test.cc @@ -26,6 +26,10 @@ #include #endif +#if defined(ORT_UNIT_TEST_HAS_CUDA_PLUGIN_EP) +#include "core/common/logging/logging.h" +#endif + #include "contrib_ops/cuda/llm/gemm_tactic_cache.h" #include "contrib_ops/cuda/llm/gemm_profiler.h" #if !defined(ORT_UNIT_TEST_HAS_CUDA_PLUGIN_EP) diff --git a/onnxruntime/test/contrib_ops/cuda_kernels/softmax_topk_kernel_test.cc b/onnxruntime/test/contrib_ops/cuda_kernels/softmax_topk_kernel_test.cc index 6b02a5a11cb8b..d4b847c3b2e17 100644 --- a/onnxruntime/test/contrib_ops/cuda_kernels/softmax_topk_kernel_test.cc +++ b/onnxruntime/test/contrib_ops/cuda_kernels/softmax_topk_kernel_test.cc @@ -373,6 +373,22 @@ TEST(CUDA_EP_Unittest, FusedSoftmaxTopKExpertMaps_NormalizedAndGraphReplay) { } } +TEST(CUDA_EP_Unittest, FusedSoftmaxTopKExpertMaps_MultipleNaNsAreRowLocal) { + for (int experts : {65, 512}) { + for (int nan_row : {0, 1}) { + SCOPED_TRACE(::testing::Message() << "experts=" << experts << " nan_row=" << nan_row); + std::vector values(2 * experts, 0.0f); + // Several threads observe NaNs in one sorting group; the other row must retain its probabilities. + for (int expert = 0; expert < experts; expert += 4) { + values[nan_row * experts + expert] = -std::numeric_limits::quiet_NaN(); + } + RunFusedRoutingMapsTest(2, experts, 10, false, false, &values); + RunFusedRoutingMapsTest(2, experts, 10, false, false, &values); + RunFusedRoutingMapsTest<__nv_bfloat16>(2, experts, 10, false, false, &values); + } + } +} + TEST(CUDA_EP_Unittest, FusedSoftmaxTopKExpertMaps_UnsupportedUsesFallback) { auto launch = [](int rows, int experts, int k, bool normalize) { return contrib::cuda::TryLaunchSoftmaxTopKExpertMaps(