[CUDA] Add native dynamic W8A8 execution to block-FP8 QMoE

docs/ContribOperators.md

@@ -6426,11 +6426,14 @@ This version of the operator has been available since version 1 of the 'com.micr
           dequantized_weight[e, n, k] = float(weight[e, n, k]) * scale[e, n / block_size, k / block_size]
         Partial edge blocks are allowed. No zero points or activation scales are accepted.
         Without a positive block_size, FP8 uses the legacy per-expert fc*_global_scale inputs instead.
-        Block-scaled FP8 does not use global scales. Activations retain the input type (weight-only quantization).
-        On CUDA SM80-or-newer GPUs, SiLU/SwiGLU block-scaled FP8 uses fused weight-only GEMV/GEMM by default.
+        Block-scaled FP8 does not use global scales. By default activations retain the input type (weight-only quantization).
+        activation_quantization='dynamic_fp8' instead selects dynamic per-token K128 activation quantization,
+        with native FP8 dot products, FP32 block-scale accumulation and FP32 routed down-projection reduction.
+        On CUDA SM80-or-newer GPUs, weight-only SiLU/SwiGLU block-scaled FP8 uses fused GEMV/GEMM by default.
         These kernels decode and scale FP8 tiles on chip without materializing full FP16/BF16 weight buffers.
         Set ORT_ENABLE_FP8_FUSED=0 before session creation to select the compact dense-dequant fallback.
-        Other activation types and unsupported routing sizes retain the fallback.
+        Other activation types and unsupported routing sizes retain the fallback in weight-only mode.
+        Dynamic activation quantization rejects unsupported configurations instead of falling back to weight-only execution.
         The WebGPU block-FP8 kernel supports block_size=128 with float32 scales and rejects projections
         that exceed 32-bit shader addressing or the device's per-dimension dispatch limit.
 
@@ -6482,6 +6485,8 @@ This version of the operator has been available since version 1 of the 'com.micr
 <dd>Alpha parameter used in activation function.</dd>
 <dt><tt>activation_beta</tt> : float</dt>
 <dd>Beta parameter used in activation function.</dd>
+<dt><tt>activation_quantization</tt> : string</dt>
+<dd>'none' (default) retains weight-only execution. 'dynamic_fp8' quantizes consecutive groups of 128 features per token to saturating nearest-even E4M3FN using FP32 div.full normalization and scale max(amax(abs(x)),1e-10) * float32(1/448). Requires block-scaled FP8 weights, block_size=128, FP32 weight scales, FP32/FP16/BF16 inputs and 128-aligned projections. Gate/up outputs narrow to the input type; their FP32 SiLU/SwiGLU result is quantized directly for FC2. FC2 stays FP32, multiplies routing weights rounded to the input type, and reduces in FP32 before the final output cast. Biases, supplied activation/global scales, zero points, router_weights and nonstandard activation parameters are unsupported. CUDA requires native FP8 MMA (SM89+) and at most 65535 token/expert pairs.</dd>
 <dt><tt>activation_type</tt> : string</dt>
 <dd>Activation function to use. Choose from relu, gelu, silu, swiglu and identity. CUDA also supports relu2 (squared ReLU). Default is relu</dd>
 <dt><tt>block_size</tt> : int</dt>
@@ -6499,7 +6504,7 @@ This version of the operator has been available since version 1 of the 'com.micr
 <dt><tt>normalize_routing_weights</tt> : int</dt>
 <dd>Whether to normalize routing weights</dd>
 <dt><tt>quant_type</tt> : string</dt>
-<dd>Quantization type: 'int' for integer quantization (default), 'fp4' for MXFP4 quantization, 'nvfp4' for NVFP4 quantization, 'fp8' for FP8 e4m3 weight-only quantization, or 'wfp4afp8' for MXFP4 weight with FP8 activation. When quant_type is 'fp4' or 'nvfp4', weights are stored in E2M1 FP4 format (2 values per byte), fc*_scales inputs contain the FP4 block scales, and fc*_global_scale inputs must be provided. 'fp4' uses Float8E8M0 block scales with block_size 32; 'nvfp4' uses Float8E4M3FN block scales with block_size 16. 'fp8' uses float8e4m3fn weights and, when block_size > 0, float32, float16, or bfloat16 square-block scales in fc*_scales instead of fc*_global_scale. The INT3 profile requires integer quantization.</dd>
+<dd>Quantization type: 'int' for integer quantization (default), 'fp4' for MXFP4 quantization, 'nvfp4' for NVFP4 quantization, 'fp8' for FP8 e4m3 weights (weight-only unless activation_quantization is set), or 'wfp4afp8' for MXFP4 weight with FP8 activation. When quant_type is 'fp4' or 'nvfp4', weights are stored in E2M1 FP4 format (2 values per byte), fc*_scales inputs contain the FP4 block scales, and fc*_global_scale inputs must be provided. 'fp4' uses Float8E8M0 block scales with block_size 32; 'nvfp4' uses Float8E4M3FN block scales with block_size 16. 'fp8' uses float8e4m3fn weights and, when block_size > 0, float32, float16, or bfloat16 square-block scales in fc*_scales instead of fc*_global_scale. The INT3 profile requires integer quantization.</dd>
 <dt><tt>swiglu_fusion</tt> : int</dt>
 <dd>0: not fused, 1: fused and interleaved. 2: fused and not interleaved.</dd>
 <dt><tt>swiglu_limit</tt> : float</dt>
@@ -9014,5 +9019,3 @@ No versioning maintained for experimental ops.
 <dt><tt>T</tt> : tensor(float)</dt>
 <dd>Constrain input and output types to float32 tensors.</dd>
 </dl>
-
-

docs/contrib_ops/cuda/moe_qmoe.md

@@ -1411,7 +1411,46 @@ partial final tiles, and the decode scratch bound. Internal CUDA tests in
 `qmoe_fp8_compaction_test.cc` check expert remapping, unused slots, and selected
 weight/bias decoding for separate and fused layouts.
 
-### 10.4 Kernel instantiation files
+### 10.4 Dynamic block-FP8 activations
+
+Set `activation_quantization="dynamic_fp8"` to opt into W8A8 execution instead of
+the default `activation_quantization="none"` weight-only path. This mode requires
+`quant_type="fp8"`, `expert_weight_bits=8`, `block_size=128`, FP32 weight scale
+grids, FP32/FP16/BF16 inputs, and hidden/intermediate dimensions divisible by 128.
+The existing row-major FP8 weight layouts and separate gate/up or fused SwiGLU
+layouts are unchanged. Standard SiLU/SwiGLU is supported; biases, zero points,
+activation clipping/parameter overrides, `router_weights`, and supplied
+activation/global scales are rejected.
+
+For each token and consecutive group of 128 features, compute the FP32 scale
+`max(amax(abs(x)), 1e-10) * float32(1/448)`, then clamp `x / scale` to `[-448,448]`
+and convert to E4M3FN with round-to-nearest-even. This quantization is performed
+before gate/up projection and again directly on its FP32 SiLU/SwiGLU result.
+The rounded FP32 reciprocal multiplication is intentional: replacing it with
+a correctly rounded division can shift a scale by one ULP and change FP8 codes.
+Normalization uses FP32 `div.full` before the nearest-even FP8 conversion, not
+`div.rn`; the latter can move a midpoint by one ULP and select a different code.
+Gate/up projection outputs round to the input type, but there is **no**
+FP16/BF16 rounding between the activation product and the second quantizer.
+For FP32 input, gate/up values and the final output remain FP32. CUDA's FLOAT
+registration is restricted to FP8 weights with FP32 scales and requires this
+dynamic mode; it does not claim existing FLOAT/integer CPU QMoE models or enable
+FLOAT weight-only execution.
+Native FP8 MMA accumulates each K128 dot in FP32, applies that group's activation
+and weight scales, then adds it to the FP32 projection accumulator.
+Down projection remains FP32, multiplies the selected routing coefficients
+rounded to the input type, and reduces in top-k order in FP32 before casting
+the final output. This is not equivalent to FP8 QDQ followed by A16 GEMM.
+
+CUDA requires SM89+ and a kernel compiled for SM89 or newer. The existing
+`onnxruntime_USE_FP8_QMOE=ON` build option is required. Dynamic mode rejects
+`ORT_ENABLE_FP8_FUSED=0` and invocations with more than 65535 token/expert pairs
+instead of silently falling back to weight-only execution. Chunk longer
+prefills at the graph boundary. Activation scratch contains FP8 values and FP32
+group scales; no dense expert-weight buffer is allocated. Debug dispatch is
+`fp8_dynamic_mma` with `DequantWeightBytes=0`.
+
+### 10.5 Kernel instantiation files
 
 | File | Template |
 |------|----------|

onnxruntime/contrib_ops/cpu/moe/moe_quantization_cpu.cc

@@ -922,6 +922,8 @@ template <typename T>
 QMoECPU<T>::QMoECPU(const OpKernelInfo& op_kernel_info)
     : OpKernel(op_kernel_info),
       MoEBaseCPU(op_kernel_info) {
+  ORT_ENFORCE(op_kernel_info.GetAttrOrDefault<std::string>("activation_quantization", "none") == "none",
+              "CPU QMoE does not support dynamic FP8 activation quantization.");
   ORT_ENFORCE(activation_type_ != ActivationType::SwiGLU || swiglu_fusion_ == 1,
               "CPU QMoE only supports interleaved SwiGLU format. Please set swiglu_fusion=1.");
   ORT_ENFORCE(op_kernel_info.GetAttr<int64_t>("expert_weight_bits", &expert_weight_bits_).IsOK());

onnxruntime/contrib_ops/cuda/cuda_contrib_kernels.cc

@@ -102,6 +102,9 @@ class CUDA_MS_OP_TYPED_CLASS_NAME(1, MLFloat16, MoE);
 class CUDA_MS_OP_TYPED_CLASS_NAME(1, BFloat16, MoE);
 class CUDA_MS_OP_TYPED_CLASS_NAME(1, BFloat16, QMoE);
 class CUDA_MS_OP_TYPED_CLASS_NAME(1, MLFloat16, QMoE);
+#if !defined(DISABLE_FLOAT8_TYPES)
+class CUDA_MS_OP_TYPED_CLASS_NAME(1, float, QMoE);
+#endif
 class CUDA_MS_OP_TYPED_CLASS_NAME(1, float_float, MultiHeadAttention);
 class CUDA_MS_OP_TYPED_CLASS_NAME(1, float_MLFloat16, MultiHeadAttention);
 class CUDA_MS_OP_TYPED_CLASS_NAME(1, MLFloat16_float, MultiHeadAttention);
@@ -470,6 +473,9 @@ Status RegisterCudaContribKernels(KernelRegistry& kernel_registry) {
       BuildKernelCreateInfo<CUDA_MS_OP_TYPED_CLASS_NAME(1, BFloat16, MoE)>,
       BuildKernelCreateInfo<CUDA_MS_OP_TYPED_CLASS_NAME(1, BFloat16, QMoE)>,
       BuildKernelCreateInfo<CUDA_MS_OP_TYPED_CLASS_NAME(1, MLFloat16, QMoE)>,
+#if !defined(DISABLE_FLOAT8_TYPES)
+      BuildKernelCreateInfo<CUDA_MS_OP_TYPED_CLASS_NAME(1, float, QMoE)>,
+#endif
       BuildKernelCreateInfo<CUDA_MS_OP_TYPED_CLASS_NAME(1, float_float, MultiHeadAttention)>,
       BuildKernelCreateInfo<CUDA_MS_OP_TYPED_CLASS_NAME(1, float_MLFloat16, MultiHeadAttention)>,
       BuildKernelCreateInfo<CUDA_MS_OP_TYPED_CLASS_NAME(1, MLFloat16_float, MultiHeadAttention)>,