Add support for bf16 x bf16 cutlass fused MoE (#10275)

Co-authored-by: Sam Li <lsam@nvidia.com>
Co-authored-by: jackeyhua <jackeyhuasjtu@gmail.com>
This commit is contained in:
Nicolas Castet
2025-11-24 20:49:39 -06:00
committed by GitHub
parent de430b6745
commit b0a26ba624
5 changed files with 162 additions and 10 deletions

View File

@@ -1202,6 +1202,7 @@ class ServerArgs:
)
self.disable_overlap_schedule = True
if is_sm100_supported():
self.attention_backend = "triton"
quantization_config = getattr(hf_config, "quantization_config", None)
quant_method = (
quantization_config.get("quant_method")
@@ -1468,8 +1469,8 @@ class ServerArgs:
def _handle_moe_kernel_config(self):
if self.moe_runner_backend == "flashinfer_cutlass":
assert (
self.quantization == "modelopt_fp4"
), "modelopt_fp4 quantization is required for Flashinfer Cutlass MOE"
self.quantization == "modelopt_fp4" or self.quantization is None
), "modelopt_fp4 quantization or bf16 is required for Flashinfer Cutlass MOE"
assert self.ep_size in [
1,
self.tp_size,