Add support for bf16 x bf16 cutlass fused MoE (#10275)
Co-authored-by: Sam Li <lsam@nvidia.com> Co-authored-by: jackeyhua <jackeyhuasjtu@gmail.com>
This commit is contained in:
@@ -1202,6 +1202,7 @@ class ServerArgs:
|
||||
)
|
||||
self.disable_overlap_schedule = True
|
||||
if is_sm100_supported():
|
||||
self.attention_backend = "triton"
|
||||
quantization_config = getattr(hf_config, "quantization_config", None)
|
||||
quant_method = (
|
||||
quantization_config.get("quant_method")
|
||||
@@ -1468,8 +1469,8 @@ class ServerArgs:
|
||||
def _handle_moe_kernel_config(self):
|
||||
if self.moe_runner_backend == "flashinfer_cutlass":
|
||||
assert (
|
||||
self.quantization == "modelopt_fp4"
|
||||
), "modelopt_fp4 quantization is required for Flashinfer Cutlass MOE"
|
||||
self.quantization == "modelopt_fp4" or self.quantization is None
|
||||
), "modelopt_fp4 quantization or bf16 is required for Flashinfer Cutlass MOE"
|
||||
assert self.ep_size in [
|
||||
1,
|
||||
self.tp_size,
|
||||
|
||||
Reference in New Issue
Block a user