diff --git a/python/sglang/srt/models/qwen3_moe.py b/python/sglang/srt/models/qwen3_moe.py index d3acc629b..3a14bc5fe 100644 --- a/python/sglang/srt/models/qwen3_moe.py +++ b/python/sglang/srt/models/qwen3_moe.py @@ -49,6 +49,7 @@ from sglang.srt.layers.moe import ( from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE from sglang.srt.layers.moe.topk import TopK +from sglang.srt.layers.moe.utils import RoutingMethodType from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.layers.radix_attention import RadixAttention from sglang.srt.layers.rotary_embedding import MRotaryEmbedding, get_rope @@ -111,6 +112,7 @@ class Qwen3MoeSparseMoeBlock(nn.Module): intermediate_size=config.moe_intermediate_size, quant_config=quant_config, prefix=add_prefix("experts", prefix), + routing_method_type=RoutingMethodType.Renormalize, ) self.gate = ReplicatedLinear( diff --git a/python/sglang/srt/server_args.py b/python/sglang/srt/server_args.py index f81b9ea72..90f0d29b9 100644 --- a/python/sglang/srt/server_args.py +++ b/python/sglang/srt/server_args.py @@ -1099,6 +1099,29 @@ class ServerArgs: f"Disabling Radix Cache for {model_arch} as it is not yet supported." ) self.disable_radix_cache = True + elif model_arch in [ + "Qwen3MoeForCausalLM", + "Qwen3VLMoeForConditionalGeneration", + ]: + if is_sm100_supported(): + quantization_config = getattr(hf_config, "quantization_config", None) + quant_method = ( + quantization_config.get("quant_method") + if quantization_config is not None + else None + ) + if self.quantization is None and quant_method is not None: + self.quantization = quant_method + if ( + self.quantization == "fp8" + and self.moe_a2a_backend == "none" + and self.moe_runner_backend == "auto" + ): + self.moe_runner_backend = "flashinfer_trtllm" + logger.info( + "Use flashinfer_trtllm as MoE runner backend on sm100 for " + f"{model_arch}" + ) elif model_arch in ["Qwen3NextForCausalLM"]: if not self.disable_radix_cache: logger.warning( @@ -1106,7 +1129,24 @@ class ServerArgs: "overlap schedule currently, try to use --disable-radix-cache if overlap schedule is necessary" ) self.disable_overlap_schedule = True - + if is_sm100_supported(): + quantization_config = getattr(hf_config, "quantization_config", None) + quant_method = ( + quantization_config.get("quant_method") + if quantization_config is not None + else None + ) + if self.quantization is None and quant_method is not None: + self.quantization = quant_method + if ( + self.quantization == "fp8" + and self.moe_a2a_backend == "none" + and self.moe_runner_backend == "auto" + ): + self.moe_runner_backend = "flashinfer_trtllm" + logger.info( + "Use flashinfer_trtllm as MoE runner backend on sm100 for Qwen3NextForCausalLM" + ) if is_deepseek_nsa(hf_config): if ( self.attention_backend is None