Add --speculative-moe-runner-backend server arg (#10183)

This commit is contained in:
Trevor Morris
2025-11-04 00:20:56 -08:00
committed by GitHub
parent 83804bc626
commit dbcf85b7f0
15 changed files with 109 additions and 68 deletions
+2 -2
View File
@@ -74,8 +74,8 @@ from sglang.srt.layers.linear import (
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.moe import (
get_moe_a2a_backend,
get_moe_runner_backend,
should_use_flashinfer_cutlass_moe_fp4_allgather,
should_use_flashinfer_trtllm_moe,
)
from sglang.srt.layers.moe.ep_moe.layer import DeepEPMoE, get_moe_impl_class
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
@@ -503,7 +503,7 @@ class MoEGate(nn.Module):
torch.bfloat16
if quant_config is not None
and quant_config.get_name() == "modelopt_fp4"
and should_use_flashinfer_trtllm_moe()
and get_moe_runner_backend().is_flashinfer_trtllm()
else torch.float32
)
self.e_score_correction_bias = nn.Parameter(