chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

Use flashinfer_trtllm moe runner backend to gain around 10% perf on b200 fp8 dpsk (#11816)

Browse Source
This commit is contained in:
b8zhong
2025-10-23 19:12:15 -07:00
committed by GitHub
parent 62eff37ba1
commit f80371ff8c
3 changed files with 76 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
python/sglang/srt/server_args.py
View File

@@ -892,14 +892,19 @@ class ServerArgs:
logger.info(
"Enable FlashInfer AllReduce Fusion on sm100 for DeepseekV3ForCausalLM"
)
if (
self.quantization == "modelopt_fp4"
and self.moe_runner_backend == "auto"
):
if self.moe_runner_backend == "auto":
self.moe_runner_backend = "flashinfer_trtllm"
logger.info(
"Use flashinfer_trtllm as moe runner backend on sm100 for DeepseekV3ForCausalLM"
"Use flashinfer_trtllm as MoE runner backend on sm100 for DeepseekV3ForCausalLM"
)
if self.quantization is None:
# Default DeepSeek V3/R1 native FP8 when not explicitly set,
# Because we need this condition for an assertion in
# flashinfer_trtllm MoE runner backend.
self.quantization = "fp8"
logger.info(
"Quantization not specified, default to fp8 for DeepSeek on sm100"
)
elif model_arch in ["GptOssForCausalLM"]:
if (