chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

Update dsv3 quantization auto setting for sm100 (#12778)

Browse Source
This commit is contained in:
Ke Bao
2025-11-06 22:39:13 +08:00
committed by GitHub
parent fc84b0730c
commit cd135bfe30
1 changed files with 22 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
python/sglang/srt/server_args.py
View File

@@ -912,19 +912,32 @@ class ServerArgs:
logger.info(
"Enable FlashInfer AllReduce Fusion on sm100 for DeepseekV3ForCausalLM"
)
if self.moe_a2a_backend == "none" and self.moe_runner_backend == "auto":
self.moe_runner_backend = "flashinfer_trtllm"
logger.info(
"Use flashinfer_trtllm as MoE runner backend on sm100 for DeepseekV3ForCausalLM"
)
if self.quantization is None:
# Default DeepSeek V3/R1 native FP8 when not explicitly set,
# Because we need this condition for an assertion in
# flashinfer_trtllm MoE runner backend.
quantization_config = getattr(hf_config, "quantization_config", None)
quant_method = (
quantization_config.get("quant_method")
if quantization_config is not None
else None
)
if self.quantization is None:
# Default DeepSeek V3/R1 native FP8 when not explicitly set,
# Because we need this condition for an assertion in
# flashinfer_trtllm MoE runner backend.
if quant_method is None:
self.quantization = "fp8"
logger.info(
"Quantization not specified, default to fp8 for DeepSeek on sm100"
)
else:
self.quantization = quant_method
if (
self.moe_a2a_backend == "none"
and self.moe_runner_backend == "auto"
and self.quantization in ["fp8", "modelopt_fp8", "modelopt_fp4"]
):
self.moe_runner_backend = "flashinfer_trtllm"
logger.info(
"Use flashinfer_trtllm as MoE runner backend on sm100 for DeepseekV3ForCausalLM"
)
elif model_arch in ["GptOssForCausalLM"]:
if (