From c143f416ce47068a0f77add5042e459f5a1147b7 Mon Sep 17 00:00:00 2001 From: b8zhong Date: Tue, 28 Oct 2025 18:59:01 -0700 Subject: [PATCH] fix: Llama 4 BF16 load on Blackwell (#12308) --- python/sglang/srt/server_args.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/python/sglang/srt/server_args.py b/python/sglang/srt/server_args.py index 9354e7eba..43b93fe40 100644 --- a/python/sglang/srt/server_args.py +++ b/python/sglang/srt/server_args.py @@ -972,10 +972,11 @@ class ServerArgs: "Use trtllm_mha as attention backend on sm100 for Llama4 model" ) if is_sm100_supported() and self.moe_runner_backend == "auto": - self.moe_runner_backend = "flashinfer_trtllm" - logger.info( - "Use flashinfer_trtllm as MoE runner backend on SM100 for Llama4" - ) + if self.quantization in {"fp8", "modelopt_fp8"}: + self.moe_runner_backend = "flashinfer_trtllm" + logger.info( + "Use flashinfer_trtllm as MoE runner backend on SM100 for Llama4" + ) elif model_arch in [ "Gemma2ForCausalLM", "Gemma3ForCausalLM",