From bacb3825fe0984a518ff6daec9163c27270e53c1 Mon Sep 17 00:00:00 2001 From: b8zhong Date: Wed, 29 Oct 2025 11:31:02 -0700 Subject: [PATCH] fix: llama 4 + trtllm gen + fp8 kv cache incompatibility (#12347) --- python/sglang/srt/server_args.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/python/sglang/srt/server_args.py b/python/sglang/srt/server_args.py index 155e244f2..f1310cc95 100644 --- a/python/sglang/srt/server_args.py +++ b/python/sglang/srt/server_args.py @@ -971,6 +971,13 @@ class ServerArgs: logger.warning( "Use trtllm_mha as attention backend on sm100 for Llama4 model" ) + if is_sm100_supported() and self.attention_backend == "trtllm_mha": + # TODO(brayden): remove this once TRTLLM MHA kernel for FP8 w/ tileSizeKv=128 is available. + # This is a Llama 4 specific issue only. + self.kv_cache_dtype = "bfloat16" + logger.warning( + "Setting kv_cache_dtype to bfloat16 for Llama4 with trtllm_mha backend, due to a missing FlashInfer TRTLLM MHA kernel for FP8 KV Cache" + ) if is_sm100_supported() and self.moe_runner_backend == "auto": if self.quantization in {"fp8", "modelopt_fp8"}: self.moe_runner_backend = "flashinfer_trtllm"