From 4eda4194f2c586214e2f3023efc85fcbdfae4e6d Mon Sep 17 00:00:00 2001 From: Sam Date: Sat, 13 Dec 2025 15:58:47 +0800 Subject: [PATCH] [Fix] Disable trtllm moe backend for draft model for a qucik fix (#15002) --- python/sglang/srt/server_args.py | 15 +++++++++++++++ test/srt/test_deepseek_v3_fp4_4gpu.py | 2 +- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/python/sglang/srt/server_args.py b/python/sglang/srt/server_args.py index 37bd21da5..42294937b 100644 --- a/python/sglang/srt/server_args.py +++ b/python/sglang/srt/server_args.py @@ -1815,6 +1815,21 @@ class ServerArgs: ): self.speculative_draft_model_revision = "main" + # Avoid using flashinfer_trtllm for speculative MoE runner backend by default + # TODO: Remove this block after verifying no accuracy regression with flashinfer_trtllm speculative backend + from sglang.srt.layers.moe.utils import MoeRunnerBackend + + if self.speculative_moe_runner_backend is None: + self.speculative_moe_runner_backend = ( + "auto" + if self.moe_runner_backend == "flashinfer_trtllm" + else self.moe_runner_backend + ) + else: + assert not MoeRunnerBackend( + self.speculative_moe_runner_backend + ).is_flashinfer_trtllm(), "Currently speculative MoE runner backend cannot be flashinfer_trtllm for risk in some draft models." + if self.speculative_algorithm == "NEXTN": self.speculative_algorithm = "EAGLE" diff --git a/test/srt/test_deepseek_v3_fp4_4gpu.py b/test/srt/test_deepseek_v3_fp4_4gpu.py index 1f236f095..2b4f94889 100644 --- a/test/srt/test_deepseek_v3_fp4_4gpu.py +++ b/test/srt/test_deepseek_v3_fp4_4gpu.py @@ -172,7 +172,7 @@ class TestDeepseekV3FP4MTP(CustomTestCase): f"{speed=:.2f} token/s\n" ) - self.assertGreater(acc_length, 2.04) + self.assertGreater(acc_length, 2.65) self.assertGreater(speed, 150)