From a3d0c2394b8ee8f1d1dd6f8b9e39ed898490067b Mon Sep 17 00:00:00 2001 From: leavelet Date: Sun, 21 Jun 2026 16:32:59 +0000 Subject: [PATCH] B300 NextN fp4: also honor speculative MoE runner backend in keep-decision (+ debug log) The NextN/MTP draft is built by the EAGLE draft worker, where get_moe_runner_backend() didn't resolve to flashinfer_trtllm, so the fp4 drop fired anyway -> 6144-vs-3072 crash in draft load. Check both the main and speculative runner backends (spec is AUTO when --speculative-moe-runner-backend unset); keep fp4 if either is a flashinfer fp4 backend. Temporary [NextN-fp4] warning logs the resolved backends. Co-Authored-By: Claude Opus 4.8 (1M context) --- python/sglang/srt/models/deepseek_nextn.py | 28 +++++++++++++++++----- 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/python/sglang/srt/models/deepseek_nextn.py b/python/sglang/srt/models/deepseek_nextn.py index 992b5266b..94b2e7f3a 100644 --- a/python/sglang/srt/models/deepseek_nextn.py +++ b/python/sglang/srt/models/deepseek_nextn.py @@ -93,12 +93,28 @@ def _should_drop_nextn_modelopt_fp4_quant_config( """ if quant_config is None or quant_config.get_name() != "modelopt_fp4": return False - backend = get_moe_runner_backend() - keep_fp4 = ( - backend.is_flashinfer_trtllm() - or backend.is_flashinfer_trtllm_routed() - or backend.is_flashinfer_cutedsl() - or backend.is_flashinfer_cutlass() + # The NextN/MTP draft is built by the EAGLE draft worker, where the effective + # MoE runner may be the *speculative* backend (which is AUTO when + # --speculative-moe-runner-backend is unset), not the main runner. Keep fp4 if + # EITHER the main or the speculative runner is a flashinfer fp4 backend. + from sglang.srt.layers.moe.utils import get_speculative_moe_runner_backend + + def _is_fp4_capable(b) -> bool: + return ( + b.is_flashinfer_trtllm() + or b.is_flashinfer_trtllm_routed() + or b.is_flashinfer_cutedsl() + or b.is_flashinfer_cutlass() + ) + + main_backend = get_moe_runner_backend() + spec_backend = get_speculative_moe_runner_backend() + keep_fp4 = _is_fp4_capable(main_backend) or _is_fp4_capable(spec_backend) + logger.warning( + "[NextN-fp4] main_runner=%s spec_runner=%s keep_fp4=%s", + main_backend, + spec_backend, + keep_fp4, ) return not keep_fp4