From 2aca99d677668a6ccfb26194858a88673d3c2c82 Mon Sep 17 00:00:00 2001 From: leavelet Date: Sun, 21 Jun 2026 16:00:23 +0000 Subject: [PATCH] B300 NextN fp4: keep modelopt_fp4 for fp4-capable runners + generalize MTP exclude-remap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GLM-5.2-NVFP4's NextN (MTP) layer-78 routed experts are fp4, but deepseek_nextn unconditionally dropped modelopt_fp4 for NextN (a DeepSeek-R1-FP4 assumption: its MTP is unquantized). Two fixes (opus-designed, full back-compat matrix): A) Drop fp4 for NextN only when the MoE runner can't consume it. Keep fp4 for flashinfer trtllm/trtllm_routed/cutedsl/cutlass (extends wxiwnd's helper to include our trtllm runner). R1-FP4/V3-FP4 (non-flashinfer-fp4 or unquantized MTP) still drop → bf16, no regression. B) The NextN hf_to_sglang_mapper was hardcoded `model.layers.61 -> model.decoder` (R1's MTP idx). For GLM (MTP=layer 78) this never remapped the layer-78 self_attn/indexer exclude patterns to the internal model.decoder prefix → NextN attention wrongly quantized fp4 → crash. Worse, keeping the hardcoded 61 would mis-remap GLM's *real* layer 61. Fix: empty the class mapper and inject `model.layers.{num_hidden_layers} -> model.decoder` per-config in loader.py (gated on num_nextn_predict_layers; dataclasses.replace, no class mutation), generalizing to any MTP index. Co-Authored-By: Claude Opus 4.8 (1M context) --- python/sglang/srt/model_loader/loader.py | 18 ++++++++++ python/sglang/srt/models/deepseek_nextn.py | 42 ++++++++++++++++++---- 2 files changed, 53 insertions(+), 7 deletions(-) diff --git a/python/sglang/srt/model_loader/loader.py b/python/sglang/srt/model_loader/loader.py index 27d189d65..79d6c0c87 100644 --- a/python/sglang/srt/model_loader/loader.py +++ b/python/sglang/srt/model_loader/loader.py @@ -247,6 +247,24 @@ def _get_quantization_config( f"{supported_dtypes}" ) hf_to_sglang_mapper = getattr(model_class, "hf_to_sglang_mapper", None) + # NextN/MTP models map their HF decoder prefix "model.layers.{N}" to the + # internal "model.decoder". N == num_hidden_layers is checkpoint specific + # (and for an N-layer model "model.layers.61" can be a real layer), so + # build a per-config copy of the mapper instead of hardcoding the index. + if hf_to_sglang_mapper is not None and getattr( + model_config.hf_config, "num_nextn_predict_layers", None + ): + nextn_layer = getattr(model_config.hf_config, "num_hidden_layers", None) + if nextn_layer is not None: + from dataclasses import replace + + hf_to_sglang_mapper = replace( + hf_to_sglang_mapper, + orig_to_new_substr={ + **hf_to_sglang_mapper.orig_to_new_substr, + f"model.layers.{nextn_layer}": "model.decoder", + }, + ) # pass mappings by reference to quant_config if hf_to_sglang_mapper is not None and quant_config is not None: quant_config.apply_weight_name_mapper(hf_to_sglang_mapper) diff --git a/python/sglang/srt/models/deepseek_nextn.py b/python/sglang/srt/models/deepseek_nextn.py index 715fa6ac2..992b5266b 100644 --- a/python/sglang/srt/models/deepseek_nextn.py +++ b/python/sglang/srt/models/deepseek_nextn.py @@ -45,6 +45,7 @@ from sglang.srt.layers.dp_attention import ( ) from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.logits_processor import LogitsProcessor +from sglang.srt.layers.moe import get_moe_runner_backend from sglang.srt.layers.quantization import Fp8Config from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.layers.vocab_parallel_embedding import ( @@ -78,6 +79,30 @@ def _log_eagle_accept_cp_draft_hidden_debug(key: str, message: str, *args): ) +def _should_drop_nextn_modelopt_fp4_quant_config( + quant_config: Optional[QuantizationConfig], +) -> bool: + """Whether to drop the modelopt_fp4 quant config for the NextN/MTP module. + + DeepSeek-R1-FP4 / V3-0324-FP4 ship an *unquantized* MTP module (no fp4 scales + for the MTP MoE, and their hf_quant_config does NOT list the MTP layer in + `ignore`); for those we must drop fp4 so the whole MTP loads bf16 (PR #7376). + GLM-5.2-NVFP4 instead ships fp4 routed experts on the MTP layer and runs on a + flashinfer fp4 MoE backend that consumes them — there we keep fp4 and let the + exclude-list make attn / indexer / shared_experts / gate bf16. + """ + if quant_config is None or quant_config.get_name() != "modelopt_fp4": + return False + backend = get_moe_runner_backend() + keep_fp4 = ( + backend.is_flashinfer_trtllm() + or backend.is_flashinfer_trtllm_routed() + or backend.is_flashinfer_cutedsl() + or backend.is_flashinfer_cutlass() + ) + return not keep_fp4 + + class DeepseekModelNextN(nn.Module): def __init__( @@ -96,7 +121,7 @@ class DeepseekModelNextN(nn.Module): else: moe_quant_config_override = None - if quant_config is not None and quant_config.get_name() == "modelopt_fp4": + if _should_drop_nextn_modelopt_fp4_quant_config(quant_config): logger.warning( "Overriding DeepseekV3ForCausalLMNextN quant config for modelopt_fp4 Deepseek model." ) @@ -498,13 +523,16 @@ class DeepseekModelNextN(nn.Module): class DeepseekV3ForCausalLMNextN(DeepseekV3ForCausalLM): - # Support amd/DeepSeek-R1-0528-MXFP4 renaming: model.layers.61*. - # Ref: HF config.json for amd/DeepSeek-R1-0528-MXFP4 - # https://huggingface.co/amd/DeepSeek-R1-0528-MXFP4/blob/main/config.json + # The NextN/MTP decoder lives at HF prefix ``model.layers.{num_hidden_layers}`` + # but internally at ``model.decoder``. The concrete layer index is checkpoint + # specific (DeepSeek-R1 / R1-0528-MXFP4 => 61, GLM-5.2 => 78), and for an + # N-layer model ``model.layers.61`` may be a *real* hidden layer — so the + # ``model.layers.{N}`` substr key is injected from model_config at load time + # (see model_loader/loader.py), never hardcoded. Kept as a (placeholder) class + # attribute so the loader knows this model family remaps its HF MTP-layer + # prefix onto ``model.decoder``. hf_to_sglang_mapper = WeightsMapper( - orig_to_new_substr={ - "model.layers.61": "model.decoder", - }, + orig_to_new_substr={}, ) def __init__(