From 34f7564df0378d2dba929549c0350cbc12713666 Mon Sep 17 00:00:00 2001 From: Kaixi Hou Date: Tue, 4 Nov 2025 14:19:37 -0800 Subject: [PATCH] [NVIDIA] Fix wrong symmetric sizes for fp4 cases (#12640) --- python/sglang/srt/layers/quantization/modelopt_quant.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/python/sglang/srt/layers/quantization/modelopt_quant.py b/python/sglang/srt/layers/quantization/modelopt_quant.py index 46bc05182..910f1d3c0 100755 --- a/python/sglang/srt/layers/quantization/modelopt_quant.py +++ b/python/sglang/srt/layers/quantization/modelopt_quant.py @@ -1602,8 +1602,13 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase): x_sf = nvfp4_block_scale_interleave(x_sf) with use_symmetric_memory(get_tp_group()) as sm: + # The x might be packed in the case of fp4. So, use the output dim of the + # weight of the second GEMM. symm_output = torch.empty( - x.shape[0], x.shape[1], dtype=output_dtype, device=x.device + x.shape[0], + layer.w2_weight.shape[1], + dtype=output_dtype, + device=x.device, ) sm.tag(symm_output)