From 9327482baa90397d63544e2b63ab92829561ae5c Mon Sep 17 00:00:00 2001 From: haoyangli-amd Date: Tue, 16 Dec 2025 14:54:31 +0800 Subject: [PATCH] [bugfix][quark] Fixed an issue where `per_token` could not be properly recognized when the token count was 1. (#14415) --- python/sglang/srt/layers/quantization/fp8_utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/python/sglang/srt/layers/quantization/fp8_utils.py b/python/sglang/srt/layers/quantization/fp8_utils.py index 6a5a70237..6da7ec66c 100644 --- a/python/sglang/srt/layers/quantization/fp8_utils.py +++ b/python/sglang/srt/layers/quantization/fp8_utils.py @@ -992,7 +992,9 @@ def apply_fp8_linear( # torch.scaled_mm supports per tensor weights + activations only # so fallback to naive if per channel or per token per_tensor_weights = weight_scale.numel() == 1 - per_tensor_activations = x_scale.numel() == 1 + # When the number of token is 1, + # per-token scale has shape (1, 1), per-tensor scale has shape (1) or (). + per_tensor_activations = (x_scale.numel() == 1) and x_scale.dim() < 2 if ( use_per_token_if_dynamic