chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[bugfix][quark] Fixed an issue where per_token could not be properly recognized when the token count was 1. (#14415)

Browse Source
This commit is contained in:
haoyangli-amd
2025-12-16 14:54:31 +08:00
committed by GitHub
parent 36fcf71fff
commit 9327482baa
1 changed files with 3 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
python/sglang/srt/layers/quantization/fp8_utils.py
View File

@@ -992,7 +992,9 @@ def apply_fp8_linear(
# torch.scaled_mm supports per tensor weights + activations only
# so fallback to naive if per channel or per token
per_tensor_weights = weight_scale.numel() == 1
per_tensor_activations = x_scale.numel() == 1
# When the number of token is 1,
# per-token scale has shape (1, 1), per-tensor scale has shape (1) or ().
per_tensor_activations = (x_scale.numel() == 1) and x_scale.dim() < 2
if (
use_per_token_if_dynamic