From 59cce5941a0fd3aad8edbfa46616c7cbda6a52cc Mon Sep 17 00:00:00 2001 From: Qiaolin Yu Date: Fri, 31 Oct 2025 19:51:28 -0700 Subject: [PATCH] Use sgl fp4 quant kernel by default (#12482) --- python/sglang/srt/layers/quantization/modelopt_quant.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/python/sglang/srt/layers/quantization/modelopt_quant.py b/python/sglang/srt/layers/quantization/modelopt_quant.py index 40fd2bb5b..7861b878a 100755 --- a/python/sglang/srt/layers/quantization/modelopt_quant.py +++ b/python/sglang/srt/layers/quantization/modelopt_quant.py @@ -7,6 +7,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional import torch from torch.nn.parameter import Parameter +from python.sglang.srt.utils.common import is_sm120_supported from sglang.srt.distributed import get_tp_group from sglang.srt.layers.dp_attention import get_dp_global_num_tokens, get_local_dp_buffer from sglang.srt.layers.moe import ( @@ -51,7 +52,10 @@ if TYPE_CHECKING: from sglang.srt.single_batch_overlap import DownGemmOverlapArgs try: - from flashinfer import fp4_quantize + if is_sm120_supported(): + from flashinfer import fp4_quantize + else: + from sgl_kernel import scaled_fp4_quant as fp4_quantize except ImportError: fp4_quantize = None