[feature] enable pre compile jit deep_gemm (#5580)

This commit is contained in:
JieXin Liang
2025-04-22 07:52:53 +08:00
committed by GitHub
parent e69a219074
commit c2942907d5
7 changed files with 549 additions and 45 deletions

View File

@@ -12,8 +12,8 @@ try:
except ImportError:
VLLM_AVAILABLE = False
from sglang.srt.layers.quantization.deep_gemm import _ENABLE_JIT_DEEPGEMM
from sglang.srt.layers.quantization.fp8_kernel import (
_enable_jit_deepgemm,
per_token_group_quant_fp8,
scaled_fp8_quant,
sglang_per_token_quant_fp8,
@@ -143,7 +143,7 @@ def apply_w8a8_block_fp8_linear(
)
gemm_a8w8_blockscale(q_input, weight, x_scale, weight_scale, output)
else:
if _enable_jit_deepgemm:
if _ENABLE_JIT_DEEPGEMM:
q_input, x_scale = sglang_per_token_group_quant_fp8(
input_2d,
block_size[1],