[feature] enable pre compile jit deep_gemm (#5580)
This commit is contained in:
@@ -12,8 +12,8 @@ try:
|
||||
except ImportError:
|
||||
VLLM_AVAILABLE = False
|
||||
|
||||
from sglang.srt.layers.quantization.deep_gemm import _ENABLE_JIT_DEEPGEMM
|
||||
from sglang.srt.layers.quantization.fp8_kernel import (
|
||||
_enable_jit_deepgemm,
|
||||
per_token_group_quant_fp8,
|
||||
scaled_fp8_quant,
|
||||
sglang_per_token_quant_fp8,
|
||||
@@ -143,7 +143,7 @@ def apply_w8a8_block_fp8_linear(
|
||||
)
|
||||
gemm_a8w8_blockscale(q_input, weight, x_scale, weight_scale, output)
|
||||
else:
|
||||
if _enable_jit_deepgemm:
|
||||
if _ENABLE_JIT_DEEPGEMM:
|
||||
q_input, x_scale = sglang_per_token_group_quant_fp8(
|
||||
input_2d,
|
||||
block_size[1],
|
||||
|
||||
Reference in New Issue
Block a user