[Performance] Optimze the performance of Qwen25VL (#15640)
Co-authored-by: Xinyuan Tong <115166877+JustinTong0323@users.noreply.github.com>
This commit is contained in:
co-authored by
Xinyuan Tong
parent
5b4f790200
commit
9a414b164c
@@ -16,6 +16,7 @@ from sglang.srt.server_args import get_global_server_args
|
||||
from sglang.srt.utils import (
|
||||
cpu_has_amx_support,
|
||||
get_bool_env_var,
|
||||
get_compiler_backend,
|
||||
is_cpu,
|
||||
is_cuda,
|
||||
is_hip,
|
||||
@@ -2748,6 +2749,7 @@ def rotate_half(x):
|
||||
return torch.cat((-x2, x1), dim=-1)
|
||||
|
||||
|
||||
@torch.compile(dynamic=True, backend=get_compiler_backend())
|
||||
def apply_rotary_pos_emb_native(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
|
||||
Reference in New Issue
Block a user