[Ascend] optimize Qwen-vl on Ascend (#10556)
Co-authored-by: wangqihui01 <wangqh10@163.com>
This commit is contained in:
co-authored by
wangqihui01
parent
d27a6f7092
commit
095093ee5a
@@ -16,14 +16,19 @@ from sglang.srt.utils import (
|
||||
get_device_capability,
|
||||
is_blackwell,
|
||||
is_cuda,
|
||||
is_npu,
|
||||
print_info_once,
|
||||
)
|
||||
|
||||
_is_cuda = is_cuda()
|
||||
_is_npu = is_npu()
|
||||
|
||||
if _is_cuda:
|
||||
from sgl_kernel.flash_attn import flash_attn_varlen_func
|
||||
|
||||
if _is_npu:
|
||||
import torch_npu
|
||||
|
||||
from sglang.srt.distributed import (
|
||||
split_tensor_along_last_dim,
|
||||
tensor_model_parallel_all_gather,
|
||||
@@ -331,10 +336,63 @@ class VisionFlash3Attention(nn.Module):
|
||||
return output
|
||||
|
||||
|
||||
class VisionAscendAttention(nn.Module):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
**kwargs,
|
||||
):
|
||||
if not _is_npu:
|
||||
raise Exception("VisionAscendAttention is only available for ascend npu")
|
||||
super().__init__()
|
||||
|
||||
def forward(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
cu_seqlens: Optional[Union[SingletonCache, torch.Tensor]],
|
||||
bsz: int,
|
||||
seq_len: int,
|
||||
**kwargs,
|
||||
) -> torch.Tensor:
|
||||
r"""
|
||||
Args:
|
||||
cu_seqlens: [b]
|
||||
Returns:
|
||||
[b * s, h, head_size]
|
||||
"""
|
||||
if cu_seqlens is None:
|
||||
cu_seqlens = _get_cu_seqlens_for_shape(bsz, seq_len, device=q.device)
|
||||
|
||||
seq_lens = cu_seqlens[1:] - cu_seqlens[:-1]
|
||||
if seq_lens.is_npu:
|
||||
# cu_seqlens must be on cpu because of operator restriction
|
||||
seq_lens = seq_lens.to("cpu")
|
||||
_, num_heads, head_size = q.shape
|
||||
num_kv_heads = k.shape[1]
|
||||
output = torch.empty_like(q)
|
||||
|
||||
# operator requires pta version >= 2.5.1
|
||||
torch_npu._npu_flash_attention_unpad(
|
||||
query=q,
|
||||
key=k,
|
||||
value=v,
|
||||
seq_len=seq_lens.to(torch.int32),
|
||||
scale_value=head_size**-0.5,
|
||||
num_heads=num_heads,
|
||||
num_kv_heads=num_kv_heads,
|
||||
out=output,
|
||||
)
|
||||
|
||||
return output
|
||||
|
||||
|
||||
QKV_BACKEND_IMPL = {
|
||||
"triton_attn": VisionTritonAttention,
|
||||
"sdpa": VisionSdpaAttention,
|
||||
"fa3": VisionFlash3Attention,
|
||||
"ascend_attn": VisionAscendAttention,
|
||||
}
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user