[VLM] Support VLM ViT Piecewise CUDA Graph (#14422)
Co-authored-by: luoyuan.luo <luoyuan.luo@antgroup.com> Co-authored-by: kousakawang <wanghanpei@bytedance.com>
This commit is contained in:
co-authored by
luoyuan.luo
kousakawang
parent
f1bbd26ff7
commit
3b8a824b8b
@@ -51,7 +51,7 @@ from sglang.srt.layers.linear import (
|
||||
from sglang.srt.layers.quantization import QuantizationConfig
|
||||
from sglang.srt.layers.rotary_embedding import apply_rotary_pos_emb
|
||||
from sglang.srt.server_args import get_global_server_args
|
||||
from sglang.srt.utils import add_prefix
|
||||
from sglang.srt.utils import add_prefix, get_bool_env_var
|
||||
|
||||
ROTARY_EMBED_CLASSES = {
|
||||
"normal": apply_rotary_pos_emb,
|
||||
@@ -273,6 +273,10 @@ class VisionTritonAttention(nn.Module):
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__()
|
||||
use_data_parallel = (
|
||||
kwargs["use_data_parallel"] if "use_data_parallel" in kwargs else False
|
||||
)
|
||||
self.tp_size = 1 if use_data_parallel else get_attention_tp_size()
|
||||
|
||||
def forward(
|
||||
self,
|
||||
@@ -290,22 +294,42 @@ class VisionTritonAttention(nn.Module):
|
||||
Returns:
|
||||
[b * s, h, head_size]
|
||||
"""
|
||||
cu_seqlens = resolve_seqlens(cu_seqlens, bsz, seq_len, device=q.device)
|
||||
if get_bool_env_var("SGLANG_VIT_ENABLE_CUDA_GRAPH") and self.tp_size == 1:
|
||||
if "output_ws" not in kwargs:
|
||||
raise RuntimeError("output_ws should be prepared for cuda-graph mode")
|
||||
|
||||
# [b * s, head, head_size]
|
||||
output = torch.empty_like(q)
|
||||
seq_lens = cu_seqlens[1:] - cu_seqlens[:-1]
|
||||
max_seqlen = seq_lens.max().item()
|
||||
context_attention_fwd(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
output,
|
||||
cu_seqlens.cuda(),
|
||||
seq_lens.cuda(),
|
||||
max_seqlen,
|
||||
is_causal=False,
|
||||
)
|
||||
if not isinstance(cu_seqlens, list):
|
||||
raise RuntimeError("cuda-graph mode cu_seqlens should be a list")
|
||||
|
||||
output = kwargs["output_ws"]
|
||||
context_attention_fwd(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
output,
|
||||
cu_seqlens[0],
|
||||
cu_seqlens[1],
|
||||
cu_seqlens[2],
|
||||
is_causal=False,
|
||||
)
|
||||
else:
|
||||
cu_seqlens = resolve_seqlens(cu_seqlens, bsz, seq_len, device=q.device)
|
||||
|
||||
# [b * s, head, head_size]
|
||||
output = torch.empty_like(q)
|
||||
|
||||
seq_lens = cu_seqlens[1:] - cu_seqlens[:-1]
|
||||
max_seqlen = seq_lens.max().item()
|
||||
context_attention_fwd(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
output,
|
||||
cu_seqlens.cuda(),
|
||||
seq_lens.cuda(),
|
||||
max_seqlen,
|
||||
is_causal=False,
|
||||
)
|
||||
|
||||
return output
|
||||
|
||||
@@ -318,6 +342,10 @@ class VisionFlash3Attention(nn.Module):
|
||||
if not _is_cuda:
|
||||
raise Exception("VisionFlash3Attention is only available for cuda")
|
||||
super().__init__()
|
||||
use_data_parallel = (
|
||||
kwargs["use_data_parallel"] if "use_data_parallel" in kwargs else False
|
||||
)
|
||||
self.tp_size = 1 if use_data_parallel else get_attention_tp_size()
|
||||
|
||||
def forward(
|
||||
self,
|
||||
@@ -335,21 +363,32 @@ class VisionFlash3Attention(nn.Module):
|
||||
Returns:
|
||||
[b * s, h, head_size]
|
||||
"""
|
||||
cu_seqlens = resolve_seqlens(cu_seqlens, bsz, seq_len, device=q.device)
|
||||
if get_bool_env_var("SGLANG_VIT_ENABLE_CUDA_GRAPH") and self.tp_size == 1:
|
||||
max_seqlen = cu_seqlens[1]
|
||||
output = flash_attn_varlen_func(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
cu_seqlens_q=cu_seqlens[0],
|
||||
cu_seqlens_k=cu_seqlens[0],
|
||||
max_seqlen_q=max_seqlen,
|
||||
max_seqlen_k=max_seqlen,
|
||||
)
|
||||
else:
|
||||
cu_seqlens = resolve_seqlens(cu_seqlens, bsz, seq_len, device=q.device)
|
||||
cu_seqlens = cu_seqlens.to(dtype=torch.int32).to(q.device)
|
||||
seq_lens = cu_seqlens[1:] - cu_seqlens[:-1]
|
||||
max_seqlen = seq_lens.max().item()
|
||||
|
||||
cu_seqlens = cu_seqlens.to(dtype=torch.int32).to(q.device)
|
||||
seq_lens = cu_seqlens[1:] - cu_seqlens[:-1]
|
||||
max_seqlen = seq_lens.max().item()
|
||||
|
||||
output = flash_attn_varlen_func(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
cu_seqlens_q=cu_seqlens,
|
||||
cu_seqlens_k=cu_seqlens,
|
||||
max_seqlen_q=max_seqlen,
|
||||
max_seqlen_k=max_seqlen,
|
||||
)
|
||||
output = flash_attn_varlen_func(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
cu_seqlens_q=cu_seqlens,
|
||||
cu_seqlens_k=cu_seqlens,
|
||||
max_seqlen_q=max_seqlen,
|
||||
max_seqlen_k=max_seqlen,
|
||||
)
|
||||
|
||||
return output
|
||||
|
||||
@@ -545,6 +584,7 @@ class VisionAttention(nn.Module):
|
||||
dropout=dropout,
|
||||
flatten_batch=flatten_batch,
|
||||
softmax_in_single_precision=softmax_in_single_precision,
|
||||
use_data_parallel=use_data_parallel,
|
||||
)
|
||||
|
||||
self.use_qkv_parallel = use_qkv_parallel
|
||||
@@ -652,6 +692,8 @@ class VisionAttention(nn.Module):
|
||||
bsz, s, _ = x_shape
|
||||
head = self.num_attention_heads_per_partition
|
||||
kv_head = self.num_attention_kv_heads_per_partition
|
||||
|
||||
attn_output_ws = kwargs["output_ws"] if "output_ws" in kwargs else None
|
||||
if self.use_qkv_parallel:
|
||||
# [b, s, embed_dim] --> [b, s, embed_dim]
|
||||
qkv, _ = self.qkv_proj(x)
|
||||
@@ -729,6 +771,7 @@ class VisionAttention(nn.Module):
|
||||
seq_len=s,
|
||||
cu_seqlens=cu_seqlens,
|
||||
attention_mask=attention_mask,
|
||||
output_ws=attn_output_ws,
|
||||
)
|
||||
|
||||
assert output.dim() == 3, output.shape
|
||||
|
||||
Reference in New Issue
Block a user