[VLM] Support apply qk norm in multi cuda streams (#15720)
Co-authored-by: luoyuan.luo <luoyuan.luo@antgroup.com>
This commit is contained in:
@@ -22,6 +22,10 @@ from sglang.srt.utils import (
|
||||
is_npu,
|
||||
print_info_once,
|
||||
)
|
||||
from sglang.srt.utils.multi_stream_utils import (
|
||||
maybe_execute_in_parallel,
|
||||
with_multi_stream,
|
||||
)
|
||||
|
||||
_is_cuda = is_cuda()
|
||||
_is_npu = is_npu()
|
||||
@@ -532,6 +536,7 @@ class VisionAttention(nn.Module):
|
||||
[torch.Tensor, torch.Tensor, Any, Any], Tuple[torch.Tensor, torch.Tensor]
|
||||
] = None,
|
||||
use_data_parallel: bool = False,
|
||||
aux_stream: Optional[torch.cuda.Stream] = None,
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__()
|
||||
@@ -620,6 +625,8 @@ class VisionAttention(nn.Module):
|
||||
tp_size=self.tp_size,
|
||||
prefix=add_prefix("proj", prefix),
|
||||
)
|
||||
self.aux_stream = aux_stream
|
||||
self.ln_events = [torch.cuda.Event(), torch.cuda.Event()]
|
||||
|
||||
def _determine_attention_backend(self, passed_backend: Optional[str]) -> str:
|
||||
"""Decide the multimodal attention backend string.
|
||||
@@ -655,20 +662,41 @@ class VisionAttention(nn.Module):
|
||||
|
||||
def _apply_qk_norm(self, q: torch.Tensor, k: torch.Tensor):
|
||||
"""apply qk norm for internvl vit attn"""
|
||||
q = q.flatten(1, 2)
|
||||
k = k.flatten(1, 2)
|
||||
|
||||
if self.tp_size > 1:
|
||||
q = tensor_model_parallel_all_gather(q.contiguous())
|
||||
k = tensor_model_parallel_all_gather(k.contiguous())
|
||||
q = self.q_norm(q)
|
||||
k = self.k_norm(k)
|
||||
if self.tp_size > 1:
|
||||
splitter = partial(split_tensor_along_last_dim, num_partitions=self.tp_size)
|
||||
q = splitter(q)[self.tp_rank]
|
||||
k = splitter(k)[self.tp_rank]
|
||||
q = q.unflatten(-1, (-1, self.head_size))
|
||||
k = k.unflatten(-1, (-1, self.head_size))
|
||||
def q_l2norm():
|
||||
q_ = q.flatten(1, 2)
|
||||
if self.tp_size > 1:
|
||||
q_ = tensor_model_parallel_all_gather(q_.contiguous())
|
||||
q_ = self.q_norm(q_)
|
||||
if self.tp_size > 1:
|
||||
splitter = partial(
|
||||
split_tensor_along_last_dim, num_partitions=self.tp_size
|
||||
)
|
||||
q_ = splitter(q_)[self.tp_rank]
|
||||
q_ = q_.unflatten(-1, (-1, self.head_size))
|
||||
return q_
|
||||
|
||||
def k_l2norm():
|
||||
k_ = k.flatten(1, 2)
|
||||
if self.tp_size > 1:
|
||||
k_ = tensor_model_parallel_all_gather(k_.contiguous())
|
||||
k_ = self.k_norm(k_)
|
||||
if self.tp_size > 1:
|
||||
splitter = partial(
|
||||
split_tensor_along_last_dim, num_partitions=self.tp_size
|
||||
)
|
||||
k_ = splitter(k_)[self.tp_rank]
|
||||
k_ = k_.unflatten(-1, (-1, self.head_size))
|
||||
return k_
|
||||
|
||||
with with_multi_stream(True):
|
||||
q, k = maybe_execute_in_parallel(
|
||||
q_l2norm,
|
||||
k_l2norm,
|
||||
self.ln_events[0],
|
||||
self.ln_events[1],
|
||||
self.aux_stream,
|
||||
)
|
||||
return q, k
|
||||
|
||||
def forward(
|
||||
|
||||
Reference in New Issue
Block a user