[VLM][feat] Support encoder DP for Qwen2.5-VL (#13126)
Co-authored-by: Shangming Cai <csmthu@gmail.com> Co-authored-by: liusy58 <xiehang.lsy@alibaba-inc.com> Co-authored-by: Yuan Luo <yuan.luo@hotmail.com>
This commit is contained in:
co-authored by
Shangming Cai
liusy58
Yuan Luo
parent
33905005ee
commit
ac81db66c2
@@ -486,13 +486,12 @@ class VisionAttention(nn.Module):
|
||||
customized_position_embedding_applier: Callable[
|
||||
[torch.Tensor, torch.Tensor, Any, Any], Tuple[torch.Tensor, torch.Tensor]
|
||||
] = None,
|
||||
use_data_parallel: bool = False,
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__()
|
||||
attn_tp_rank = get_attention_tp_rank()
|
||||
attn_tp_size = get_attention_tp_size()
|
||||
self.tp_size = attn_tp_size
|
||||
self.tp_rank = attn_tp_rank
|
||||
self.tp_size = 1 if use_data_parallel else get_attention_tp_size()
|
||||
self.tp_rank = 0 if use_data_parallel else get_attention_tp_rank()
|
||||
self.dropout = dropout
|
||||
self.head_size = embed_dim // num_heads
|
||||
self.hidden_size_per_attention_head = dist_utils.divide(
|
||||
|
||||
Reference in New Issue
Block a user