[VLM] Support cos sin cache for Qwen3-VL & GLM-4.1V (#15205)

Co-authored-by: luoyuan.luo <luoyuan.luo@antgroup.com>
This commit is contained in:
Yuan Luo
2025-12-18 08:39:50 +08:00
committed by GitHub
co-authored by luoyuan.luo
parent d20699a33c
commit 8fa3dc36c5
4 changed files with 100 additions and 80 deletions
+20 -10
View File
@@ -675,6 +675,8 @@ class VisionAttention(nn.Module):
x: torch.Tensor,
cu_seqlens: Optional[torch.Tensor] = None,
position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
rotary_pos_emb_cos: Optional[torch.Tensor] = None,
rotary_pos_emb_sin: Optional[torch.Tensor] = None,
attention_mask: Optional[torch.Tensor] = None,
**kwargs,
) -> torch.Tensor:
@@ -724,26 +726,34 @@ class VisionAttention(nn.Module):
rearrange(x, "s b ... -> b s ...").contiguous() for x in (q, k, v)
]
if position_embeddings is not None:
original_shape = q.shape
cos = None
sin = None
if position_embeddings is not None:
if self.customized_position_embedding_applier is not None:
q, k = self.customized_position_embedding_applier(
q, k, position_embeddings, x_shape
)
q = q.view(original_shape)
k = k.view(original_shape)
else:
cos, sin = position_embeddings
elif rotary_pos_emb_cos is not None and rotary_pos_emb_sin is not None:
cos = rotary_pos_emb_cos
sin = rotary_pos_emb_sin
# [total_tokens, head, head_size]
q = q.view(-1, head, self.head_size)
k = k.view(-1, head, self.head_size)
if cos is not None and sin is not None:
original_shape = q.shape
q, k = apply_rotary_pos_emb(q, k, cos, sin)
# [total_tokens, head, head_size]
q = q.view(-1, head, self.head_size)
k = k.view(-1, head, self.head_size)
q = q.view(original_shape)
k = k.view(original_shape)
if cos.size(-1) * 2 == self.head_size:
cos = torch.cat([cos, cos], dim=-1)
sin = torch.cat([sin, sin], dim=-1)
q, k = apply_rotary_pos_emb(q, k, cos, sin)
q = q.view(original_shape)
k = k.view(original_shape)
if q.dim() == 4:
# [b, s, head, head_size] --> [b * s, head, head_size]