model: adapt mllama4 to VisionAttention (#8512)
Co-authored-by: root <mickjagger19@icloud.com>
This commit is contained in:
@@ -4,7 +4,7 @@ import dataclasses
|
||||
import functools
|
||||
import math
|
||||
from functools import lru_cache, partial
|
||||
from typing import Any, Optional, Tuple, Union
|
||||
from typing import Any, Callable, Optional, Tuple, Union
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
@@ -308,6 +308,7 @@ class VisionFlash3Attention(nn.Module):
|
||||
cu_seqlens = cu_seqlens.to(dtype=torch.int32).to(q.device)
|
||||
seq_lens = cu_seqlens[1:] - cu_seqlens[:-1]
|
||||
max_seqlen = seq_lens.max().item()
|
||||
|
||||
output = flash_attn_varlen_func(
|
||||
q,
|
||||
k,
|
||||
@@ -358,6 +359,9 @@ class VisionAttention(nn.Module):
|
||||
qkv_bias: bool = True,
|
||||
qk_normalization: bool = False,
|
||||
layer_norm_eps: float = 1e-06,
|
||||
customized_position_embedding_applier: Callable[
|
||||
[torch.Tensor, torch.Tensor, Any, Any], Tuple[torch.Tensor, torch.Tensor]
|
||||
] = None,
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__()
|
||||
@@ -392,6 +396,7 @@ class VisionAttention(nn.Module):
|
||||
self.dummy_dim, eps=layer_norm_eps, var_hidden_size=embed_dim
|
||||
)
|
||||
|
||||
# priority: server_args > passed qkv_backend > sdpa
|
||||
if global_server_args_dict["mm_attention_backend"] is None:
|
||||
if qkv_backend is None:
|
||||
qkv_backend = "sdpa"
|
||||
@@ -401,6 +406,9 @@ class VisionAttention(nn.Module):
|
||||
|
||||
print_info_once(f"Using {qkv_backend} as multimodal attention backend.")
|
||||
|
||||
self.customized_position_embedding_applier = (
|
||||
customized_position_embedding_applier
|
||||
)
|
||||
self.qkv_backend = QKV_BACKEND_IMPL[qkv_backend](
|
||||
head_dim=self.head_size,
|
||||
num_heads=self.num_attention_heads_per_partition,
|
||||
@@ -473,13 +481,13 @@ class VisionAttention(nn.Module):
|
||||
if x.dim() == 2:
|
||||
x = x.unsqueeze(0)
|
||||
assert x.dim() == 3, x.shape
|
||||
bsz, s, _ = x.shape
|
||||
x_shape = x.shape
|
||||
bsz, s, _ = x_shape
|
||||
head = self.num_attention_heads_per_partition
|
||||
kv_head = self.num_attention_kv_heads_per_partition
|
||||
if self.use_qkv_parallel:
|
||||
# [b, s, embed_dim] --> [b, s, embed_dim]
|
||||
qkv, _ = self.qkv_proj(x)
|
||||
|
||||
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
|
||||
|
||||
# [b, s, embed_dim] --> [b * s, head, head_size]
|
||||
@@ -508,16 +516,25 @@ class VisionAttention(nn.Module):
|
||||
]
|
||||
|
||||
if position_embeddings is not None:
|
||||
cos, sin = position_embeddings
|
||||
original_shape = q.shape
|
||||
# [total_tokens, head, head_size]
|
||||
q = q.view(-1, head, self.head_size)
|
||||
k = k.view(-1, head, self.head_size)
|
||||
|
||||
q, k = apply_rotary_pos_emb(q, k, cos, sin)
|
||||
if self.customized_position_embedding_applier is not None:
|
||||
q, k = self.customized_position_embedding_applier(
|
||||
q, k, position_embeddings, x_shape
|
||||
)
|
||||
q = q.view(original_shape)
|
||||
k = k.view(original_shape)
|
||||
else:
|
||||
cos, sin = position_embeddings
|
||||
|
||||
q = q.view(original_shape)
|
||||
k = k.view(original_shape)
|
||||
# [total_tokens, head, head_size]
|
||||
q = q.view(-1, head, self.head_size)
|
||||
k = k.view(-1, head, self.head_size)
|
||||
|
||||
q, k = apply_rotary_pos_emb(q, k, cos, sin)
|
||||
|
||||
q = q.view(original_shape)
|
||||
k = k.view(original_shape)
|
||||
|
||||
if q.dim() == 4:
|
||||
# [b, s, head, head_size] --> [b * s, head, head_size]
|
||||
|
||||
Reference in New Issue
Block a user