[VLM] feat: true on policy for vlm + fsdp (#14636)

Co-authored-by: Xinyuan Tong <115166877+JustinTong0323@users.noreply.github.com>
This commit is contained in:
Nan Jiang
2026-01-01 16:54:39 -08:00
committed by GitHub
co-authored by Xinyuan Tong
parent 00562ee14a
commit 7254986342
12 changed files with 203 additions and 57 deletions
+25 -2
View File
@@ -565,11 +565,25 @@ class VisionAttention(nn.Module):
self.dummy_dim = (num_dummy_heads + num_heads) * self.head_size
if self.qk_normalization:
norm_kwargs = (
dict(
weight_dtype=torch.float32,
cast_x_before_out_mul=True,
)
if get_global_server_args().rl_on_policy_target is not None
else {}
)
self.q_norm = RMSNorm(
self.dummy_dim, eps=layer_norm_eps, var_hidden_size=embed_dim
self.dummy_dim,
eps=layer_norm_eps,
var_hidden_size=embed_dim,
**norm_kwargs,
)
self.k_norm = RMSNorm(
self.dummy_dim, eps=layer_norm_eps, var_hidden_size=embed_dim
self.dummy_dim,
eps=layer_norm_eps,
var_hidden_size=embed_dim,
**norm_kwargs,
)
# Select attention backend via a unified method
@@ -720,6 +734,15 @@ class VisionAttention(nn.Module):
if x.dim() == 2:
x = x.unsqueeze(0)
assert x.dim() == 3, x.shape
if (
get_global_server_args().rl_on_policy_target is not None
and position_embeddings is not None
):
assert isinstance(position_embeddings, tuple), (
"expected position_embeddings to be a tuple of two tensors,\n"
f"but got {type(position_embeddings)}, change if needed"
)
position_embeddings = tuple(p.to(x.dtype) for p in position_embeddings)
x_shape = x.shape
bsz, s, _ = x_shape
head = self.num_attention_heads_per_partition