[VLM] feat: true on policy for vlm + fsdp (#14636)

Co-authored-by: Xinyuan Tong <115166877+JustinTong0323@users.noreply.github.com>
This commit is contained in:
Nan Jiang
2026-01-01 16:54:39 -08:00
committed by GitHub
co-authored by Xinyuan Tong
parent 00562ee14a
commit 7254986342
12 changed files with 203 additions and 57 deletions
+45 -10
View File
@@ -32,13 +32,17 @@ from sglang.srt.distributed import (
from sglang.srt.distributed.parallel_state import get_pp_group
from sglang.srt.environ import envs
from sglang.srt.layers.attention.vision import VisionAttention
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
from sglang.srt.layers.linear import ColumnParallelLinear, RowParallelLinear
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.pooler import Pooler, PoolingType
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.layers.rotary_embedding import get_rope
from sglang.srt.layers.utils import PPMissingLayer, get_layer_id
from sglang.srt.layers.vocab_parallel_embedding import ParallelLMHead
from sglang.srt.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from sglang.srt.managers.mm_utils import (
MultiModalityDataPaddingPatternMultimodalTokens,
general_mm_embed_routine,
@@ -278,6 +282,7 @@ class Qwen3VLMoeVisionModel(nn.Module, RotaryPosMixin):
use_data_parallel: bool = False,
) -> None:
super().__init__()
self.pp_group = get_pp_group()
self.hidden_size = vision_config.hidden_size
self.num_heads = vision_config.num_heads
self.num_position_embeddings = vision_config.num_position_embeddings
@@ -297,7 +302,17 @@ class Qwen3VLMoeVisionModel(nn.Module, RotaryPosMixin):
1 + len(self.deepstack_visual_indexes)
)
self.patch_embed = Qwen3VLVisionPatchEmbed(config=vision_config)
self.pos_embed = nn.Embedding(self.num_position_embeddings, self.hidden_size)
if self.pp_group.is_first_rank:
self.pos_embed = VocabParallelEmbedding(
self.num_position_embeddings,
self.hidden_size,
quant_config=quant_config,
enable_tp=not is_dp_attention_enabled(),
prefix=add_prefix("pos_embed", prefix),
)
else:
self.pos_embed = PPMissingLayer()
norm_layer = partial(nn.LayerNorm, eps=norm_eps)
head_dim = self.hidden_size // self.num_heads
self.rotary_pos_emb = get_rope(
@@ -549,6 +564,18 @@ class Qwen3LLMModel(Qwen3Model):
len(config.vision_config.deepstack_visual_indexes)
)
def get_deepstack_embeds(
self, layer_idx: int, input_deepstack_embeds: Optional[torch.Tensor]
) -> Optional[torch.Tensor]:
"""Get deepstack embeddings for a given layer index, or None if not applicable."""
if (
input_deepstack_embeds is None
or layer_idx not in self.deepstack_embed_to_decoder_layer
):
return None
sep = self.hidden_size * layer_idx
return input_deepstack_embeds[:, sep : sep + self.hidden_size]
def forward(
self,
input_ids: torch.Tensor,
@@ -580,20 +607,26 @@ class Qwen3LLMModel(Qwen3Model):
hidden_states + residual if residual is not None else hidden_states
)
# SGLang applies residual at the START of the next layer, not at the END like HuggingFace.
# See: https://github.com/huggingface/transformers/blob/v5.0.0rc0/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py#L549
# To match HF behavior, deepstack must be added AFTER residual: (hidden_states + residual) + deepstack
# The order matters because addition with different tensors is not associative in practice.
# Deepstack for prev_layer is applied at the start of current layer via post_residual_addition.
deepstack_embeds = self.get_deepstack_embeds(
layer_idx - 1, input_deepstack_embeds
)
hidden_states, residual = layer(
positions,
hidden_states,
forward_batch,
residual,
post_residual_addition=deepstack_embeds,
)
# process deepstack
if (
input_deepstack_embeds is not None
and layer_idx in self.deepstack_embed_to_decoder_layer
):
sep = self.hidden_size * layer_idx
hidden_states += input_deepstack_embeds[:, sep : sep + self.hidden_size]
# Handle deepstack for the last processed layer if it exists.
last_deepstack = self.get_deepstack_embeds(
self.end_layer - 1, input_deepstack_embeds
)
if not self.pp_group.is_last_rank:
return PPProxyTensors(
@@ -607,7 +640,9 @@ class Qwen3LLMModel(Qwen3Model):
if residual is None:
hidden_states = self.norm(hidden_states)
else:
hidden_states, _ = self.norm(hidden_states, residual)
hidden_states, _ = self.norm(
hidden_states, residual, post_residual_addition=last_deepstack
)
if len(aux_hidden_states) == 0:
return hidden_states