[VLM] feat: true on policy for vlm + fsdp (#14636)
Co-authored-by: Xinyuan Tong <115166877+JustinTong0323@users.noreply.github.com>
This commit is contained in:
co-authored by
Xinyuan Tong
parent
00562ee14a
commit
7254986342
@@ -32,13 +32,17 @@ from sglang.srt.distributed import (
|
||||
from sglang.srt.distributed.parallel_state import get_pp_group
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.attention.vision import VisionAttention
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
from sglang.srt.layers.linear import ColumnParallelLinear, RowParallelLinear
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.pooler import Pooler, PoolingType
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.layers.rotary_embedding import get_rope
|
||||
from sglang.srt.layers.utils import PPMissingLayer, get_layer_id
|
||||
from sglang.srt.layers.vocab_parallel_embedding import ParallelLMHead
|
||||
from sglang.srt.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from sglang.srt.managers.mm_utils import (
|
||||
MultiModalityDataPaddingPatternMultimodalTokens,
|
||||
general_mm_embed_routine,
|
||||
@@ -278,6 +282,7 @@ class Qwen3VLMoeVisionModel(nn.Module, RotaryPosMixin):
|
||||
use_data_parallel: bool = False,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.pp_group = get_pp_group()
|
||||
self.hidden_size = vision_config.hidden_size
|
||||
self.num_heads = vision_config.num_heads
|
||||
self.num_position_embeddings = vision_config.num_position_embeddings
|
||||
@@ -297,7 +302,17 @@ class Qwen3VLMoeVisionModel(nn.Module, RotaryPosMixin):
|
||||
1 + len(self.deepstack_visual_indexes)
|
||||
)
|
||||
self.patch_embed = Qwen3VLVisionPatchEmbed(config=vision_config)
|
||||
self.pos_embed = nn.Embedding(self.num_position_embeddings, self.hidden_size)
|
||||
if self.pp_group.is_first_rank:
|
||||
self.pos_embed = VocabParallelEmbedding(
|
||||
self.num_position_embeddings,
|
||||
self.hidden_size,
|
||||
quant_config=quant_config,
|
||||
enable_tp=not is_dp_attention_enabled(),
|
||||
prefix=add_prefix("pos_embed", prefix),
|
||||
)
|
||||
else:
|
||||
self.pos_embed = PPMissingLayer()
|
||||
|
||||
norm_layer = partial(nn.LayerNorm, eps=norm_eps)
|
||||
head_dim = self.hidden_size // self.num_heads
|
||||
self.rotary_pos_emb = get_rope(
|
||||
@@ -549,6 +564,18 @@ class Qwen3LLMModel(Qwen3Model):
|
||||
len(config.vision_config.deepstack_visual_indexes)
|
||||
)
|
||||
|
||||
def get_deepstack_embeds(
|
||||
self, layer_idx: int, input_deepstack_embeds: Optional[torch.Tensor]
|
||||
) -> Optional[torch.Tensor]:
|
||||
"""Get deepstack embeddings for a given layer index, or None if not applicable."""
|
||||
if (
|
||||
input_deepstack_embeds is None
|
||||
or layer_idx not in self.deepstack_embed_to_decoder_layer
|
||||
):
|
||||
return None
|
||||
sep = self.hidden_size * layer_idx
|
||||
return input_deepstack_embeds[:, sep : sep + self.hidden_size]
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
@@ -580,20 +607,26 @@ class Qwen3LLMModel(Qwen3Model):
|
||||
hidden_states + residual if residual is not None else hidden_states
|
||||
)
|
||||
|
||||
# SGLang applies residual at the START of the next layer, not at the END like HuggingFace.
|
||||
# See: https://github.com/huggingface/transformers/blob/v5.0.0rc0/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py#L549
|
||||
# To match HF behavior, deepstack must be added AFTER residual: (hidden_states + residual) + deepstack
|
||||
# The order matters because addition with different tensors is not associative in practice.
|
||||
# Deepstack for prev_layer is applied at the start of current layer via post_residual_addition.
|
||||
deepstack_embeds = self.get_deepstack_embeds(
|
||||
layer_idx - 1, input_deepstack_embeds
|
||||
)
|
||||
hidden_states, residual = layer(
|
||||
positions,
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
residual,
|
||||
post_residual_addition=deepstack_embeds,
|
||||
)
|
||||
|
||||
# process deepstack
|
||||
if (
|
||||
input_deepstack_embeds is not None
|
||||
and layer_idx in self.deepstack_embed_to_decoder_layer
|
||||
):
|
||||
sep = self.hidden_size * layer_idx
|
||||
hidden_states += input_deepstack_embeds[:, sep : sep + self.hidden_size]
|
||||
# Handle deepstack for the last processed layer if it exists.
|
||||
last_deepstack = self.get_deepstack_embeds(
|
||||
self.end_layer - 1, input_deepstack_embeds
|
||||
)
|
||||
|
||||
if not self.pp_group.is_last_rank:
|
||||
return PPProxyTensors(
|
||||
@@ -607,7 +640,9 @@ class Qwen3LLMModel(Qwen3Model):
|
||||
if residual is None:
|
||||
hidden_states = self.norm(hidden_states)
|
||||
else:
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
hidden_states, _ = self.norm(
|
||||
hidden_states, residual, post_residual_addition=last_deepstack
|
||||
)
|
||||
|
||||
if len(aux_hidden_states) == 0:
|
||||
return hidden_states
|
||||
|
||||
Reference in New Issue
Block a user