[piecewise] Refactor VLM to support input embed buffer and remove external embedder hack (#14155)

python/sglang/srt/models/qwen2_5_vl.py

@@ -57,7 +57,10 @@ from sglang.srt.layers.pooler import Pooler, PoolingType
 from sglang.srt.layers.quantization.base_config import QuantizationConfig
 from sglang.srt.layers.utils import PPMissingLayer, get_layer_id
 from sglang.srt.layers.vocab_parallel_embedding import ParallelLMHead
-from sglang.srt.managers.mm_utils import MultiModalityDataPaddingPatternMultimodalTokens
+from sglang.srt.managers.mm_utils import (
+    MultiModalityDataPaddingPatternMultimodalTokens,
+    general_mm_embed_routine,
+)
 from sglang.srt.managers.schedule_batch import (
     Modality,
     MultimodalDataItem,
@@ -624,21 +627,11 @@ class Qwen2_5_VLForConditionalGeneration(nn.Module):
                     f"(3, seq_len) positions, but got {positions.size()}"
                 )
 
-        input_embeds = forward_batch.input_embeds
-        # It may seem strange to assign input_embeds again even after passing it as an argument.
-        # This is for compatibility considerations.
-        # In the 'extend' scenario, this forward function is called from two places:
-        # 1. model_runner calls forward directly,
-        # 2. piece_wise_cuda_graph_runner calls forward and replay.
-
-        # Currently,
-        # In 'extend', input_embeds is passed in.
-        # In 'decode', input_ids is passed in.
-
-        hidden_states = self.model(
+        hidden_states = general_mm_embed_routine(
             input_ids=input_ids,
             forward_batch=forward_batch,
-            input_embeds=input_embeds,
+            language_model=self.model,
+            multimodal_model=self,
             positions=positions,
             pp_proxy_tensors=pp_proxy_tensors,
         )