VLM: enhance VL embedding model with video input support and revise warm-up strategy (#16635)

Co-authored-by: Mick <mickjagger19@icloud.com>

python/sglang/srt/entrypoints/http_server.py

@@ -1514,8 +1514,13 @@ def _execute_server_warmup(server_args: ServerArgs):
         # TODO Workaround the bug that embedding errors for list of size 1
         if server_args.dp_size == 1:
             json_data["input_ids"] = json_data["input_ids"][0]
-    elif is_vlm and server_args.disaggregation_mode == "null":
+    elif (
+        is_vlm
+        and server_args.disaggregation_mode == "null"
+        and model_info["is_generation"]
+    ):
         # TODO: ChatCompletionRequest does not have bootstrap info required by disaggregation mode, disable image-warmup for now
+        # Only use chat completions format for generation models, not embedding models
         json_data = {
             "model": _global_state.tokenizer_manager.served_model_name,
             "messages": [

python/sglang/srt/entrypoints/openai/protocol.py

@@ -780,6 +780,7 @@ class ChatCompletionStreamResponse(BaseModel):
 class MultimodalEmbeddingInput(BaseModel):
     text: Optional[str] = None
     image: Optional[str] = None
+    video: Optional[str] = None
 
 
 EmbeddingInput = Union[

python/sglang/srt/entrypoints/openai/serving_embedding.py

@@ -89,16 +89,18 @@ class OpenAIServingEmbedding(OpenAIServingBase):
                 # Handle multimodal embedding inputs
                 texts = []
                 images = []
+                videos = []
                 for item in prompt:
                     # Use padding for text if None - this could be improved
                     texts.append(item.text if item.text is not None else "padding")
                     images.append(item.image if item.image is not None else None)
+                    videos.append(item.video if item.video is not None else None)
 
                 generate_prompts = []
                 # Check if we have a chat template for multimodal embeddings
                 if self.template_manager.chat_template_name is not None:
                     convs = generate_embedding_convs(
-                        texts, images, self.template_manager.chat_template_name
+                        texts, images, videos, self.template_manager.chat_template_name
                     )
                     for conv in convs:
                         generate_prompts.append(conv.get_prompt())
@@ -109,11 +111,13 @@ class OpenAIServingEmbedding(OpenAIServingBase):
                     prompt_kwargs = {
                         "text": generate_prompts[0],
                         "image_data": images[0],
+                        "video_data": videos[0],
                     }
                 else:
                     prompt_kwargs = {
                         "text": generate_prompts,
                         "image_data": images,
+                        "video_data": videos,
                     }
             else:
                 # List of integers (token IDs) or empty list