From d2ea44f7756e243ac6f603357496fd2d6a1d29fd Mon Sep 17 00:00:00 2001 From: yuhao <1160289632@qq.com> Date: Thu, 8 Jan 2026 22:12:01 +0800 Subject: [PATCH] VLM: enhance VL embedding model with video input support and revise warm-up strategy (#16635) Co-authored-by: Mick --- python/sglang/srt/entrypoints/http_server.py | 7 ++++++- python/sglang/srt/entrypoints/openai/protocol.py | 1 + python/sglang/srt/entrypoints/openai/serving_embedding.py | 6 +++++- python/sglang/srt/managers/tokenizer_manager.py | 2 ++ python/sglang/srt/parser/conversation.py | 6 ++++-- 5 files changed, 18 insertions(+), 4 deletions(-) diff --git a/python/sglang/srt/entrypoints/http_server.py b/python/sglang/srt/entrypoints/http_server.py index 88705cc35..1b6b55d88 100644 --- a/python/sglang/srt/entrypoints/http_server.py +++ b/python/sglang/srt/entrypoints/http_server.py @@ -1514,8 +1514,13 @@ def _execute_server_warmup(server_args: ServerArgs): # TODO Workaround the bug that embedding errors for list of size 1 if server_args.dp_size == 1: json_data["input_ids"] = json_data["input_ids"][0] - elif is_vlm and server_args.disaggregation_mode == "null": + elif ( + is_vlm + and server_args.disaggregation_mode == "null" + and model_info["is_generation"] + ): # TODO: ChatCompletionRequest does not have bootstrap info required by disaggregation mode, disable image-warmup for now + # Only use chat completions format for generation models, not embedding models json_data = { "model": _global_state.tokenizer_manager.served_model_name, "messages": [ diff --git a/python/sglang/srt/entrypoints/openai/protocol.py b/python/sglang/srt/entrypoints/openai/protocol.py index 14cc9676b..2204b66db 100644 --- a/python/sglang/srt/entrypoints/openai/protocol.py +++ b/python/sglang/srt/entrypoints/openai/protocol.py @@ -780,6 +780,7 @@ class ChatCompletionStreamResponse(BaseModel): class MultimodalEmbeddingInput(BaseModel): text: Optional[str] = None image: Optional[str] = None + video: Optional[str] = None EmbeddingInput = Union[ diff --git a/python/sglang/srt/entrypoints/openai/serving_embedding.py b/python/sglang/srt/entrypoints/openai/serving_embedding.py index 08e48ddd4..223d6c3e6 100644 --- a/python/sglang/srt/entrypoints/openai/serving_embedding.py +++ b/python/sglang/srt/entrypoints/openai/serving_embedding.py @@ -89,16 +89,18 @@ class OpenAIServingEmbedding(OpenAIServingBase): # Handle multimodal embedding inputs texts = [] images = [] + videos = [] for item in prompt: # Use padding for text if None - this could be improved texts.append(item.text if item.text is not None else "padding") images.append(item.image if item.image is not None else None) + videos.append(item.video if item.video is not None else None) generate_prompts = [] # Check if we have a chat template for multimodal embeddings if self.template_manager.chat_template_name is not None: convs = generate_embedding_convs( - texts, images, self.template_manager.chat_template_name + texts, images, videos, self.template_manager.chat_template_name ) for conv in convs: generate_prompts.append(conv.get_prompt()) @@ -109,11 +111,13 @@ class OpenAIServingEmbedding(OpenAIServingBase): prompt_kwargs = { "text": generate_prompts[0], "image_data": images[0], + "video_data": videos[0], } else: prompt_kwargs = { "text": generate_prompts, "image_data": images, + "video_data": videos, } else: # List of integers (token IDs) or empty list diff --git a/python/sglang/srt/managers/tokenizer_manager.py b/python/sglang/srt/managers/tokenizer_manager.py index 45cf37b48..67a3633da 100644 --- a/python/sglang/srt/managers/tokenizer_manager.py +++ b/python/sglang/srt/managers/tokenizer_manager.py @@ -683,6 +683,8 @@ class TokenizerManager(TokenizerCommunicatorMixin, TokenizerManagerMultiItemMixi if self.mm_processor and obj.contains_mm_input(): if obj.image_data is not None and not isinstance(obj.image_data, list): obj.image_data = [obj.image_data] + if obj.video_data is not None and not isinstance(obj.video_data, list): + obj.video_data = [obj.video_data] if obj.audio_data is not None and not isinstance(obj.audio_data, list): obj.audio_data = [obj.audio_data] self._validate_mm_limits(obj) diff --git a/python/sglang/srt/parser/conversation.py b/python/sglang/srt/parser/conversation.py index 17fd93053..8a639b645 100644 --- a/python/sglang/srt/parser/conversation.py +++ b/python/sglang/srt/parser/conversation.py @@ -510,11 +510,11 @@ def chat_template_exists(template_name: str) -> bool: def generate_embedding_convs( - texts: List[str], images: List[str], template_name: str + texts: List[str], images: List[str], videos: List[str], template_name: str ) -> List[Conversation]: conv_template = chat_templates[template_name].copy() convs = [] - for text, image in zip(texts, images): + for text, image, video in zip(texts, images, videos): conv = Conversation( name=conv_template.name, system_template=conv_template.system_template, @@ -544,6 +544,8 @@ def generate_embedding_convs( else conv.image_token ) real_content += image_token + if video is not None: + real_content += conv.video_token if text is not None: real_content += text conv.append_message(conv.roles[0], real_content)