chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

VLM: enhance VL embedding model with video input support and revise warm-up strategy (#16635)

Browse Source 
Co-authored-by: Mick <mickjagger19@icloud.com>
This commit is contained in:
yuhao
2026-01-08 22:12:01 +08:00
committed by GitHub
parent 20ca2c6e1e
commit d2ea44f775
5 changed files with 18 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
python/sglang/srt/entrypoints/openai/serving_embedding.py
View File

@@ -89,16 +89,18 @@ class OpenAIServingEmbedding(OpenAIServingBase):
# Handle multimodal embedding inputs
texts = []
images = []
videos = []
for item in prompt:
# Use padding for text if None - this could be improved
texts.append(item.text if item.text is not None else "padding")
images.append(item.image if item.image is not None else None)
videos.append(item.video if item.video is not None else None)
generate_prompts = []
# Check if we have a chat template for multimodal embeddings
if self.template_manager.chat_template_name is not None:
convs = generate_embedding_convs(
texts, images, self.template_manager.chat_template_name
texts, images, videos, self.template_manager.chat_template_name
)
for conv in convs:
generate_prompts.append(conv.get_prompt())
@@ -109,11 +111,13 @@ class OpenAIServingEmbedding(OpenAIServingBase):
prompt_kwargs = {
"text": generate_prompts[0],
"image_data": images[0],
"video_data": videos[0],
}
else:
prompt_kwargs = {
"text": generate_prompts,
"image_data": images,
"video_data": videos,
}
else:
# List of integers (token IDs) or empty list