chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 11 Packages Projects Releases Wiki Activity

[vlm][minicpm] support input formats of processor output and embedding (#19614)

Browse Source
This commit is contained in:
Ken J
2026-03-04 09:11:12 -08:00
committed by GitHub
parent c03deb8175
commit 44208d2adf
4 changed files with 251 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
python/sglang/srt/models/minicpmo.py
View File

@@ -44,6 +44,7 @@ from sglang.srt.managers.mm_utils import (
)
from sglang.srt.managers.schedule_batch import (
MultimodalDataItem,
MultimodalInputFormat,
MultimodalInputs,
flatten_nested_list,
)
@@ -1803,6 +1804,10 @@ class MiniCPMO(MiniCPMBaseModel):
return audio_embs
def get_image_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor:
if items and items[0].format == MultimodalInputFormat.PRECOMPUTED_EMBEDDING:
result = torch.cat([item.feature for item in items])
return result.reshape(-1, result.shape[-1])
# list of tensors
pixel_values = flatten_nested_list([item.feature for item in items])
tgt_sizes = torch.stack(

18
python/sglang/srt/models/minicpmv.py
View File

@@ -51,7 +51,11 @@ from sglang.srt.managers.mm_utils import (
MultiModalityDataPaddingPatternTokenPairs,
general_mm_embed_routine,
)
from sglang.srt.managers.schedule_batch import MultimodalDataItem, MultimodalInputs
from sglang.srt.managers.schedule_batch import (
MultimodalDataItem,
MultimodalInputFormat,
MultimodalInputs,
)
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
from sglang.srt.model_loader.utils import set_default_torch_dtype
from sglang.srt.model_loader.weight_utils import default_weight_loader
@@ -939,6 +943,10 @@ class MiniCPMV2_6(MiniCPMBaseModel):
return vision_embedding
def get_image_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor:
if items and items[0].format == MultimodalInputFormat.PRECOMPUTED_EMBEDDING:
result = torch.cat([item.feature for item in items])
return result.reshape(-1, result.shape[-1])
# list of tensors
pixel_values = flatten_nested_list([item.feature for item in items])
tgt_sizes = torch.stack(
@@ -1097,6 +1105,10 @@ class MiniCPMV4_0(MiniCPMBaseModel):
return vision_embedding
def get_image_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor:
if items and items[0].format == MultimodalInputFormat.PRECOMPUTED_EMBEDDING:
result = torch.cat([item.feature for item in items])
return result.reshape(-1, result.shape[-1])
# list of tensors
pixel_values = flatten_nested_list([item.feature for item in items])
tgt_sizes = torch.stack(
@@ -1259,6 +1271,10 @@ class MiniCPMV4_5(MiniCPMBaseModel):
return vision_embedding
def get_image_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor:
if items and items[0].format == MultimodalInputFormat.PRECOMPUTED_EMBEDDING:
result = torch.cat([item.feature for item in items])
return result.reshape(-1, result.shape[-1])
# list of tensors
pixel_values = flatten_nested_list([item.feature for item in items])
tgt_sizes = torch.stack(

148
python/sglang/srt/multimodal/processors/minicpm.py
View File

@@ -2,11 +2,15 @@ from typing import List, Union
import torch
from sglang.srt.managers.schedule_batch import Modality, MultimodalDataItem
from sglang.srt.managers.schedule_batch import (
Modality,
MultimodalDataItem,
)
from sglang.srt.models.minicpmo import MiniCPMO
from sglang.srt.models.minicpmv import MiniCPMV
from sglang.srt.multimodal.processors.base_processor import (
BaseMultimodalProcessor,
BaseMultiModalProcessorOutput,
MultimodalSpecialTokens,
)
@@ -34,6 +38,137 @@ class MiniCPMMultimodalProcessor(BaseMultimodalProcessor):
image_token_id=self.im_token_id,
).build(_processor)
@staticmethod
def _has_special_format(image_data, audio_data):
"""Check if any input items use processor_output or precomputed_embedding format."""
for data in list(image_data or []) + list(audio_data or []):
if isinstance(data, dict) and data.get("format") in (
"processor_output",
"precomputed_embedding",
):
return True
return False
async def _process_special_format(
self, image_data, audio_data, input_text, request_obj, **kwargs
):
"""Handle processor_output and precomputed_embedding input formats.
Delegates to the base class process_and_combine_mm_data which has
built-in support for these formats.
"""
if isinstance(input_text, list):
user_input_ids = input_text
prompt = ""
else:
user_input_ids = None
prompt = input_text or ""
# Normalize dicts: the HF MiniCPM processor returns "tgt_sizes" (plural)
# but the base class ATTR_NAME_TO_MODALITY maps "tgt_size" (singular).
# Also flatten the nested batch dimension so the structure matches
# what the NORMAL path produces (flat list of per-patch tensors).
normalized_images = []
for d in image_data or []:
if isinstance(d, dict):
d = dict(d)
if "tgt_sizes" in d and "tgt_size" not in d:
d["tgt_size"] = d.pop("tgt_sizes")
if d.get("format") == "processor_output":
pixel_values = d.get("pixel_values")
tgt_size = d.get("tgt_size")
if pixel_values is not None and tgt_size is not None:
pv_flat, ts_flat = [], []
for pixel_b, tgt_b in zip(pixel_values, tgt_size):
if isinstance(pixel_b, (list, tuple)):
for pixel_n, tgt_n in zip(pixel_b, tgt_b):
pv_flat.append(pixel_n)
ts_flat.append(tgt_n)
else:
pv_flat.append(pixel_b)
ts_flat.append(tgt_b)
d["pixel_values"] = pv_flat
d["tgt_size"] = ts_flat
normalized_images.append(d)
else:
normalized_images.append(d)
normalized_audios = list(audio_data or [])
if not prompt and (normalized_images or normalized_audios):
images = [d for d in normalized_images if isinstance(d, dict)]
audios = [d for d in normalized_audios if isinstance(d, dict)]
raw_img_dropped = len(normalized_images) - len(images)
raw_aud_dropped = len(normalized_audios) - len(audios)
if raw_img_dropped > 0 or raw_aud_dropped > 0:
raise ValueError(
f"[minicpm] Cannot process raw media with pre-tokenized "
f"input_ids. Provide multimodal data in 'processor_output' or "
f"'precomputed_embedding' format, or use a text prompt instead. "
f"(raw images dropped: {raw_img_dropped}, "
f"raw audios dropped: {raw_aud_dropped})"
)
base_output = BaseMultiModalProcessorOutput(
input_text=prompt,
images=images,
audios=audios,
)
else:
base_output = self.load_mm_data(
prompt=prompt,
image_data=normalized_images,
audio_data=audio_data,
multimodal_tokens=self.mm_tokens,
)
if base_output is None:
return None
mm_items, input_ids_tensor, ret = self.process_and_combine_mm_data(
base_output, self.mm_tokens
)
if user_input_ids is not None:
input_ids_tensor = torch.tensor(user_input_ids, dtype=torch.long)
for mm_item in mm_items:
if mm_item.modality == Modality.IMAGE:
image_offsets = self.get_mm_items_offset_by_pair(
input_ids=input_ids_tensor,
mm_start_id=self.im_start_id,
mm_end_id=self.im_end_id,
)
slice_offsets = self.get_mm_items_offset_by_pair(
input_ids=input_ids_tensor,
mm_start_id=self.slice_start_id,
mm_end_id=self.slice_end_id,
)
image_offsets.extend(slice_offsets)
mm_item.offsets = sorted(image_offsets)
elif mm_item.modality == Modality.AUDIO:
if (
self.audio_start_id is not None
and self.audio_end_id is not None
):
mm_item.offsets = self.get_mm_items_offset_by_pair(
input_ids=input_ids_tensor,
mm_start_id=self.audio_start_id,
mm_end_id=self.audio_end_id,
)
return {
"mm_items": mm_items,
"input_ids": input_ids_tensor.flatten().tolist(),
"audio_start_id": self.audio_start_id,
"audio_end_id": self.audio_end_id,
"im_token_id": self.im_token_id,
"im_start_id": self.im_start_id,
"im_end_id": self.im_end_id,
"slice_start_id": self.slice_start_id,
"slice_end_id": self.slice_end_id,
}
async def process_mm_data_async(
self,
image_data: List[Union[str, bytes]],
@@ -42,6 +177,17 @@ class MiniCPMMultimodalProcessor(BaseMultimodalProcessor):
request_obj,
**kwargs,
):
if isinstance(input_text, list) or self._has_special_format(
image_data, audio_data
):
return await self._process_special_format(
image_data=image_data,
audio_data=audio_data,
input_text=input_text,
request_obj=request_obj,
**kwargs,
)
base_output = self.load_mm_data(
prompt=input_text,
audio_data=audio_data,