From 44208d2adf5a9b761d8fb34852a473ea616cf141 Mon Sep 17 00:00:00 2001 From: Ken J Date: Wed, 4 Mar 2026 09:11:12 -0800 Subject: [PATCH] [vlm][minicpm] support input formats of processor output and embedding (#19614) --- python/sglang/srt/models/minicpmo.py | 5 + python/sglang/srt/models/minicpmv.py | 18 ++- .../srt/multimodal/processors/minicpm.py | 148 +++++++++++++++++- test/registered/vlm/test_vlm_input_format.py | 82 ++++++++++ 4 files changed, 251 insertions(+), 2 deletions(-) diff --git a/python/sglang/srt/models/minicpmo.py b/python/sglang/srt/models/minicpmo.py index 0d9d728a2..fc03e29bf 100644 --- a/python/sglang/srt/models/minicpmo.py +++ b/python/sglang/srt/models/minicpmo.py @@ -44,6 +44,7 @@ from sglang.srt.managers.mm_utils import ( ) from sglang.srt.managers.schedule_batch import ( MultimodalDataItem, + MultimodalInputFormat, MultimodalInputs, flatten_nested_list, ) @@ -1803,6 +1804,10 @@ class MiniCPMO(MiniCPMBaseModel): return audio_embs def get_image_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor: + if items and items[0].format == MultimodalInputFormat.PRECOMPUTED_EMBEDDING: + result = torch.cat([item.feature for item in items]) + return result.reshape(-1, result.shape[-1]) + # list of tensors pixel_values = flatten_nested_list([item.feature for item in items]) tgt_sizes = torch.stack( diff --git a/python/sglang/srt/models/minicpmv.py b/python/sglang/srt/models/minicpmv.py index c2c3b32b6..cd4489152 100644 --- a/python/sglang/srt/models/minicpmv.py +++ b/python/sglang/srt/models/minicpmv.py @@ -51,7 +51,11 @@ from sglang.srt.managers.mm_utils import ( MultiModalityDataPaddingPatternTokenPairs, general_mm_embed_routine, ) -from sglang.srt.managers.schedule_batch import MultimodalDataItem, MultimodalInputs +from sglang.srt.managers.schedule_batch import ( + MultimodalDataItem, + MultimodalInputFormat, + MultimodalInputs, +) from sglang.srt.model_executor.forward_batch_info import ForwardBatch from sglang.srt.model_loader.utils import set_default_torch_dtype from sglang.srt.model_loader.weight_utils import default_weight_loader @@ -939,6 +943,10 @@ class MiniCPMV2_6(MiniCPMBaseModel): return vision_embedding def get_image_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor: + if items and items[0].format == MultimodalInputFormat.PRECOMPUTED_EMBEDDING: + result = torch.cat([item.feature for item in items]) + return result.reshape(-1, result.shape[-1]) + # list of tensors pixel_values = flatten_nested_list([item.feature for item in items]) tgt_sizes = torch.stack( @@ -1097,6 +1105,10 @@ class MiniCPMV4_0(MiniCPMBaseModel): return vision_embedding def get_image_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor: + if items and items[0].format == MultimodalInputFormat.PRECOMPUTED_EMBEDDING: + result = torch.cat([item.feature for item in items]) + return result.reshape(-1, result.shape[-1]) + # list of tensors pixel_values = flatten_nested_list([item.feature for item in items]) tgt_sizes = torch.stack( @@ -1259,6 +1271,10 @@ class MiniCPMV4_5(MiniCPMBaseModel): return vision_embedding def get_image_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor: + if items and items[0].format == MultimodalInputFormat.PRECOMPUTED_EMBEDDING: + result = torch.cat([item.feature for item in items]) + return result.reshape(-1, result.shape[-1]) + # list of tensors pixel_values = flatten_nested_list([item.feature for item in items]) tgt_sizes = torch.stack( diff --git a/python/sglang/srt/multimodal/processors/minicpm.py b/python/sglang/srt/multimodal/processors/minicpm.py index defc047aa..2a375c9da 100644 --- a/python/sglang/srt/multimodal/processors/minicpm.py +++ b/python/sglang/srt/multimodal/processors/minicpm.py @@ -2,11 +2,15 @@ from typing import List, Union import torch -from sglang.srt.managers.schedule_batch import Modality, MultimodalDataItem +from sglang.srt.managers.schedule_batch import ( + Modality, + MultimodalDataItem, +) from sglang.srt.models.minicpmo import MiniCPMO from sglang.srt.models.minicpmv import MiniCPMV from sglang.srt.multimodal.processors.base_processor import ( BaseMultimodalProcessor, + BaseMultiModalProcessorOutput, MultimodalSpecialTokens, ) @@ -34,6 +38,137 @@ class MiniCPMMultimodalProcessor(BaseMultimodalProcessor): image_token_id=self.im_token_id, ).build(_processor) + @staticmethod + def _has_special_format(image_data, audio_data): + """Check if any input items use processor_output or precomputed_embedding format.""" + for data in list(image_data or []) + list(audio_data or []): + if isinstance(data, dict) and data.get("format") in ( + "processor_output", + "precomputed_embedding", + ): + return True + return False + + async def _process_special_format( + self, image_data, audio_data, input_text, request_obj, **kwargs + ): + """Handle processor_output and precomputed_embedding input formats. + + Delegates to the base class process_and_combine_mm_data which has + built-in support for these formats. + """ + if isinstance(input_text, list): + user_input_ids = input_text + prompt = "" + else: + user_input_ids = None + prompt = input_text or "" + + # Normalize dicts: the HF MiniCPM processor returns "tgt_sizes" (plural) + # but the base class ATTR_NAME_TO_MODALITY maps "tgt_size" (singular). + # Also flatten the nested batch dimension so the structure matches + # what the NORMAL path produces (flat list of per-patch tensors). + normalized_images = [] + for d in image_data or []: + if isinstance(d, dict): + d = dict(d) + if "tgt_sizes" in d and "tgt_size" not in d: + d["tgt_size"] = d.pop("tgt_sizes") + if d.get("format") == "processor_output": + pixel_values = d.get("pixel_values") + tgt_size = d.get("tgt_size") + if pixel_values is not None and tgt_size is not None: + pv_flat, ts_flat = [], [] + for pixel_b, tgt_b in zip(pixel_values, tgt_size): + if isinstance(pixel_b, (list, tuple)): + for pixel_n, tgt_n in zip(pixel_b, tgt_b): + pv_flat.append(pixel_n) + ts_flat.append(tgt_n) + else: + pv_flat.append(pixel_b) + ts_flat.append(tgt_b) + d["pixel_values"] = pv_flat + d["tgt_size"] = ts_flat + normalized_images.append(d) + else: + normalized_images.append(d) + + normalized_audios = list(audio_data or []) + + if not prompt and (normalized_images or normalized_audios): + images = [d for d in normalized_images if isinstance(d, dict)] + audios = [d for d in normalized_audios if isinstance(d, dict)] + + raw_img_dropped = len(normalized_images) - len(images) + raw_aud_dropped = len(normalized_audios) - len(audios) + if raw_img_dropped > 0 or raw_aud_dropped > 0: + raise ValueError( + f"[minicpm] Cannot process raw media with pre-tokenized " + f"input_ids. Provide multimodal data in 'processor_output' or " + f"'precomputed_embedding' format, or use a text prompt instead. " + f"(raw images dropped: {raw_img_dropped}, " + f"raw audios dropped: {raw_aud_dropped})" + ) + + base_output = BaseMultiModalProcessorOutput( + input_text=prompt, + images=images, + audios=audios, + ) + else: + base_output = self.load_mm_data( + prompt=prompt, + image_data=normalized_images, + audio_data=audio_data, + multimodal_tokens=self.mm_tokens, + ) + + if base_output is None: + return None + + mm_items, input_ids_tensor, ret = self.process_and_combine_mm_data( + base_output, self.mm_tokens + ) + + if user_input_ids is not None: + input_ids_tensor = torch.tensor(user_input_ids, dtype=torch.long) + for mm_item in mm_items: + if mm_item.modality == Modality.IMAGE: + image_offsets = self.get_mm_items_offset_by_pair( + input_ids=input_ids_tensor, + mm_start_id=self.im_start_id, + mm_end_id=self.im_end_id, + ) + slice_offsets = self.get_mm_items_offset_by_pair( + input_ids=input_ids_tensor, + mm_start_id=self.slice_start_id, + mm_end_id=self.slice_end_id, + ) + image_offsets.extend(slice_offsets) + mm_item.offsets = sorted(image_offsets) + elif mm_item.modality == Modality.AUDIO: + if ( + self.audio_start_id is not None + and self.audio_end_id is not None + ): + mm_item.offsets = self.get_mm_items_offset_by_pair( + input_ids=input_ids_tensor, + mm_start_id=self.audio_start_id, + mm_end_id=self.audio_end_id, + ) + + return { + "mm_items": mm_items, + "input_ids": input_ids_tensor.flatten().tolist(), + "audio_start_id": self.audio_start_id, + "audio_end_id": self.audio_end_id, + "im_token_id": self.im_token_id, + "im_start_id": self.im_start_id, + "im_end_id": self.im_end_id, + "slice_start_id": self.slice_start_id, + "slice_end_id": self.slice_end_id, + } + async def process_mm_data_async( self, image_data: List[Union[str, bytes]], @@ -42,6 +177,17 @@ class MiniCPMMultimodalProcessor(BaseMultimodalProcessor): request_obj, **kwargs, ): + if isinstance(input_text, list) or self._has_special_format( + image_data, audio_data + ): + return await self._process_special_format( + image_data=image_data, + audio_data=audio_data, + input_text=input_text, + request_obj=request_obj, + **kwargs, + ) + base_output = self.load_mm_data( prompt=input_text, audio_data=audio_data, diff --git a/test/registered/vlm/test_vlm_input_format.py b/test/registered/vlm/test_vlm_input_format.py index 3fc282a8f..57a2c9a98 100644 --- a/test/registered/vlm/test_vlm_input_format.py +++ b/test/registered/vlm/test_vlm_input_format.py @@ -502,5 +502,87 @@ class TestInternVLUnderstandsImage(VLMInputTestBase, unittest.IsolatedAsyncioTes return dict(processor_output, format="processor_output") +class TestMiniCPMVUnderstandsImage(VLMInputTestBase, unittest.IsolatedAsyncioTestCase): + model_path = "openbmb/MiniCPM-V-4" + chat_template = "minicpmv" + + @classmethod + def setUpClass(cls): + assert cls.model_path is not None, "Set model_path in subclass" + assert cls.chat_template is not None, "Set chat_template in subclass" + cls.image_urls = [IMAGE_MAN_IRONING_URL, IMAGE_SGL_LOGO_URL] + cls.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + cls.main_image = [] + for image_url in cls.image_urls: + response = requests.get(image_url) + cls.main_image.append(Image.open(BytesIO(response.content))) + + cls.processor = AutoProcessor.from_pretrained( + cls.model_path, trust_remote_code=True + ) + cls._init_visual() + + @classmethod + def _init_visual(cls): + model = AutoModel.from_pretrained( + cls.model_path, trust_remote_code=True, torch_dtype=torch.bfloat16 + ) + cls.vpm_model = model.vpm.eval().to(cls.device) + cls.resampler_model = model.resampler.eval().to(cls.device) + del model + + def visual_func(processor_output): + pixel_values = processor_output["pixel_values"] + tgt_sizes = processor_output["tgt_sizes"] + + pixel_values_flat = [] + tgt_sizes_flat = [] + for pixel_b, tgt_b in zip(pixel_values, tgt_sizes): + if isinstance(pixel_b, (list, tuple)): + for pixel_n, tgt_n in zip(pixel_b, tgt_b): + pixel_values_flat.append(pixel_n) + tgt_sizes_flat.append(tgt_n) + else: + pixel_values_flat.append(pixel_b) + tgt_sizes_flat.append(tgt_b) + + tgt_sizes_tensor = torch.stack(tgt_sizes_flat, dim=0) + device = cls.vpm_model.embeddings.position_embedding.weight.device + dtype = cls.vpm_model.embeddings.position_embedding.weight.dtype + + all_pixel_values_lst = [ + i.flatten(end_dim=1).permute(1, 0) for i in pixel_values_flat + ] + max_patches = int( + (tgt_sizes_tensor[:, 0] * tgt_sizes_tensor[:, 1]).max().item() + ) + all_pixel_values = torch.nn.utils.rnn.pad_sequence( + all_pixel_values_lst, batch_first=True, padding_value=0.0 + ) + B, L, _ = all_pixel_values.shape + all_pixel_values = all_pixel_values.permute(0, 2, 1).reshape(B, 3, -1, L) + patch_attn_mask = torch.zeros( + (B, 1, max_patches), dtype=torch.bool, device=device + ) + tgt_sizes_dev = tgt_sizes_tensor.to(device) + mask_shapes = tgt_sizes_dev[:, 0] * tgt_sizes_dev[:, 1] + patch_attn_mask[:, 0, :] = torch.arange( + max_patches, device=device + ).unsqueeze(0) < mask_shapes.unsqueeze(1) + + vision_output = cls.vpm_model( + all_pixel_values.type(dtype), + patch_attention_mask=patch_attn_mask, + tgt_sizes=tgt_sizes_tensor, + ) + vision_embedding = vision_output.last_hidden_state + return cls.resampler_model(vision_embedding, tgt_sizes_tensor) + + cls.visual = visual_func + + def _processor_output_image_data(self, processor_output): + return dict(processor_output, format="processor_output") + + if __name__ == "__main__": unittest.main()