From f0c208959794abcd603c6495e6829b2ff6a06a46 Mon Sep 17 00:00:00 2001 From: Ken J Date: Thu, 26 Feb 2026 22:46:48 -0800 Subject: [PATCH] [vlm][internVL] Support processor and embedding inputs for InternVL (#19127) --- python/sglang/srt/models/internvl.py | 7 + .../multimodal/processors/base_processor.py | 19 ++- .../srt/multimodal/processors/internvl.py | 110 ++++++++++++- test/registered/vlm/test_vlm_input_format.py | 153 ++++++++++++++++++ 4 files changed, 282 insertions(+), 7 deletions(-) diff --git a/python/sglang/srt/models/internvl.py b/python/sglang/srt/models/internvl.py index b7be90101..e5d6a5b70 100644 --- a/python/sglang/srt/models/internvl.py +++ b/python/sglang/srt/models/internvl.py @@ -616,6 +616,10 @@ class InternVLChatModel(nn.Module): image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). """ pixel_values = torch.cat([item.feature for item in items]) + # If already precomputed embeddings (not raw pixel values), skip vision encoder. + # Normal pixel_values are 4D [N, C, H, W]; precomputed embeddings are 2D or 3D. + if pixel_values.dim() != 4: + return pixel_values image_features = self.extract_feature(pixel_values) return image_features @@ -623,6 +627,9 @@ class InternVLChatModel(nn.Module): # items: each item corresponds to one video (recommended) # item.feature shape: [num_frames, 3, 448, 448] (or [num_tiles, 3, 448, 448]) pixel_values = torch.cat([item.feature for item in items], dim=0) + # If already precomputed embeddings, skip vision encoder. + if pixel_values.dim() != 4: + return pixel_values video_features = self.extract_feature(pixel_values) return video_features diff --git a/python/sglang/srt/multimodal/processors/base_processor.py b/python/sglang/srt/multimodal/processors/base_processor.py index c21339ef8..22adf71e2 100644 --- a/python/sglang/srt/multimodal/processors/base_processor.py +++ b/python/sglang/srt/multimodal/processors/base_processor.py @@ -182,6 +182,13 @@ class BaseMultimodalProcessor(ABC): self.server_args = server_args self.transport_mode = transport_mode + # Resolve tokenizer: some processors (e.g. InternVL) pass a tokenizer + # directly as _processor rather than a processor that wraps a tokenizer. + if hasattr(self._processor, "tokenizer"): + self._tokenizer = self._processor.tokenizer + else: + self._tokenizer = self._processor + # FIXME: not accurate, model and image specific self.NUM_TOKEN_PER_FRAME = 330 @@ -252,7 +259,7 @@ class BaseMultimodalProcessor(ABC): Use prompt and img_grid_thw to build input_ids """ if not isinstance(prompt, list): - prompt = self._processor.tokenizer.encode(prompt) + prompt = self._tokenizer.encode(prompt) img_token_id = self.IM_TOKEN_ID spatial_merge_size = self.spatial_merge_size @@ -636,7 +643,7 @@ class BaseMultimodalProcessor(ABC): multimodal_tokens_pattern = multimodal_tokens.get_combined_regex() if isinstance(prompt, list) and return_text: assert len(prompt) and isinstance(prompt[0], int) - prompt = self._processor.tokenizer.decode(prompt) + prompt = self._tokenizer.decode(prompt) else: prompt = prompt @@ -709,7 +716,7 @@ class BaseMultimodalProcessor(ABC): # Convert prompt into str if isinstance(prompt, list) and return_text: assert len(prompt) and isinstance(prompt[0], int) - prompt_str = self._processor.tokenizer.decode(prompt) + prompt_str = self._tokenizer.decode(prompt) else: assert isinstance(prompt, str) prompt_str = prompt @@ -793,7 +800,7 @@ class BaseMultimodalProcessor(ABC): multimodal_tokens_pattern = multimodal_tokens.get_combined_regex() if isinstance(prompt, list) and return_text: assert len(prompt) and isinstance(prompt[0], int) - prompt = self._processor.tokenizer.decode(prompt) + prompt = self._tokenizer.decode(prompt) else: prompt = prompt @@ -988,7 +995,7 @@ class BaseMultimodalProcessor(ABC): all_loaded_data = base_output.organize_results() # Handle text-only case if not all_loaded_data: - input_ids = self._processor.tokenizer( + input_ids = self._tokenizer( base_output.input_text, return_tensors="pt", add_special_tokens=True, @@ -1044,7 +1051,7 @@ class BaseMultimodalProcessor(ABC): ) # Fallback tokenization if no raw items were processed if input_ids is None: - input_ids = self._processor.tokenizer( + input_ids = self._tokenizer( base_output.input_text, return_tensors="pt", add_special_tokens=True, diff --git a/python/sglang/srt/multimodal/processors/internvl.py b/python/sglang/srt/multimodal/processors/internvl.py index 31d45c9c8..2c05608d9 100644 --- a/python/sglang/srt/multimodal/processors/internvl.py +++ b/python/sglang/srt/multimodal/processors/internvl.py @@ -9,11 +9,15 @@ import torch from decord import VideoReader, cpu, gpu from PIL import Image -from sglang.srt.managers.schedule_batch import Modality, MultimodalDataItem +from sglang.srt.managers.schedule_batch import ( + Modality, + MultimodalDataItem, +) from sglang.srt.models.interns1 import InternS1ForConditionalGeneration from sglang.srt.models.internvl import InternVLChatModel from sglang.srt.multimodal.processors.base_processor import ( BaseMultimodalProcessor, + BaseMultiModalProcessorOutput, MultimodalSpecialTokens, ) @@ -255,9 +259,113 @@ class InternVLProcessor(BaseMultimodalProcessor): frames_per_video = max(1, max_total_frames // max(num_videos, 1)) return max(1, min(int(requested), int(frames_per_video))) + @staticmethod + def _has_special_format(image_data, video_data): + """Check if any input items use processor_output or precomputed_embedding format.""" + for data in list(image_data or []) + list(video_data or []): + if isinstance(data, dict) and data.get("format") in ( + "processor_output", + "precomputed_embedding", + ): + return True + return False + + async def _process_special_format( + self, image_data, video_data, input_text, request_obj, **kwargs + ): + """Handle processor_output and precomputed_embedding input formats. + + Delegates to the base class process_and_combine_mm_data which has + built-in support for these formats. + """ + # When user provides input_ids directly, input_text may be a list of ints + if isinstance(input_text, list): + user_input_ids = input_text + prompt = "" + else: + user_input_ids = None + prompt = input_text or "" + + # When the prompt is empty (user provided input_ids directly), + # load_mm_data can't match multimodal tokens to data items. + # Build BaseMultiModalProcessorOutput directly from the dict items. + if not prompt and (image_data or video_data): + images = [d for d in (image_data or []) if isinstance(d, dict)] + videos = [d for d in (video_data or []) if isinstance(d, dict)] + + # Raise if raw (non-dict) images/videos were silently filtered out. + # InternVL cannot process raw images without a text prompt because + # dynamic tiling and placeholder expansion require the prompt string. + raw_img_dropped = len(image_data or []) - len(images) + raw_vid_dropped = len(video_data or []) - len(videos) + if raw_img_dropped > 0 or raw_vid_dropped > 0: + raise ValueError( + f"[internvl] Cannot process raw images/videos with pre-tokenized " + f"input_ids. Provide multimodal data in 'processor_output' or " + f"'precomputed_embedding' format, or use a text prompt instead. " + f"(raw images dropped: {raw_img_dropped}, " + f"raw videos dropped: {raw_vid_dropped})" + ) + + base_output = BaseMultiModalProcessorOutput( + input_text=prompt, + images=images, + videos=videos, + ) + else: + base_output = self.load_mm_data( + prompt=prompt, + image_data=image_data, + video_data=video_data, + multimodal_tokens=self.mm_tokens, + discard_alpha_channel=True, + ) + + mm_items, input_ids_tensor, ret = self.process_and_combine_mm_data( + base_output, self.mm_tokens + ) + + # If user provided input_ids directly, use those and recompute offsets + if user_input_ids is not None: + input_ids_tensor = torch.tensor(user_input_ids, dtype=torch.long) + for mm_item in mm_items: + if ( + mm_item.modality == Modality.VIDEO + and self.video_token_id is not None + ): + mm_token_id = self.video_token_id + else: + mm_token_id = self.img_context_token_id + mm_item.offsets = self.get_mm_items_offset( + input_ids=input_ids_tensor, + mm_token_id=mm_token_id, + ) + + return { + "input_ids": input_ids_tensor.flatten().tolist(), + "mm_items": mm_items, + "im_start_id": self.img_start_token_id, + "im_end_id": self.img_end_token_id, + "im_token_id": self.img_context_token_id, + "video_token_id": self.video_token_id, + } + async def process_mm_data_async( self, image_data, input_text, request_obj, **kwargs ): + video_data = getattr(request_obj, "video_data", None) or [] + + # Handle processor_output and precomputed_embedding formats + if isinstance(input_text, list) or self._has_special_format( + image_data, video_data + ): + return await self._process_special_format( + image_data=image_data, + video_data=video_data, + input_text=input_text, + request_obj=request_obj, + **kwargs, + ) is_internlm2 = self.llm_arch == "InternLM2ForCausalLM" diff --git a/test/registered/vlm/test_vlm_input_format.py b/test/registered/vlm/test_vlm_input_format.py index f717c3b5f..3fc282a8f 100644 --- a/test/registered/vlm/test_vlm_input_format.py +++ b/test/registered/vlm/test_vlm_input_format.py @@ -349,5 +349,158 @@ class TestKimiVLImageUnderstandsImage( # return dict(processor_output, format="processor_output") +class TestInternVLUnderstandsImage(VLMInputTestBase, unittest.IsolatedAsyncioTestCase): + model_path = "OpenGVLab/InternVL2-2B" + chat_template = "internvl-2-5" + + @classmethod + def setUpClass(cls): + assert cls.model_path is not None, "Set model_path in subclass" + assert cls.chat_template is not None, "Set chat_template in subclass" + cls.image_urls = [IMAGE_MAN_IRONING_URL, IMAGE_SGL_LOGO_URL] + cls.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + cls.main_image = [] + for image_url in cls.image_urls: + response = requests.get(image_url) + cls.main_image.append(Image.open(BytesIO(response.content))) + + # InternVL models (2, 3, 3.5) do not ship a standard HuggingFace + # Processor; AutoProcessor.from_pretrained returns a bare tokenizer. + # Use AutoTokenizer explicitly so the intent is clear. + from transformers import AutoTokenizer + + cls.processor = AutoTokenizer.from_pretrained( + cls.model_path, trust_remote_code=True + ) + cls._init_visual() + + @classmethod + def _init_visual(cls): + model = AutoModel.from_pretrained( + cls.model_path, trust_remote_code=True, torch_dtype=torch.bfloat16 + ) + cls.vision_model = model.vision_model.eval().to(cls.device) + cls.mlp1 = model.mlp1.eval().to(cls.device) + + config = model.config + cls.internvl_config = config + image_size = getattr(config, "force_image_size", None) or ( + config.vision_config.image_size + ) + patch_size = config.vision_config.patch_size + cls.num_image_token = int( + (image_size // patch_size) ** 2 * (config.downsample_ratio**2) + ) + cls.internvl_image_size = image_size + cls.internvl_downsample_ratio = config.downsample_ratio + cls.internvl_ps_version = config.ps_version + cls.internvl_select_layer = config.select_layer + + del model + + def pixel_shuffle(x, scale_factor): + n, w, h, c = x.size() + x = x.view(n, w, int(h * scale_factor), int(c / scale_factor)) + x = x.permute(0, 2, 1, 3).contiguous() + x = x.view( + n, + int(h * scale_factor), + int(w * scale_factor), + int(c / (scale_factor * scale_factor)), + ) + if cls.internvl_ps_version != "v1": + x = x.permute(0, 2, 1, 3).contiguous() + return x + + def visual_func(processor_output): + pixel_values = processor_output["pixel_values"].to( + cls.device, dtype=torch.bfloat16 + ) + if cls.internvl_select_layer == -1: + vit_embeds = cls.vision_model( + pixel_values=pixel_values, + output_hidden_states=False, + return_dict=True, + ).last_hidden_state + else: + vit_embeds = cls.vision_model( + pixel_values=pixel_values, + output_hidden_states=True, + return_dict=True, + ).hidden_states[cls.internvl_select_layer] + vit_embeds = vit_embeds[:, 1:, :] + + h = w = int(vit_embeds.shape[1] ** 0.5) + vit_embeds = vit_embeds.reshape(vit_embeds.shape[0], h, w, -1) + vit_embeds = pixel_shuffle( + vit_embeds, scale_factor=cls.internvl_downsample_ratio + ) + vit_embeds = vit_embeds.reshape( + vit_embeds.shape[0], -1, vit_embeds.shape[-1] + ) + vit_embeds = cls.mlp1(vit_embeds) + return vit_embeds + + cls.visual = visual_func + + def get_processor_output(self, req=None): + """Override to handle InternVL's custom preprocessing. + + Uses shared ``image_to_pixel_values`` from ``internvl_utils`` for + image preprocessing (dynamic tiling + normalize) and expands + ```` placeholders into ```` + context tokens + + ```` — mirroring the logic in + ``InternVLProcessor.process_internlm2_mm_data_async``. + """ + from sglang.srt.multimodal.internvl_utils import image_to_pixel_values + from sglang.srt.multimodal.processors.internvl import InternVLProcessor + + if req is None: + req = self.get_completion_request() + conv = generate_chat_conv(req, template_name=self.chat_template) + text = conv.get_prompt() + + # Preprocess images using the shared utility (dynamic tiling + + # bicubic resize + ImageNet normalize), same pipeline as the engine. + all_pixel_values = [] + num_patches_list = [] + for img in self.main_image: + pv = image_to_pixel_values( + img, + input_size=self.internvl_image_size, + max_num_tiles=InternVLProcessor.IMAGE_MAX_NUM, + use_thumbnail=True, + ) + all_pixel_values.append(pv) + num_patches_list.append(pv.shape[0]) + + pixel_values = torch.cat(all_pixel_values, dim=0).to(self.device) + + # Expand each placeholder into + *N + . + # This mirrors InternVLProcessor.process_internlm2_mm_data_async. + ph = "<<<__IMG_PH__>>>" + expanded_text = text.replace(InternVLProcessor.IMG_CONTEXT, ph) + for num_patches in num_patches_list: + image_tokens = ( + InternVLProcessor.IMG_START + + InternVLProcessor.IMG_CONTEXT * (self.num_image_token * num_patches) + + InternVLProcessor.IMG_END + ) + expanded_text = expanded_text.replace(ph, image_tokens, 1) + # Remove any remaining placeholders (more placeholders than images) + expanded_text = expanded_text.replace(ph, "") + + # Tokenize the expanded text + input_ids = self.processor(expanded_text, return_tensors="pt")["input_ids"] + + return { + "input_ids": input_ids, + "pixel_values": pixel_values, + }, text + + def _processor_output_image_data(self, processor_output): + return dict(processor_output, format="processor_output") + + if __name__ == "__main__": unittest.main()