chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[vlm][internVL] Support processor and embedding inputs for InternVL (#19127)

Browse Source
This commit is contained in:
Ken J
2026-02-26 22:46:48 -08:00
committed by GitHub
parent 8a56cc5836
commit f0c2089597
4 changed files with 282 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
python/sglang/srt/models/internvl.py
View File

@@ -616,6 +616,10 @@ class InternVLChatModel(nn.Module):
image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`).
"""
pixel_values = torch.cat([item.feature for item in items])
# If already precomputed embeddings (not raw pixel values), skip vision encoder.
# Normal pixel_values are 4D [N, C, H, W]; precomputed embeddings are 2D or 3D.
if pixel_values.dim() != 4:
return pixel_values
image_features = self.extract_feature(pixel_values)
return image_features
@@ -623,6 +627,9 @@ class InternVLChatModel(nn.Module):
# items: each item corresponds to one video (recommended)
# item.feature shape: [num_frames, 3, 448, 448] (or [num_tiles, 3, 448, 448])
pixel_values = torch.cat([item.feature for item in items], dim=0)
# If already precomputed embeddings, skip vision encoder.
if pixel_values.dim() != 4:
return pixel_values
video_features = self.extract_feature(pixel_values)
return video_features

19
python/sglang/srt/multimodal/processors/base_processor.py
View File

@@ -182,6 +182,13 @@ class BaseMultimodalProcessor(ABC):
self.server_args = server_args
self.transport_mode = transport_mode
# Resolve tokenizer: some processors (e.g. InternVL) pass a tokenizer
# directly as _processor rather than a processor that wraps a tokenizer.
if hasattr(self._processor, "tokenizer"):
self._tokenizer = self._processor.tokenizer
else:
self._tokenizer = self._processor
# FIXME: not accurate, model and image specific
self.NUM_TOKEN_PER_FRAME = 330
@@ -252,7 +259,7 @@ class BaseMultimodalProcessor(ABC):
Use prompt and img_grid_thw to build input_ids
"""
if not isinstance(prompt, list):
prompt = self._processor.tokenizer.encode(prompt)
prompt = self._tokenizer.encode(prompt)
img_token_id = self.IM_TOKEN_ID
spatial_merge_size = self.spatial_merge_size
@@ -636,7 +643,7 @@ class BaseMultimodalProcessor(ABC):
multimodal_tokens_pattern = multimodal_tokens.get_combined_regex()
if isinstance(prompt, list) and return_text:
assert len(prompt) and isinstance(prompt[0], int)
prompt = self._processor.tokenizer.decode(prompt)
prompt = self._tokenizer.decode(prompt)
else:
prompt = prompt
@@ -709,7 +716,7 @@ class BaseMultimodalProcessor(ABC):
# Convert prompt into str
if isinstance(prompt, list) and return_text:
assert len(prompt) and isinstance(prompt[0], int)
prompt_str = self._processor.tokenizer.decode(prompt)
prompt_str = self._tokenizer.decode(prompt)
else:
assert isinstance(prompt, str)
prompt_str = prompt
@@ -793,7 +800,7 @@ class BaseMultimodalProcessor(ABC):
multimodal_tokens_pattern = multimodal_tokens.get_combined_regex()
if isinstance(prompt, list) and return_text:
assert len(prompt) and isinstance(prompt[0], int)
prompt = self._processor.tokenizer.decode(prompt)
prompt = self._tokenizer.decode(prompt)
else:
prompt = prompt
@@ -988,7 +995,7 @@ class BaseMultimodalProcessor(ABC):
all_loaded_data = base_output.organize_results()
# Handle text-only case
if not all_loaded_data:
input_ids = self._processor.tokenizer(
input_ids = self._tokenizer(
base_output.input_text,
return_tensors="pt",
add_special_tokens=True,
@@ -1044,7 +1051,7 @@ class BaseMultimodalProcessor(ABC):
)
# Fallback tokenization if no raw items were processed
if input_ids is None:
input_ids = self._processor.tokenizer(
input_ids = self._tokenizer(
base_output.input_text,
return_tensors="pt",
add_special_tokens=True,

110
python/sglang/srt/multimodal/processors/internvl.py
View File

@@ -9,11 +9,15 @@ import torch
from decord import VideoReader, cpu, gpu
from PIL import Image
from sglang.srt.managers.schedule_batch import Modality, MultimodalDataItem
from sglang.srt.managers.schedule_batch import (
Modality,
MultimodalDataItem,
)
from sglang.srt.models.interns1 import InternS1ForConditionalGeneration
from sglang.srt.models.internvl import InternVLChatModel
from sglang.srt.multimodal.processors.base_processor import (
BaseMultimodalProcessor,
BaseMultiModalProcessorOutput,
MultimodalSpecialTokens,
)
@@ -255,9 +259,113 @@ class InternVLProcessor(BaseMultimodalProcessor):
frames_per_video = max(1, max_total_frames // max(num_videos, 1))
return max(1, min(int(requested), int(frames_per_video)))
@staticmethod
def _has_special_format(image_data, video_data):
"""Check if any input items use processor_output or precomputed_embedding format."""
for data in list(image_data or []) + list(video_data or []):
if isinstance(data, dict) and data.get("format") in (
"processor_output",
"precomputed_embedding",
):
return True
return False
async def _process_special_format(
self, image_data, video_data, input_text, request_obj, **kwargs
):
"""Handle processor_output and precomputed_embedding input formats.
Delegates to the base class process_and_combine_mm_data which has
built-in support for these formats.
"""
# When user provides input_ids directly, input_text may be a list of ints
if isinstance(input_text, list):
user_input_ids = input_text
prompt = ""
else:
user_input_ids = None
prompt = input_text or ""
# When the prompt is empty (user provided input_ids directly),
# load_mm_data can't match multimodal tokens to data items.
# Build BaseMultiModalProcessorOutput directly from the dict items.
if not prompt and (image_data or video_data):
images = [d for d in (image_data or []) if isinstance(d, dict)]
videos = [d for d in (video_data or []) if isinstance(d, dict)]
# Raise if raw (non-dict) images/videos were silently filtered out.
# InternVL cannot process raw images without a text prompt because
# dynamic tiling and placeholder expansion require the prompt string.
raw_img_dropped = len(image_data or []) - len(images)
raw_vid_dropped = len(video_data or []) - len(videos)
if raw_img_dropped > 0 or raw_vid_dropped > 0:
raise ValueError(
f"[internvl] Cannot process raw images/videos with pre-tokenized "
f"input_ids. Provide multimodal data in 'processor_output' or "
f"'precomputed_embedding' format, or use a text prompt instead. "
f"(raw images dropped: {raw_img_dropped}, "
f"raw videos dropped: {raw_vid_dropped})"
)
base_output = BaseMultiModalProcessorOutput(
input_text=prompt,
images=images,
videos=videos,
)
else:
base_output = self.load_mm_data(
prompt=prompt,
image_data=image_data,
video_data=video_data,
multimodal_tokens=self.mm_tokens,
discard_alpha_channel=True,
)
mm_items, input_ids_tensor, ret = self.process_and_combine_mm_data(
base_output, self.mm_tokens
)
# If user provided input_ids directly, use those and recompute offsets
if user_input_ids is not None:
input_ids_tensor = torch.tensor(user_input_ids, dtype=torch.long)
for mm_item in mm_items:
if (
mm_item.modality == Modality.VIDEO
and self.video_token_id is not None
):
mm_token_id = self.video_token_id
else:
mm_token_id = self.img_context_token_id
mm_item.offsets = self.get_mm_items_offset(
input_ids=input_ids_tensor,
mm_token_id=mm_token_id,
)
return {
"input_ids": input_ids_tensor.flatten().tolist(),
"mm_items": mm_items,
"im_start_id": self.img_start_token_id,
"im_end_id": self.img_end_token_id,
"im_token_id": self.img_context_token_id,
"video_token_id": self.video_token_id,
}
async def process_mm_data_async(
self, image_data, input_text, request_obj, **kwargs
):
video_data = getattr(request_obj, "video_data", None) or []
# Handle processor_output and precomputed_embedding formats
if isinstance(input_text, list) or self._has_special_format(
image_data, video_data
):
return await self._process_special_format(
image_data=image_data,
video_data=video_data,
input_text=input_text,
request_obj=request_obj,
**kwargs,
)
is_internlm2 = self.llm_arch == "InternLM2ForCausalLM"