chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 11 Packages Projects Releases Wiki Activity

[vlm][internVL] Support processor and embedding inputs for InternVL (#19127)

Browse Source
This commit is contained in:
Ken J
2026-02-26 22:46:48 -08:00
committed by GitHub
parent 8a56cc5836
commit f0c2089597
4 changed files with 282 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
python/sglang/srt/models/internvl.py
View File

@@ -616,6 +616,10 @@ class InternVLChatModel(nn.Module):
image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`).
"""
pixel_values = torch.cat([item.feature for item in items])
# If already precomputed embeddings (not raw pixel values), skip vision encoder.
# Normal pixel_values are 4D [N, C, H, W]; precomputed embeddings are 2D or 3D.
if pixel_values.dim() != 4:
return pixel_values
image_features = self.extract_feature(pixel_values)
return image_features
@@ -623,6 +627,9 @@ class InternVLChatModel(nn.Module):
# items: each item corresponds to one video (recommended)
# item.feature shape: [num_frames, 3, 448, 448] (or [num_tiles, 3, 448, 448])
pixel_values = torch.cat([item.feature for item in items], dim=0)
# If already precomputed embeddings, skip vision encoder.
if pixel_values.dim() != 4:
return pixel_values
video_features = self.extract_feature(pixel_values)
return video_features

19
python/sglang/srt/multimodal/processors/base_processor.py
View File

@@ -182,6 +182,13 @@ class BaseMultimodalProcessor(ABC):
self.server_args = server_args
self.transport_mode = transport_mode
# Resolve tokenizer: some processors (e.g. InternVL) pass a tokenizer
# directly as _processor rather than a processor that wraps a tokenizer.
if hasattr(self._processor, "tokenizer"):
self._tokenizer = self._processor.tokenizer
else:
self._tokenizer = self._processor
# FIXME: not accurate, model and image specific
self.NUM_TOKEN_PER_FRAME = 330
@@ -252,7 +259,7 @@ class BaseMultimodalProcessor(ABC):
Use prompt and img_grid_thw to build input_ids
"""
if not isinstance(prompt, list):
prompt = self._processor.tokenizer.encode(prompt)
prompt = self._tokenizer.encode(prompt)
img_token_id = self.IM_TOKEN_ID
spatial_merge_size = self.spatial_merge_size
@@ -636,7 +643,7 @@ class BaseMultimodalProcessor(ABC):
multimodal_tokens_pattern = multimodal_tokens.get_combined_regex()
if isinstance(prompt, list) and return_text:
assert len(prompt) and isinstance(prompt[0], int)
prompt = self._processor.tokenizer.decode(prompt)
prompt = self._tokenizer.decode(prompt)
else:
prompt = prompt
@@ -709,7 +716,7 @@ class BaseMultimodalProcessor(ABC):
# Convert prompt into str
if isinstance(prompt, list) and return_text:
assert len(prompt) and isinstance(prompt[0], int)
prompt_str = self._processor.tokenizer.decode(prompt)
prompt_str = self._tokenizer.decode(prompt)
else:
assert isinstance(prompt, str)
prompt_str = prompt
@@ -793,7 +800,7 @@ class BaseMultimodalProcessor(ABC):
multimodal_tokens_pattern = multimodal_tokens.get_combined_regex()
if isinstance(prompt, list) and return_text:
assert len(prompt) and isinstance(prompt[0], int)
prompt = self._processor.tokenizer.decode(prompt)
prompt = self._tokenizer.decode(prompt)
else:
prompt = prompt
@@ -988,7 +995,7 @@ class BaseMultimodalProcessor(ABC):
all_loaded_data = base_output.organize_results()
# Handle text-only case
if not all_loaded_data:
input_ids = self._processor.tokenizer(
input_ids = self._tokenizer(
base_output.input_text,
return_tensors="pt",
add_special_tokens=True,
@@ -1044,7 +1051,7 @@ class BaseMultimodalProcessor(ABC):
)
# Fallback tokenization if no raw items were processed
if input_ids is None:
input_ids = self._processor.tokenizer(
input_ids = self._tokenizer(
base_output.input_text,
return_tensors="pt",
add_special_tokens=True,

110
python/sglang/srt/multimodal/processors/internvl.py
View File

@@ -9,11 +9,15 @@ import torch
from decord import VideoReader, cpu, gpu
from PIL import Image
from sglang.srt.managers.schedule_batch import Modality, MultimodalDataItem
from sglang.srt.managers.schedule_batch import (
Modality,
MultimodalDataItem,
)
from sglang.srt.models.interns1 import InternS1ForConditionalGeneration
from sglang.srt.models.internvl import InternVLChatModel
from sglang.srt.multimodal.processors.base_processor import (
BaseMultimodalProcessor,
BaseMultiModalProcessorOutput,
MultimodalSpecialTokens,
)
@@ -255,9 +259,113 @@ class InternVLProcessor(BaseMultimodalProcessor):
frames_per_video = max(1, max_total_frames // max(num_videos, 1))
return max(1, min(int(requested), int(frames_per_video)))
@staticmethod
def _has_special_format(image_data, video_data):
"""Check if any input items use processor_output or precomputed_embedding format."""
for data in list(image_data or []) + list(video_data or []):
if isinstance(data, dict) and data.get("format") in (
"processor_output",
"precomputed_embedding",
):
return True
return False
async def _process_special_format(
self, image_data, video_data, input_text, request_obj, **kwargs
):
"""Handle processor_output and precomputed_embedding input formats.
Delegates to the base class process_and_combine_mm_data which has
built-in support for these formats.
"""
# When user provides input_ids directly, input_text may be a list of ints
if isinstance(input_text, list):
user_input_ids = input_text
prompt = ""
else:
user_input_ids = None
prompt = input_text or ""
# When the prompt is empty (user provided input_ids directly),
# load_mm_data can't match multimodal tokens to data items.
# Build BaseMultiModalProcessorOutput directly from the dict items.
if not prompt and (image_data or video_data):
images = [d for d in (image_data or []) if isinstance(d, dict)]
videos = [d for d in (video_data or []) if isinstance(d, dict)]
# Raise if raw (non-dict) images/videos were silently filtered out.
# InternVL cannot process raw images without a text prompt because
# dynamic tiling and placeholder expansion require the prompt string.
raw_img_dropped = len(image_data or []) - len(images)
raw_vid_dropped = len(video_data or []) - len(videos)
if raw_img_dropped > 0 or raw_vid_dropped > 0:
raise ValueError(
f"[internvl] Cannot process raw images/videos with pre-tokenized "
f"input_ids. Provide multimodal data in 'processor_output' or "
f"'precomputed_embedding' format, or use a text prompt instead. "
f"(raw images dropped: {raw_img_dropped}, "
f"raw videos dropped: {raw_vid_dropped})"
)
base_output = BaseMultiModalProcessorOutput(
input_text=prompt,
images=images,
videos=videos,
)
else:
base_output = self.load_mm_data(
prompt=prompt,
image_data=image_data,
video_data=video_data,
multimodal_tokens=self.mm_tokens,
discard_alpha_channel=True,
)
mm_items, input_ids_tensor, ret = self.process_and_combine_mm_data(
base_output, self.mm_tokens
)
# If user provided input_ids directly, use those and recompute offsets
if user_input_ids is not None:
input_ids_tensor = torch.tensor(user_input_ids, dtype=torch.long)
for mm_item in mm_items:
if (
mm_item.modality == Modality.VIDEO
and self.video_token_id is not None
):
mm_token_id = self.video_token_id
else:
mm_token_id = self.img_context_token_id
mm_item.offsets = self.get_mm_items_offset(
input_ids=input_ids_tensor,
mm_token_id=mm_token_id,
)
return {
"input_ids": input_ids_tensor.flatten().tolist(),
"mm_items": mm_items,
"im_start_id": self.img_start_token_id,
"im_end_id": self.img_end_token_id,
"im_token_id": self.img_context_token_id,
"video_token_id": self.video_token_id,
}
async def process_mm_data_async(
self, image_data, input_text, request_obj, **kwargs
):
video_data = getattr(request_obj, "video_data", None) or []
# Handle processor_output and precomputed_embedding formats
if isinstance(input_text, list) or self._has_special_format(
image_data, video_data
):
return await self._process_special_format(
image_data=image_data,
video_data=video_data,
input_text=input_text,
request_obj=request_obj,
**kwargs,
)
is_internlm2 = self.llm_arch == "InternLM2ForCausalLM"

153
test/registered/vlm/test_vlm_input_format.py
View File

@@ -349,5 +349,158 @@ class TestKimiVLImageUnderstandsImage(
# return dict(processor_output, format="processor_output")
class TestInternVLUnderstandsImage(VLMInputTestBase, unittest.IsolatedAsyncioTestCase):
model_path = "OpenGVLab/InternVL2-2B"
chat_template = "internvl-2-5"
@classmethod
def setUpClass(cls):
assert cls.model_path is not None, "Set model_path in subclass"
assert cls.chat_template is not None, "Set chat_template in subclass"
cls.image_urls = [IMAGE_MAN_IRONING_URL, IMAGE_SGL_LOGO_URL]
cls.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
cls.main_image = []
for image_url in cls.image_urls:
response = requests.get(image_url)
cls.main_image.append(Image.open(BytesIO(response.content)))
# InternVL models (2, 3, 3.5) do not ship a standard HuggingFace
# Processor; AutoProcessor.from_pretrained returns a bare tokenizer.
# Use AutoTokenizer explicitly so the intent is clear.
from transformers import AutoTokenizer
cls.processor = AutoTokenizer.from_pretrained(
cls.model_path, trust_remote_code=True
)
cls._init_visual()
@classmethod
def _init_visual(cls):
model = AutoModel.from_pretrained(
cls.model_path, trust_remote_code=True, torch_dtype=torch.bfloat16
)
cls.vision_model = model.vision_model.eval().to(cls.device)
cls.mlp1 = model.mlp1.eval().to(cls.device)
config = model.config
cls.internvl_config = config
image_size = getattr(config, "force_image_size", None) or (
config.vision_config.image_size
)
patch_size = config.vision_config.patch_size
cls.num_image_token = int(
(image_size // patch_size) ** 2 * (config.downsample_ratio**2)
)
cls.internvl_image_size = image_size
cls.internvl_downsample_ratio = config.downsample_ratio
cls.internvl_ps_version = config.ps_version
cls.internvl_select_layer = config.select_layer
del model
def pixel_shuffle(x, scale_factor):
n, w, h, c = x.size()
x = x.view(n, w, int(h * scale_factor), int(c / scale_factor))
x = x.permute(0, 2, 1, 3).contiguous()
x = x.view(
n,
int(h * scale_factor),
int(w * scale_factor),
int(c / (scale_factor * scale_factor)),
)
if cls.internvl_ps_version != "v1":
x = x.permute(0, 2, 1, 3).contiguous()
return x
def visual_func(processor_output):
pixel_values = processor_output["pixel_values"].to(
cls.device, dtype=torch.bfloat16
)
if cls.internvl_select_layer == -1:
vit_embeds = cls.vision_model(
pixel_values=pixel_values,
output_hidden_states=False,
return_dict=True,
).last_hidden_state
else:
vit_embeds = cls.vision_model(
pixel_values=pixel_values,
output_hidden_states=True,
return_dict=True,
).hidden_states[cls.internvl_select_layer]
vit_embeds = vit_embeds[:, 1:, :]
h = w = int(vit_embeds.shape[1] ** 0.5)
vit_embeds = vit_embeds.reshape(vit_embeds.shape[0], h, w, -1)
vit_embeds = pixel_shuffle(
vit_embeds, scale_factor=cls.internvl_downsample_ratio
)
vit_embeds = vit_embeds.reshape(
vit_embeds.shape[0], -1, vit_embeds.shape[-1]
)
vit_embeds = cls.mlp1(vit_embeds)
return vit_embeds
cls.visual = visual_func
def get_processor_output(self, req=None):
"""Override to handle InternVL's custom preprocessing.
Uses shared ``image_to_pixel_values`` from ``internvl_utils`` for
image preprocessing (dynamic tiling + normalize) and expands
``<IMG_CONTEXT>`` placeholders into ``<img>`` + context tokens +
``</img>`` — mirroring the logic in
``InternVLProcessor.process_internlm2_mm_data_async``.
"""
from sglang.srt.multimodal.internvl_utils import image_to_pixel_values
from sglang.srt.multimodal.processors.internvl import InternVLProcessor
if req is None:
req = self.get_completion_request()
conv = generate_chat_conv(req, template_name=self.chat_template)
text = conv.get_prompt()
# Preprocess images using the shared utility (dynamic tiling +
# bicubic resize + ImageNet normalize), same pipeline as the engine.
all_pixel_values = []
num_patches_list = []
for img in self.main_image:
pv = image_to_pixel_values(
img,
input_size=self.internvl_image_size,
max_num_tiles=InternVLProcessor.IMAGE_MAX_NUM,
use_thumbnail=True,
)
all_pixel_values.append(pv)
num_patches_list.append(pv.shape[0])
pixel_values = torch.cat(all_pixel_values, dim=0).to(self.device)
# Expand each <IMG_CONTEXT> placeholder into <img> + <IMG_CONTEXT>*N + </img>.
# This mirrors InternVLProcessor.process_internlm2_mm_data_async.
ph = "<<<__IMG_PH__>>>"
expanded_text = text.replace(InternVLProcessor.IMG_CONTEXT, ph)
for num_patches in num_patches_list:
image_tokens = (
InternVLProcessor.IMG_START
+ InternVLProcessor.IMG_CONTEXT * (self.num_image_token * num_patches)
+ InternVLProcessor.IMG_END
)
expanded_text = expanded_text.replace(ph, image_tokens, 1)
# Remove any remaining placeholders (more placeholders than images)
expanded_text = expanded_text.replace(ph, "")
# Tokenize the expanded text
input_ids = self.processor(expanded_text, return_tensors="pt")["input_ids"]
return {
"input_ids": input_ids,
"pixel_values": pixel_values,
}, text
def _processor_output_image_data(self, processor_output):
return dict(processor_output, format="processor_output")
if __name__ == "__main__":
unittest.main()