chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[VLM] Support Video for InternVL3_5 (#15942)

Browse Source 
Co-authored-by: luoyuan.luo <luoyuan.luo@antgroup.com>
This commit is contained in:
Yuan Luo
2025-12-30 17:07:49 +08:00
committed by GitHub
parent db3821a9ef
commit 94bcc19bce
2 changed files with 432 additions and 124 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
python/sglang/srt/models/internvl.py
View File

@@ -539,6 +539,7 @@ class InternVLChatModel(nn.Module):
self.external_mm_data_embedding_funcs = {
Modality.IMAGE: self.get_image_feature,
Modality.VIDEO: self.get_video_feature,
}
self.model = self.language_model.model
@@ -594,6 +595,13 @@ class InternVLChatModel(nn.Module):
image_features = self.extract_feature(pixel_values)
return image_features
def get_video_feature(self, items: List[MultimodalDataItem]):
# items: each item corresponds to one video (recommended)
# item.feature shape: [num_frames, 3, 448, 448] (or [num_tiles, 3, 448, 448])
pixel_values = torch.cat([item.feature for item in items], dim=0)
video_features = self.extract_feature(pixel_values)
return video_features
@torch.no_grad()
def forward(
self,

548
python/sglang/srt/multimodal/processors/internvl.py
View File

@@ -1,6 +1,8 @@
# Adapted from https://huggingface.co/OpenGVLab/InternVL2-4B/blob/main/modeling_intern_vit.py
import logging
from functools import lru_cache
from typing import List
import numpy as np
import torch
@@ -15,26 +17,44 @@ from sglang.srt.multimodal.processors.base_processor import (
MultimodalSpecialTokens,
)
logger = logging.getLogger(__name__)
class InternVLImageProcessor(BaseMultimodalProcessor):
class InternVLProcessor(BaseMultimodalProcessor):
models = [InternVLChatModel, InternS1ForConditionalGeneration]
IMAGENET_MEAN = [0.485, 0.456, 0.406]
IMAGENET_STD = [0.229, 0.224, 0.225]
DEFAULT_VIDEO_NUM_FRAMES = 32
VIDEO_MAX_NUM = 1
VIDEO_USE_THUMBNAIL = False
CONTEXT_FALLBACK = 40960
CONTEXT_RESERVED = 256
# OpenAI multimodal placeholder tokens
IMAGE_PLACEHOLDER_TOKEN = "<image>"
VIDEO_PLACEHOLDER_TOKEN = "<video>"
IMG_START = "<img>"
IMG_END = "</img>"
IMG_CONTEXT = "<IMG_CONTEXT>"
@staticmethod
@lru_cache(maxsize=1)
def _get_normalize_tensors(device="cuda", dtype=torch.float32):
mean = torch.tensor(
InternVLImageProcessor.IMAGENET_MEAN, device=device, dtype=dtype
InternVLProcessor.IMAGENET_MEAN, device=device, dtype=dtype
).view(-1, 1, 1)
std = torch.tensor(
InternVLImageProcessor.IMAGENET_STD, device=device, dtype=dtype
InternVLProcessor.IMAGENET_STD, device=device, dtype=dtype
).view(-1, 1, 1)
return mean, std
def __init__(self, hf_config, server_args, _image_processor, *args, **kwargs):
super().__init__(hf_config, server_args, _image_processor, *args, **kwargs)
image_size = (
getattr(hf_config, "force_image_size", None)
or hf_config.vision_config.image_size
@@ -45,88 +65,65 @@ class InternVLImageProcessor(BaseMultimodalProcessor):
if isinstance(patch_size, list):
patch_size = patch_size[0]
self.IMG_CONTEXT_TOKEN = "<IMG_CONTEXT>"
self.IMG_START_TOKEN = "<img>"
self.IMG_END_TOKEN = "</img>"
self.num_image_token = int(
(image_size // patch_size) ** 2 * (hf_config.downsample_ratio**2)
)
if hasattr(self._processor, "tokenizer"):
tokenizer = self._processor.tokenizer
else:
tokenizer = self._processor
self.tokenizer = tokenizer
self.img_start_token_id = tokenizer.convert_tokens_to_ids(self.IMG_START_TOKEN)
self.img_end_token_id = tokenizer.convert_tokens_to_ids(self.IMG_END_TOKEN)
llm_arch = hf_config.llm_config.architectures[0]
self.llm_arch = llm_arch
video_token_map = {
"Qwen2ForCausalLM": "<|video_pad|>",
"Qwen3ForCausalLM": "<|video_pad|>",
"Qwen3MoeForCausalLM": "<|video_pad|>",
"GptOssForCausalLM": "<|reserved_200000|>",
}
self.VIDEO_CONTEXT_TOKEN = video_token_map.get(llm_arch, None)
self.video_token_id = (
tokenizer.convert_tokens_to_ids(self.VIDEO_CONTEXT_TOKEN)
if self.VIDEO_CONTEXT_TOKEN
else None
)
self.num_image_token = int(
(image_size // patch_size) ** 2 * (hf_config.downsample_ratio**2)
)
self.img_start_token_id = tokenizer.convert_tokens_to_ids(self.IMG_START)
self.img_end_token_id = tokenizer.convert_tokens_to_ids(self.IMG_END)
# Placeholder token use <image>/<video>
# Offset token id use IMG_CONTEXT / VIDEO_CONTEXT
self.mm_tokens = MultimodalSpecialTokens(
image_token="<IMG_CONTEXT>",
image_token_id=tokenizer.convert_tokens_to_ids(self.IMG_CONTEXT_TOKEN),
image_token=self.IMAGE_PLACEHOLDER_TOKEN,
image_token_id=tokenizer.convert_tokens_to_ids(self.IMG_CONTEXT),
video_token=self.VIDEO_PLACEHOLDER_TOKEN,
video_token_id=self.video_token_id,
).build(_image_processor)
@staticmethod
def get_index(bound, fps, max_frame, first_idx=0, num_segments=32):
if bound:
start, end = bound[0], bound[1]
else:
start, end = -100000, 100000
start_idx = max(first_idx, round(start * fps))
end_idx = min(round(end * fps), max_frame)
seg_size = float(end_idx - start_idx) / num_segments
frame_indices = np.array(
[
int(start_idx + (seg_size / 2) + np.round(seg_size * idx))
for idx in range(num_segments)
]
)
return frame_indices
# Cache token id for IMG_CONTEXT (used by both branches)
self.img_context_token_id = tokenizer.convert_tokens_to_ids(self.IMG_CONTEXT)
@staticmethod
def load_video(video_path, bound=None, input_size=448, max_num=1, num_segments=32):
try:
vr = VideoReader(video_path, ctx=gpu(0), num_threads=1)
use_gpu = True
except (RuntimeError, OSError) as e:
print(
f"[WARNING] Load video on gpu decoding failed: {e}. Falling back to CPU."
# InternLM2 legacy multimodal tokens: use <IMG_CONTEXT> as placeholder
self.mm_tokens_internlm2 = MultimodalSpecialTokens(
image_token=self.IMG_CONTEXT,
image_token_id=self.img_context_token_id,
).build(_image_processor)
self.max_context_len = (
getattr(server_args, "context_length", None)
or getattr(server_args, "max_context_len", None)
or getattr(hf_config, "max_position_embeddings", None)
or getattr(
getattr(hf_config, "llm_config", None), "max_position_embeddings", None
)
vr = VideoReader(video_path, ctx=cpu(0), num_threads=1)
use_gpu = False
max_frame = len(vr) - 1
fps = float(vr.get_avg_fps())
pixel_values_list = []
num_patches_list = []
frame_indices = InternVLImageProcessor.get_index(
bound, fps, max_frame, first_idx=0, num_segments=num_segments
or self.CONTEXT_FALLBACK
)
mean, std = InternVLImageProcessor._get_normalize_tensors(device="cuda")
for frame_index in frame_indices:
# Load frame
frame = vr[frame_index]
if use_gpu:
img = frame.cuda().permute(2, 0, 1).float() / 255.0
else:
img_np = frame.asnumpy()
img = torch.from_numpy(img_np).permute(2, 0, 1).cuda().float() / 255.0
img = (img - mean) / std
tiles = InternVLImageProcessor.dynamic_preprocess(
img, image_size=input_size, max_num=max_num, use_thumbnail=True
)
pixel_values_list.append(tiles)
num_patches_list.append(tiles.shape[0])
pixel_values = torch.cat(pixel_values_list, dim=0)
return pixel_values, num_patches_list
@staticmethod
def dynamic_preprocess(tensor, image_size=448, max_num=12, use_thumbnail=False):
# Tensor: (C,H,W) float on GPU
C, H, W = tensor.shape
aspect_ratio = W / H
@@ -187,69 +184,249 @@ class InternVLImageProcessor(BaseMultimodalProcessor):
return torch.stack(tiles).to(torch.bfloat16)
@staticmethod
def _open_video_reader(path: str) -> VideoReader:
try:
return VideoReader(path, ctx=gpu(0), num_threads=1)
except (RuntimeError, OSError) as e:
logger.warning(
"[internvl] VideoReader gpu decode failed (%s), fallback CPU", e
)
return VideoReader(path, ctx=cpu(0), num_threads=1)
def _ensure_placeholders_before_assistant(
self, prompt: str, placeholder: str, want: int
) -> str:
if want <= 0:
return prompt
have = (prompt or "").count(placeholder)
missing = want - have
if missing <= 0:
return prompt
insert = "\n" + "\n".join([placeholder] * missing) + "\n"
marker = "<|im_start|>assistant"
idx = (prompt or "").rfind(marker)
if idx != -1:
return (prompt or "")[:idx] + insert + (prompt or "")[idx:]
return (prompt or "") + insert
def _token_len(self, text: str) -> int:
try:
ids = self.tokenizer(text, return_tensors="pt")["input_ids"].flatten()
return int(ids.numel())
except Exception:
return 0
def _resolve_video_num_frames(
self, *, requested: int, num_videos: int, text_len: int, image_tile_cnt: int
) -> int:
if num_videos <= 0:
return 0
if not self.VIDEO_CONTEXT_TOKEN or not self.video_token_id:
return 0
image_tokens = image_tile_cnt * self.num_image_token
budget = (
int(self.max_context_len)
- int(text_len)
- int(image_tokens)
- int(self.CONTEXT_RESERVED)
)
if budget <= 0:
return 1
max_total_frames = max(1, budget // self.num_image_token)
frames_per_video = max(1, max_total_frames // max(num_videos, 1))
return max(1, min(int(requested), int(frames_per_video)))
async def process_mm_data_async(
self, image_data, input_text, request_obj, **kwargs
):
is_internlm2 = self.llm_arch == "InternLM2ForCausalLM"
if is_internlm2:
return await self.process_internlm2_mm_data_async(
image_data=image_data,
input_text=input_text,
request_obj=request_obj,
**kwargs,
)
else:
# Default branch uses OpenAI-style placeholders
return await self.process_qwen_mm_data_async(
image_data=image_data,
input_text=input_text,
request_obj=request_obj,
**kwargs,
)
async def process_qwen_mm_data_async(
self, image_data, input_text, request_obj, **kwargs
):
# Qwen/Qwen3 branch: OpenAI-style placeholders <image>/<video>
prompt = input_text or ""
video_data = getattr(request_obj, "video_data", None) or []
if image_data:
prompt = self._ensure_placeholders_before_assistant(
prompt, self.IMAGE_PLACEHOLDER_TOKEN, len(image_data)
)
if video_data:
prompt = self._ensure_placeholders_before_assistant(
prompt, self.VIDEO_PLACEHOLDER_TOKEN, len(video_data)
)
logger.info(
"[internvl][qwen] placeholders image=%d video=%d",
prompt.count(self.IMAGE_PLACEHOLDER_TOKEN),
prompt.count(self.VIDEO_PLACEHOLDER_TOKEN),
)
base_output = self.load_mm_data(
prompt=input_text,
prompt=prompt,
image_data=image_data,
multimodal_tokens=self.mm_tokens,
video_data=video_data,
multimodal_tokens=self.mm_tokens, # expects <image>/<video>
discard_alpha_channel=True,
)
num_patches_list = []
pixel_values = []
logger.info(
"[internvl][qwen] loaded images=%d videos=%d",
len(base_output.images),
len(base_output.videos),
)
mean, std = InternVLImageProcessor._get_normalize_tensors(device="cuda")
mean, std = self._get_normalize_tensors(device="cuda")
# Process each input with allocated frames
for image_index, image in enumerate(base_output.images):
try:
# TODO: video input
# Convert PIL to GPU tensor
if isinstance(image, Image.Image):
img_np = np.array(image.convert("RGB"))
tensor = (
torch.from_numpy(img_np).permute(2, 0, 1).cuda().float() / 255.0
)
else:
tensor = image.cuda() # assume already tensor
# ----- Images -> tiles -----
num_patches_list: List[int] = []
pixel_values_list: List[torch.Tensor] = []
tensor = (tensor - mean) / std
tiles = self.dynamic_preprocess(
tensor, image_size=448, max_num=12, use_thumbnail=True
for image in base_output.images:
if isinstance(image, Image.Image):
img_np = np.array(image.convert("RGB"))
tensor = (
torch.from_numpy(img_np).permute(2, 0, 1).cuda().float() / 255.0
)
else:
tensor = image.cuda()
tensor = (tensor - mean) / std
tiles = self.dynamic_preprocess(
tensor, image_size=448, max_num=12, use_thumbnail=True
)
pixel_values_list.append(tiles)
num_patches_list.append(int(tiles.shape[0]))
if image_data and not pixel_values_list:
raise ValueError(
"[internvl][qwen] image_data provided but no images parsed from prompt placeholders"
)
image_tensor = (
torch.cat(pixel_values_list, dim=0) if pixel_values_list else None
)
# ----- Videos -> frame tiles (optional) -----
video_tensor = None
video_patch_lists = []
video_pixel_values = []
requested_frames = int(
kwargs.get("video_num_frames", self.DEFAULT_VIDEO_NUM_FRAMES)
)
num_frames = self._resolve_video_num_frames(
requested=requested_frames,
num_videos=len(base_output.videos),
text_len=self._token_len(base_output.input_text or prompt),
image_tile_cnt=int(sum(num_patches_list)) if num_patches_list else 0,
)
if base_output.videos and num_frames > 0 and self.video_token_id is not None:
for video in base_output.videos:
vr = (
video
if isinstance(video, VideoReader)
else self._open_video_reader(str(video))
)
max_frame = len(vr) - 1
frame_indices = (
[0]
if num_frames == 1
else np.linspace(0, max_frame, num=num_frames, dtype=int).tolist()
)
pixel_values.append(tiles)
num_patches_list.append(tiles.shape[0])
per_video_tiles = []
per_video_patch_cnt = []
for fi in frame_indices:
frame = vr[int(fi)]
img_np = (
frame.asnumpy()
if hasattr(frame, "asnumpy")
else np.array(frame)
)
frame_t = (
torch.from_numpy(img_np).permute(2, 0, 1).cuda().float() / 255.0
)
frame_t = (frame_t - mean) / std
except Exception as e:
print(f"[Error] Failed to process image {image_index}: {e}")
return None
tiles = self.dynamic_preprocess(
frame_t,
image_size=448,
max_num=self.VIDEO_MAX_NUM,
use_thumbnail=self.VIDEO_USE_THUMBNAIL,
)
per_video_tiles.append(tiles)
per_video_patch_cnt.append(int(tiles.shape[0]))
# Concatenate all
pixel_values = torch.cat(pixel_values, dim=0)
pv = torch.cat(per_video_tiles, dim=0)
video_pixel_values.append(pv)
video_patch_lists.append(per_video_patch_cnt)
original_placeholder = "<<<__IMG_CONTEXT_PLACEHOLDER__>>>"
video_tensor = (
torch.cat(video_pixel_values, dim=0) if video_pixel_values else None
)
input_text = base_output.input_text.replace(
self.IMG_CONTEXT_TOKEN, original_placeholder
)
# ----- Build prompt text with <img> + CONTEXT*n + </img> -----
img_ph = "<<<__IMG_PLACEHOLDER__>>>"
vid_ph = "<<<__VID_PLACEHOLDER__>>>"
input_text_updated = input_text
input_text_mid = base_output.input_text or prompt
input_text_mid = input_text_mid.replace(self.IMAGE_PLACEHOLDER_TOKEN, img_ph)
if self.VIDEO_CONTEXT_TOKEN and self.video_token_id is not None:
input_text_mid = input_text_mid.replace(
self.VIDEO_PLACEHOLDER_TOKEN, vid_ph
)
else:
input_text_mid = input_text_mid.replace(self.VIDEO_PLACEHOLDER_TOKEN, "")
input_text_updated = input_text_mid
# Expand images
for num_patches in num_patches_list:
image_tokens = (
self.IMG_START_TOKEN
+ self.IMG_CONTEXT_TOKEN * self.num_image_token * num_patches
+ self.IMG_END_TOKEN
)
input_text_updated = input_text_updated.replace(
original_placeholder, image_tokens, 1
self.IMG_START
+ (self.IMG_CONTEXT * (self.num_image_token * int(num_patches)))
+ self.IMG_END
)
input_text_updated = input_text_updated.replace(img_ph, image_tokens, 1)
input_text_updated = input_text_updated.replace(
original_placeholder, self.IMG_CONTEXT_TOKEN
)
# Expand videos (each frame is one <img>...</img>)
if video_patch_lists and self.VIDEO_CONTEXT_TOKEN:
for frame_patch_list in video_patch_lists:
frame_lines = []
for i, patch_cnt in enumerate(frame_patch_list):
ctx_cnt = int(self.num_image_token) * int(patch_cnt)
frame_tokens = (
self.IMG_START
+ (self.VIDEO_CONTEXT_TOKEN * ctx_cnt)
+ self.IMG_END
)
frame_lines.append(f"Frame {i+1}: {frame_tokens}")
video_tokens = "\n".join(frame_lines) + "\n"
input_text_updated = input_text_updated.replace(vid_ph, video_tokens, 1)
# Tokenize
input_ids_tensor = self.tokenizer(input_text_updated, return_tensors="pt")[
@@ -257,24 +434,147 @@ class InternVLImageProcessor(BaseMultimodalProcessor):
].flatten()
input_ids = input_ids_tensor.tolist()
# Get image token offsets
image_offsets = self.get_mm_items_offset(
input_ids=input_ids_tensor.to("cuda"),
mm_token_id=self.mm_tokens.image_token_id,
)
items = [
MultimodalDataItem(
feature=pixel_values,
modality=Modality.IMAGE,
offsets=image_offsets,
# Offsets
image_offsets = []
if image_tensor is not None:
image_offsets = self.get_mm_items_offset(
input_ids=input_ids_tensor.to("cuda"),
mm_token_id=self.img_context_token_id,
)
video_offsets = []
if video_tensor is not None and self.video_token_id is not None:
video_offsets = self.get_mm_items_offset(
input_ids=input_ids_tensor.to("cuda"),
mm_token_id=self.video_token_id,
)
items = []
if image_tensor is not None:
items.append(
MultimodalDataItem(
feature=image_tensor, modality=Modality.IMAGE, offsets=image_offsets
)
)
if video_tensor is not None:
items.append(
MultimodalDataItem(
feature=video_tensor, modality=Modality.VIDEO, offsets=video_offsets
)
)
]
return {
"input_ids": input_ids,
"mm_items": items,
"im_start_id": self.img_start_token_id,
"im_end_id": self.img_end_token_id,
"im_token_id": self.mm_tokens.image_token_id,
"im_token_id": self.img_context_token_id,
"video_token_id": self.video_token_id,
}
async def process_internlm2_mm_data_async(
self, image_data, input_text, request_obj, **kwargs
):
# InternLM2 branch: legacy placeholder <IMG_CONTEXT> (stable for InternLM2 prompt behavior)
prompt = input_text or ""
video_data = getattr(request_obj, "video_data", None) or []
if video_data:
logger.warning(
"[internvl][internlm2] video input ignored for InternLM2 branch"
)
# Convert any OpenAI-style <image> into <IMG_CONTEXT>
prompt = prompt.replace(self.IMAGE_PLACEHOLDER_TOKEN, self.IMG_CONTEXT)
if image_data:
prompt = self._ensure_placeholders_before_assistant(
prompt, self.IMG_CONTEXT, len(image_data)
)
logger.info(
"[internvl][internlm2] placeholders img_context=%d",
prompt.count(self.IMG_CONTEXT),
)
base_output = self.load_mm_data(
prompt=prompt,
image_data=image_data,
multimodal_tokens=self.mm_tokens_internlm2, # expects <IMG_CONTEXT>
discard_alpha_channel=True,
)
mean, std = self._get_normalize_tensors(device="cuda")
num_patches_list: List[int] = []
pixel_values_list: List[torch.Tensor] = []
for image in base_output.images:
if isinstance(image, Image.Image):
img_np = np.array(image.convert("RGB"))
tensor = (
torch.from_numpy(img_np).permute(2, 0, 1).cuda().float() / 255.0
)
else:
tensor = image.cuda()
tensor = (tensor - mean) / std
tiles = self.dynamic_preprocess(
tensor, image_size=448, max_num=12, use_thumbnail=True
)
pixel_values_list.append(tiles)
num_patches_list.append(int(tiles.shape[0]))
if image_data and not pixel_values_list:
raise ValueError(
"[internvl][internlm2] image_data provided but no images parsed from prompt placeholders"
)
pixel_values = (
torch.cat(pixel_values_list, dim=0) if pixel_values_list else None
)
# Expand each <IMG_CONTEXT> into <img> + <IMG_CONTEXT>*N + </img>
ph = "<<<__IMG_CONTEXT_PLACEHOLDER__>>>"
input_text_base = (base_output.input_text or prompt).replace(
self.IMG_CONTEXT, ph
)
input_text_updated = input_text_base
for num_patches in num_patches_list:
image_tokens = (
self.IMG_START
+ (self.IMG_CONTEXT * (self.num_image_token * int(num_patches)))
+ self.IMG_END
)
input_text_updated = input_text_updated.replace(ph, image_tokens, 1)
# Tokenize
input_ids_tensor = self.tokenizer(input_text_updated, return_tensors="pt")[
"input_ids"
].flatten()
input_ids = input_ids_tensor.tolist()
# Offsets
image_offsets = []
if pixel_values is not None:
image_offsets = self.get_mm_items_offset(
input_ids=input_ids_tensor.to("cuda"),
mm_token_id=self.img_context_token_id,
)
items = []
if pixel_values is not None:
items.append(
MultimodalDataItem(
feature=pixel_values, modality=Modality.IMAGE, offsets=image_offsets
)
)
return {
"input_ids": input_ids,
"mm_items": items,
"im_start_id": self.img_start_token_id,
"im_end_id": self.img_end_token_id,
"im_token_id": self.img_context_token_id,
"video_token_id": self.video_token_id,
}