chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

vlm: refactor engine vlm params and support processor output as input (#14091)

Browse Source 
Co-authored-by: Mick <mickjagger19@icloud.com>
Co-authored-by: zhaochenyang20 <zhaochenyang20@gmail.com>
Co-authored-by: Xinyuan Tong <115166877+JustinTong0323@users.noreply.github.com>
Co-authored-by: BenYao21 <cyao22@asu.edu>
Co-authored-by: minleminzui <minleminzui@gmail.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Co-authored-by: 赵晨阳 <zhaochen20@outlook.com>
This commit is contained in:
mlmz
2025-12-20 18:31:24 +08:00
committed by GitHub
co-authored by Mick zhaochenyang20 Xinyuan Tong BenYao21 minleminzui gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> 赵晨阳
parent 165f5c04cb
commit 1f1f05a85e
16 changed files with 783 additions and 305 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/multimodal/processors/base_processor.py
+158 -48
View File
@@ -12,9 +12,12 @@ import torch
from PIL import Image
from transformers import BaseImageProcessorFast
from sglang.srt.environ import envs
from sglang.srt.managers.schedule_batch import Modality, MultimodalDataItem
from sglang.srt.utils import is_npu, load_audio, load_image, load_video, logger
from sglang.srt.managers.schedule_batch import (
Modality,
MultimodalDataItem,
MultimodalInputFormat,
)
from sglang.srt.utils import envs, is_npu, load_audio, load_image, load_video, logger
from sglang.srt.utils.cuda_ipc_transport_utils import (
MM_FEATURE_CACHE_SIZE,
MM_ITEM_MEMORY_POOL_RECYCLE_INTERVAL,
@@ -29,7 +32,7 @@ SGL_USE_CUDA_IPC = envs.SGLANG_USE_CUDA_IPC_TRANSPORT.get()
@dataclasses.dataclass
class BaseMultiModalProcessorOutput:
# input_text, with each frame of video/image represented with a image_token
# input_text with all multimodality placeholder token expanded
input_text: str
# frames loaded from image, in given order
@@ -385,11 +388,18 @@ class BaseMultimodalProcessor(ABC):
"""
Load a single multimodal data.
If data is precomputed, returns directly.
If data is processor_output or precomputed embedding, return directly.
Static method that can be pickled for multiprocessing"""
if isinstance(data, dict):
return data
data_format = data.get("format")
if data_format in (
MultimodalInputFormat.PROCESSOR_OUTPUT.name,
MultimodalInputFormat.PRECOMPUTED_EMBEDDING.name,
"processor_output",
"precomputed_embedding",
):
return data
try:
if modality == Modality.IMAGE:
img, _ = load_image(data)
@@ -431,9 +441,10 @@ class BaseMultimodalProcessor(ABC):
try:
data = next(data_iterator)
except StopIteration:
raise ValueError(
f"Mismatch: More '{text_part}' tokens found than corresponding data items provided."
logger.warning(
f"Mismatch: More '{modality.name}' tokens found than corresponding data provided."
)
return futures, task_info
frame_count_limit = None
if modality == Modality.IMAGE and image_estimated_frames_iter:
@@ -475,6 +486,77 @@ class BaseMultimodalProcessor(ABC):
return futures, task_info
@staticmethod
def _validate_one_modality(modality: Modality, data_list: Optional[list]):
if data_list is None:
return
if not isinstance(data_list, list):
raise TypeError(
f"{modality.name} must be a list or None, got {type(data_list)}"
)
formatted_indices = []
for idx, item in enumerate(data_list):
if isinstance(item, dict):
fmt = item.get("format")
if fmt in {"processor_output", "precomputed_embedding"}:
formatted_indices.append(idx)
if formatted_indices:
if len(data_list) != 1:
raise ValueError(
f"For {modality}, when providing a 'processor_output' or "
f"'precomputed_embedding', you must pass exactly one item; "
f"received {len(data_list)} items (formatted at indices {formatted_indices})."
)
@staticmethod
def validate_mm_data(
image_data: Optional[list] = None,
video_data: Optional[list] = None,
audio_data: Optional[list] = None,
):
"""
Validate multimodal input lists per modality.
Rule per modality (image/video/audio):
- Either the list has exactly one item and that single item is a dict with
format in {"processor_output", "precomputed_embedding"};
- Or, the list contains only "normal" items (i.e., does not include any
item whose format is one of the two above).
Empty or None lists are considered valid.
"""
BaseMultimodalProcessor._validate_one_modality(Modality.IMAGE, image_data)
BaseMultimodalProcessor._validate_one_modality(Modality.VIDEO, video_data)
BaseMultimodalProcessor._validate_one_modality(Modality.AUDIO, audio_data)
def _process_loaded_mm_data(self, modality, raw_data, result):
images, videos, audios = [], [], []
is_precomputed = isinstance(raw_data, dict) and raw_data.get("format") in [
MultimodalInputFormat.PROCESSOR_OUTPUT.name,
MultimodalInputFormat.PRECOMPUTED_EMBEDDING.name,
"processor_output",
"precomputed_embedding",
]
if modality == Modality.IMAGE:
if is_precomputed:
images.append(result)
else:
if isinstance(result, list):
images.extend(result)
else:
images.append(result)
elif modality == Modality.VIDEO:
videos.append(result)
elif modality == Modality.AUDIO:
audios.append(result)
return is_precomputed, images, videos, audios
def load_mm_data(
self,
prompt: str,
@@ -495,8 +577,10 @@ class BaseMultimodalProcessor(ABC):
discard_alpha_channel: if True, discards the alpha channel in the returned images
"""
multimodal_tokens_pattern = multimodal_tokens.get_combined_regex()
BaseMultimodalProcessor.validate_mm_data(image_data, video_data, audio_data)
multimodal_tokens_pattern = multimodal_tokens.get_combined_regex()
if isinstance(prompt, list) and return_text:
assert len(prompt) and isinstance(prompt[0], int)
prompt = self._processor.tokenizer.decode(prompt)
@@ -506,7 +590,6 @@ class BaseMultimodalProcessor(ABC):
assert isinstance(prompt, str)
# split text into list of normal text and special tokens
text_parts = re.split(multimodal_tokens_pattern, prompt)
# collect all data
data_iterators = {}
if multimodal_tokens.image_token and image_data:
@@ -531,29 +614,31 @@ class BaseMultimodalProcessor(ABC):
# Process results
images, videos, audios = [], [], []
new_text_parts = []
has_precomputed_input = False
for text_part in text_parts:
try:
if multimodal_tokens_pattern.match(text_part):
modality, raw_data, frame_limit = next(task_info_iter)
is_precomputed = isinstance(raw_data, dict)
result = next(futures_iter).result()
is_precomputed, new_imgs, new_vids, new_auds = (
self._process_loaded_mm_data(modality, raw_data, result)
)
has_precomputed_input |= is_precomputed
images.extend(new_imgs)
videos.extend(new_vids)
audios.extend(new_auds)
if modality == Modality.IMAGE:
# If data is already processed it will be a
# dictionary(precomputed). In this case we want to keep the
# expanded tokens in text_part. Otherwise, we will
# call the processor code, so keep only a single image
# token.
mm_tokens = (
text_part
if is_precomputed
else multimodal_tokens.image_token
)
frames = [result] if not isinstance(result, list) else result
if frames:
# only for minicpmv
images += frames
new_text_parts += mm_tokens * len(frames)
if is_precomputed:
new_text_parts += [text_part]
else:
count = len(new_imgs)
if count > 0:
new_text_parts += [
multimodal_tokens.image_token
] * count
elif modality == Modality.VIDEO:
# load as video
mm_tokens = (
@@ -561,7 +646,6 @@ class BaseMultimodalProcessor(ABC):
if is_precomputed
else multimodal_tokens.video_token
)
videos += [result]
new_text_parts += mm_tokens
elif modality == Modality.AUDIO:
# audio
@@ -570,12 +654,19 @@ class BaseMultimodalProcessor(ABC):
if is_precomputed
else multimodal_tokens.audio_token
)
audios += [result]
new_text_parts += mm_tokens
else:
# normal text
new_text_parts += [text_part]
except StopIteration as e:
# when precomputed_input is presented with multi-images, StopIteration is expected
if has_precomputed_input:
new_text_parts += [text_part]
continue
raise RuntimeError(
f"An exception occurred while loading multimodal data: {e}"
)
except Exception as e:
raise RuntimeError(
f"An exception occurred while loading multimodal data: {e}"
@@ -601,7 +692,6 @@ class BaseMultimodalProcessor(ABC):
mask = input_ids == mm_token_id
start_positions = (mask & ~torch.roll(mask, 1)).nonzero(as_tuple=True)[0]
end_positions = (mask & ~torch.roll(mask, -1)).nonzero(as_tuple=True)[0]
return list(zip(start_positions.tolist(), end_positions.tolist()))
@staticmethod
@@ -614,35 +704,42 @@ class BaseMultimodalProcessor(ABC):
return list(zip(indices_start.tolist(), indices_end.tolist()))
def collect_mm_items_from_processor_output(
self, data_dict: dict
self, data_dict: dict, modality: Modality = None
) -> List[MultimodalDataItem]:
"""Create mm_items directly from processor output."""
"""
Create mm_items directly from processor output, with one item for each modality
Note that the data_dict can be passed via offline engine api
"""
items: dict[Modality, MultimodalDataItem] = {}
for attr_name, value in data_dict.items():
if attr_name == "input_ids":
continue
# Get modality for this attribute
modality = self.ATTR_NAME_TO_MODALITY.get(attr_name)
current_modality = modality or self.ATTR_NAME_TO_MODALITY.get(attr_name)
if attr_name == "precomputed_embeddings":
modality_str = data_dict.get("modality")
modality = Modality.IMAGE
current_modality = Modality.IMAGE
if modality_str:
try:
modality = Modality.from_str(modality_str)
current_modality = Modality.from_str(modality_str)
except ValueError:
pass
if modality:
if current_modality:
# Create item if needed
if modality not in items:
items[modality] = MultimodalDataItem(modality=modality)
if current_modality not in items:
items[current_modality] = MultimodalDataItem(
modality=current_modality
)
if attr_name in self.FEATURE_NAMES:
attr_name = "feature"
items[modality].set(attr_name, value)
items[current_modality].set(attr_name, value)
return list(items.values())
@@ -678,9 +775,9 @@ class BaseMultimodalProcessor(ABC):
Tuple of (list of mm_items, input_ids)
"""
# Collect all items and categorize them
all_items = base_output.organize_results()
all_loaded_data = base_output.organize_results()
# Handle text-only case
if not all_items:
if not all_loaded_data:
input_ids = self._processor.tokenizer(
base_output.input_text,
return_tensors="pt",
@@ -689,9 +786,9 @@ class BaseMultimodalProcessor(ABC):
return [], input_ids, {}
dict_items, raw_images, raw_audios, raw_videos = [], [], [], []
for modality, item in all_items:
for modality, item in all_loaded_data:
if isinstance(item, dict):
dict_items.append(item)
dict_items.append((modality, item))
elif modality == Modality.IMAGE:
raw_images.append(item)
elif modality == Modality.AUDIO:
@@ -717,12 +814,25 @@ class BaseMultimodalProcessor(ABC):
else:
ret = None
# Handle dict items (already processed)
for dict_item in dict_items:
all_collected_items.extend(
self.collect_mm_items_from_processor_output(dict_item)
)
# Handle dict items (processed or precomputed)
for modality, dict_item in dict_items:
input_format = dict_item.get("format", None)
if input_format == "processor_output":
items = self.collect_mm_items_from_processor_output(dict_item)
for item in items:
item.format = MultimodalInputFormat.PROCESSOR_OUTPUT
all_collected_items.extend(items)
elif input_format == "precomputed_embedding":
feature = dict_item["feature"]
del dict_item["feature"]
all_collected_items.append(
MultimodalDataItem(
modality=modality,
feature=feature,
format=MultimodalInputFormat.PRECOMPUTED_EMBEDDING,
model_specific_data=dict_item,
)
)
# Fallback tokenization if no raw items were processed
if input_ids is None:
input_ids = self._processor.tokenizer(