chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[vlm][internVL] Support processor and embedding inputs for InternVL (#19127)

Browse Source
This commit is contained in:
Ken J
2026-02-26 22:46:48 -08:00
committed by GitHub
parent 8a56cc5836
commit f0c2089597
4 changed files with 282 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
python/sglang/srt/multimodal/processors/base_processor.py
View File

@@ -182,6 +182,13 @@ class BaseMultimodalProcessor(ABC):
self.server_args = server_args
self.transport_mode = transport_mode
# Resolve tokenizer: some processors (e.g. InternVL) pass a tokenizer
# directly as _processor rather than a processor that wraps a tokenizer.
if hasattr(self._processor, "tokenizer"):
self._tokenizer = self._processor.tokenizer
else:
self._tokenizer = self._processor
# FIXME: not accurate, model and image specific
self.NUM_TOKEN_PER_FRAME = 330
@@ -252,7 +259,7 @@ class BaseMultimodalProcessor(ABC):
Use prompt and img_grid_thw to build input_ids
"""
if not isinstance(prompt, list):
prompt = self._processor.tokenizer.encode(prompt)
prompt = self._tokenizer.encode(prompt)
img_token_id = self.IM_TOKEN_ID
spatial_merge_size = self.spatial_merge_size
@@ -636,7 +643,7 @@ class BaseMultimodalProcessor(ABC):
multimodal_tokens_pattern = multimodal_tokens.get_combined_regex()
if isinstance(prompt, list) and return_text:
assert len(prompt) and isinstance(prompt[0], int)
prompt = self._processor.tokenizer.decode(prompt)
prompt = self._tokenizer.decode(prompt)
else:
prompt = prompt
@@ -709,7 +716,7 @@ class BaseMultimodalProcessor(ABC):
# Convert prompt into str
if isinstance(prompt, list) and return_text:
assert len(prompt) and isinstance(prompt[0], int)
prompt_str = self._processor.tokenizer.decode(prompt)
prompt_str = self._tokenizer.decode(prompt)
else:
assert isinstance(prompt, str)
prompt_str = prompt
@@ -793,7 +800,7 @@ class BaseMultimodalProcessor(ABC):
multimodal_tokens_pattern = multimodal_tokens.get_combined_regex()
if isinstance(prompt, list) and return_text:
assert len(prompt) and isinstance(prompt[0], int)
prompt = self._processor.tokenizer.decode(prompt)
prompt = self._tokenizer.decode(prompt)
else:
prompt = prompt
@@ -988,7 +995,7 @@ class BaseMultimodalProcessor(ABC):
all_loaded_data = base_output.organize_results()
# Handle text-only case
if not all_loaded_data:
input_ids = self._processor.tokenizer(
input_ids = self._tokenizer(
base_output.input_text,
return_tensors="pt",
add_special_tokens=True,
@@ -1044,7 +1051,7 @@ class BaseMultimodalProcessor(ABC):
)
# Fallback tokenization if no raw items were processed
if input_ids is None:
input_ids = self._processor.tokenizer(
input_ids = self._tokenizer(
base_output.input_text,
return_tensors="pt",
add_special_tokens=True,