chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 13 Packages Projects Releases Wiki Activity

Fix InternVL and vision attention for non-CUDA backends (e.g. XPU) (#19997)

Browse Source 
Co-authored-by: Yang Wang <mr.yang.wang@outlook.com>
This commit is contained in:
Juan Muneton
2026-03-14 23:24:41 -07:00
committed by GitHub
co-authored by Yang Wang
parent 1ac6a26464
commit 7458407437
6 changed files with 184 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/multimodal/processors/internvl.py
+17 -10
View File
@@ -19,6 +19,7 @@ from sglang.srt.multimodal.processors.base_processor import (
BaseMultiModalProcessorOutput,
MultimodalSpecialTokens,
)
from sglang.srt.utils import get_device
from sglang.srt.utils.video_decoder import VideoDecoderWrapper
logger = logging.getLogger(__name__)
@@ -434,7 +435,7 @@ class InternVLProcessor(BaseMultimodalProcessor):
len(base_output.videos),
)
mean, std = self._get_normalize_tensors(device="cuda")
mean, std = self._get_normalize_tensors(device=get_device())
# ----- Images -> tiles -----
num_patches_list: List[int] = []
@@ -444,10 +445,11 @@ class InternVLProcessor(BaseMultimodalProcessor):
if isinstance(image, Image.Image):
img_np = np.array(image.convert("RGB"))
tensor = (
torch.from_numpy(img_np).permute(2, 0, 1).cuda().float() / 255.0
torch.from_numpy(img_np).permute(2, 0, 1).to(get_device()).float()
/ 255.0
)
else:
tensor = image.cuda()
tensor = image.to(get_device())
tensor = (tensor - mean) / std
tiles = self.dynamic_preprocess(
@@ -496,7 +498,11 @@ class InternVLProcessor(BaseMultimodalProcessor):
for fi in frame_indices:
img_np = vr[int(fi)]
frame_t = (
torch.from_numpy(img_np).permute(2, 0, 1).cuda().float() / 255.0
torch.from_numpy(img_np)
.permute(2, 0, 1)
.to(get_device())
.float()
/ 255.0
)
frame_t = (frame_t - mean) / std
@@ -568,14 +574,14 @@ class InternVLProcessor(BaseMultimodalProcessor):
image_offsets = []
if image_tensor is not None:
image_offsets = self.get_mm_items_offset(
input_ids=input_ids_tensor.to("cuda"),
input_ids=input_ids_tensor.to(get_device()),
mm_token_id=self.img_context_token_id,
)
video_offsets = []
if video_tensor is not None and self.video_token_id is not None:
video_offsets = self.get_mm_items_offset(
input_ids=input_ids_tensor.to("cuda"),
input_ids=input_ids_tensor.to(get_device()),
mm_token_id=self.video_token_id,
)
@@ -633,7 +639,7 @@ class InternVLProcessor(BaseMultimodalProcessor):
discard_alpha_channel=True,
)
mean, std = self._get_normalize_tensors(device="cuda")
mean, std = self._get_normalize_tensors(device=get_device())
num_patches_list: List[int] = []
pixel_values_list: List[torch.Tensor] = []
@@ -642,10 +648,11 @@ class InternVLProcessor(BaseMultimodalProcessor):
if isinstance(image, Image.Image):
img_np = np.array(image.convert("RGB"))
tensor = (
torch.from_numpy(img_np).permute(2, 0, 1).cuda().float() / 255.0
torch.from_numpy(img_np).permute(2, 0, 1).to(get_device()).float()
/ 255.0
)
else:
tensor = image.cuda()
tensor = image.to(get_device())
tensor = (tensor - mean) / std
tiles = self.dynamic_preprocess(
@@ -688,7 +695,7 @@ class InternVLProcessor(BaseMultimodalProcessor):
image_offsets = []
if pixel_values is not None:
image_offsets = self.get_mm_items_offset(
input_ids=input_ids_tensor.to("cuda"),
input_ids=input_ids_tensor.to(get_device()),
mm_token_id=self.img_context_token_id,
)