chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 11 Packages Projects Releases Wiki Activity

Fix InternVL and vision attention for non-CUDA backends (e.g. XPU) (#19997)

Browse Source 
Co-authored-by: Yang Wang <mr.yang.wang@outlook.com>
This commit is contained in:
Juan Muneton
2026-03-14 23:24:41 -07:00
committed by GitHub
parent 1ac6a26464
commit 7458407437
6 changed files with 184 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
python/sglang/srt/layers/attention/vision.py
View File

@@ -22,6 +22,7 @@ from sglang.srt.utils import (
is_cuda,
is_hip,
is_npu,
is_xpu,
print_info_once,
)
from sglang.srt.utils.multi_stream_utils import (
@@ -32,6 +33,7 @@ from sglang.srt.utils.multi_stream_utils import (
_is_cuda = is_cuda()
_is_npu = is_npu()
_is_hip = is_hip()
_is_xpu = is_xpu()
if _is_cuda:
from flashinfer.prefill import cudnn_batch_prefill_with_kv_cache
@@ -366,8 +368,8 @@ class VisionTritonAttention(nn.Module):
k,
v,
output,
cu_seqlens.cuda(),
seq_lens.cuda(),
cu_seqlens.to(q.device),
seq_lens.to(q.device),
max_seqlen,
is_causal=False,
)
@@ -905,6 +907,8 @@ class VisionAttention(nn.Module):
backend = "aiter_attn"
else:
backend = "triton_attn"
elif _is_xpu:
backend = "triton_attn"
else:
backend = "sdpa"
if backend == "fa3" and is_blackwell_supported():

2
python/sglang/srt/managers/scheduler.py
View File

@@ -1237,7 +1237,7 @@ class Scheduler(
self.schedule_stream = self.device_module.Stream(priority=0)
if self.device == "cpu":
self.schedule_stream.synchronize = lambda: None # No-op for CPU
with CudaStreamContext(self.schedule_stream):
with self.device_module.StreamContext(self.schedule_stream):
dispatch_event_loop(self)
@DynamicGradMode()

27
python/sglang/srt/multimodal/processors/internvl.py
View File

@@ -19,6 +19,7 @@ from sglang.srt.multimodal.processors.base_processor import (
BaseMultiModalProcessorOutput,
MultimodalSpecialTokens,
)
from sglang.srt.utils import get_device
from sglang.srt.utils.video_decoder import VideoDecoderWrapper
logger = logging.getLogger(__name__)
@@ -434,7 +435,7 @@ class InternVLProcessor(BaseMultimodalProcessor):
len(base_output.videos),
)
mean, std = self._get_normalize_tensors(device="cuda")
mean, std = self._get_normalize_tensors(device=get_device())
# ----- Images -> tiles -----
num_patches_list: List[int] = []
@@ -444,10 +445,11 @@ class InternVLProcessor(BaseMultimodalProcessor):
if isinstance(image, Image.Image):
img_np = np.array(image.convert("RGB"))
tensor = (
torch.from_numpy(img_np).permute(2, 0, 1).cuda().float() / 255.0
torch.from_numpy(img_np).permute(2, 0, 1).to(get_device()).float()
/ 255.0
)
else:
tensor = image.cuda()
tensor = image.to(get_device())
tensor = (tensor - mean) / std
tiles = self.dynamic_preprocess(
@@ -496,7 +498,11 @@ class InternVLProcessor(BaseMultimodalProcessor):
for fi in frame_indices:
img_np = vr[int(fi)]
frame_t = (
torch.from_numpy(img_np).permute(2, 0, 1).cuda().float() / 255.0
torch.from_numpy(img_np)
.permute(2, 0, 1)
.to(get_device())
.float()
/ 255.0
)
frame_t = (frame_t - mean) / std
@@ -568,14 +574,14 @@ class InternVLProcessor(BaseMultimodalProcessor):
image_offsets = []
if image_tensor is not None:
image_offsets = self.get_mm_items_offset(
input_ids=input_ids_tensor.to("cuda"),
input_ids=input_ids_tensor.to(get_device()),
mm_token_id=self.img_context_token_id,
)
video_offsets = []
if video_tensor is not None and self.video_token_id is not None:
video_offsets = self.get_mm_items_offset(
input_ids=input_ids_tensor.to("cuda"),
input_ids=input_ids_tensor.to(get_device()),
mm_token_id=self.video_token_id,
)
@@ -633,7 +639,7 @@ class InternVLProcessor(BaseMultimodalProcessor):
discard_alpha_channel=True,
)
mean, std = self._get_normalize_tensors(device="cuda")
mean, std = self._get_normalize_tensors(device=get_device())
num_patches_list: List[int] = []
pixel_values_list: List[torch.Tensor] = []
@@ -642,10 +648,11 @@ class InternVLProcessor(BaseMultimodalProcessor):
if isinstance(image, Image.Image):
img_np = np.array(image.convert("RGB"))
tensor = (
torch.from_numpy(img_np).permute(2, 0, 1).cuda().float() / 255.0
torch.from_numpy(img_np).permute(2, 0, 1).to(get_device()).float()
/ 255.0
)
else:
tensor = image.cuda()
tensor = image.to(get_device())
tensor = (tensor - mean) / std
tiles = self.dynamic_preprocess(
@@ -688,7 +695,7 @@ class InternVLProcessor(BaseMultimodalProcessor):
image_offsets = []
if pixel_values is not None:
image_offsets = self.get_mm_items_offset(
input_ids=input_ids_tensor.to("cuda"),
input_ids=input_ids_tensor.to(get_device()),
mm_token_id=self.img_context_token_id,
)

10
python/sglang/srt/server_args.py
View File

@@ -751,6 +751,7 @@ class ServerArgs:
self._handle_hpu_backends()
self._handle_cpu_backends()
self._handle_npu_backends()
self._handle_xpu_backends()
# Handle piecewise CUDA graph.
self._handle_piecewise_cuda_graph()
@@ -1027,6 +1028,15 @@ class ServerArgs:
)
self.piecewise_cuda_graph_compiler = "eager"
def _handle_xpu_backends(self):
if self.device == "xpu":
if not self.disable_piecewise_cuda_graph:
logger.warning(
"XPU platform does not support piecewise CUDA graph, ignoring --disable-piecewise-cuda-graph"
" flag and disabling piecewise CUDA graph."
)
self.disable_piecewise_cuda_graph = True
def _handle_piecewise_cuda_graph(self):
# Skip auto-disable when enforce flag is set (for testing)
if self.enforce_piecewise_cuda_graph:

4
test/srt/run_suite.py
View File

@@ -71,10 +71,12 @@ suite_xeon = {
}
# Add Intel XPU tests
# NOTE: please sort the test cases alphabetically by the test file name
suite_xpu = {
"per-commit-xpu": [
TestFile("xpu/test_intel_xpu_backend.py"),
TestFile("xpu/test_deepseek_ocr.py"),
TestFile("xpu/test_internvl.py"),
TestFile("xpu/test_intel_xpu_backend.py"),
],
}

147
test/srt/xpu/test_internvl.py Normal file
View File

@@ -0,0 +1,147 @@
"""
XPU tests for InternVL models (InternVL2.5-2B, InternVL3.5-2B).
Uses the same structure as test_vision_openai_server_a.py: OpenAI /v1 chat API
and ImageOpenAITestMixin. An XPU-specific base injects --device xpu and
--attention-backend intel_xpu.
Usage (pick module path to match your cwd):
From test/srt/xpu:
python3 -m unittest test_internvl.TestInternVL25Server.test_single_image_chat_completion
python3 -m unittest test_internvl
From test/srt:
python3 -m unittest xpu.test_internvl.TestInternVL25Server.test_single_image_chat_completion
python3 -m unittest xpu.test_internvl
From repo root:
python3 -m unittest test.srt.xpu.test_internvl.TestInternVL25Server.test_single_image_chat_completion
python3 -m unittest test.srt.xpu.test_internvl
"""
import os
import unittest
from sglang.test.vlm_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
ImageOpenAITestMixin,
TestOpenAIMLLMServerBase,
kill_process_tree,
popen_launch_server,
)
# XPU args injected into server launch for all InternVL XPU tests
XPU_ARGS = [
"--device",
"xpu",
"--attention-backend",
"intel_xpu",
]
# Longer launch timeout for InternVL3.5 (can be slow to start on XPU)
INTERNVL35_LAUNCH_TIMEOUT = 900
class InternVLXPUServerBase(TestOpenAIMLLMServerBase):
"""Base for InternVL tests on XPU. Injects XPU args and sets SGLANG_USE_SGL_XPU."""
use_sgl_xpu = True # subclasses override for Triton backend
launch_timeout = None # subclasses can set to override (seconds)
@classmethod
def setUpClass(cls):
cls.base_url = DEFAULT_URL_FOR_TEST
cls.api_key = "sk-123456"
os.environ["SGLANG_USE_SGL_XPU"] = "1" if cls.use_sgl_xpu else "0"
other_args = list(XPU_ARGS) + list(cls.extra_args)
if cls.trust_remote_code:
other_args.extend(cls.fixed_args)
else:
other_args.extend(
arg for arg in cls.fixed_args if arg != "--trust-remote-code"
)
timeout = (
cls.launch_timeout
if cls.launch_timeout is not None
else DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH
)
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=timeout,
api_key=cls.api_key,
other_args=other_args,
)
cls.base_url += "/v1"
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
class TestInternVL25Server(ImageOpenAITestMixin, InternVLXPUServerBase):
"""InternVL2.5-2B on XPU with SGL XPU backend."""
model = "OpenGVLab/InternVL2_5-2B"
use_sgl_xpu = True
extra_args = [
"--cuda-graph-max-bs=4",
]
def test_video_images_chat_completion(self):
# Video test uses 10 frames and exceeds max_prefill_tokens (23124 > 16384).
pass
class TestInternVL25TritonServer(ImageOpenAITestMixin, InternVLXPUServerBase):
"""InternVL2.5-2B on XPU with Triton (non-SGL) backend."""
model = "OpenGVLab/InternVL2_5-2B"
use_sgl_xpu = False
extra_args = [
"--cuda-graph-max-bs=4",
]
def test_video_images_chat_completion(self):
# Video test exceeds max_prefill_tokens on XPU with default limits.
pass
class TestInternVL35_2BServer(ImageOpenAITestMixin, InternVLXPUServerBase):
"""InternVL3.5-2B on XPU with SGL XPU backend."""
model = "OpenGVLab/InternVL3_5-2B"
use_sgl_xpu = True
launch_timeout = INTERNVL35_LAUNCH_TIMEOUT
extra_args = [
"--cuda-graph-max-bs=4",
]
def test_video_images_chat_completion(self):
# Video test exceeds max_prefill_tokens (23202 > 14588) on InternVL3.5.
pass
class TestInternVL35_2BTritonServer(ImageOpenAITestMixin, InternVLXPUServerBase):
"""InternVL3.5-2B on XPU with Triton (non-SGL) backend."""
model = "OpenGVLab/InternVL3_5-2B"
use_sgl_xpu = False
launch_timeout = INTERNVL35_LAUNCH_TIMEOUT
extra_args = [
"--cuda-graph-max-bs=4",
]
def test_video_images_chat_completion(self):
# Video test exceeds max_prefill_tokens on InternVL3.5.
pass
if __name__ == "__main__":
unittest.main()