diff --git a/python/sglang/srt/layers/attention/vision.py b/python/sglang/srt/layers/attention/vision.py index 77a8cde46..087e76baf 100644 --- a/python/sglang/srt/layers/attention/vision.py +++ b/python/sglang/srt/layers/attention/vision.py @@ -22,6 +22,7 @@ from sglang.srt.utils import ( is_cuda, is_hip, is_npu, + is_xpu, print_info_once, ) from sglang.srt.utils.multi_stream_utils import ( @@ -32,6 +33,7 @@ from sglang.srt.utils.multi_stream_utils import ( _is_cuda = is_cuda() _is_npu = is_npu() _is_hip = is_hip() +_is_xpu = is_xpu() if _is_cuda: from flashinfer.prefill import cudnn_batch_prefill_with_kv_cache @@ -366,8 +368,8 @@ class VisionTritonAttention(nn.Module): k, v, output, - cu_seqlens.cuda(), - seq_lens.cuda(), + cu_seqlens.to(q.device), + seq_lens.to(q.device), max_seqlen, is_causal=False, ) @@ -905,6 +907,8 @@ class VisionAttention(nn.Module): backend = "aiter_attn" else: backend = "triton_attn" + elif _is_xpu: + backend = "triton_attn" else: backend = "sdpa" if backend == "fa3" and is_blackwell_supported(): diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index acd507277..5afc964a1 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -1237,7 +1237,7 @@ class Scheduler( self.schedule_stream = self.device_module.Stream(priority=0) if self.device == "cpu": self.schedule_stream.synchronize = lambda: None # No-op for CPU - with CudaStreamContext(self.schedule_stream): + with self.device_module.StreamContext(self.schedule_stream): dispatch_event_loop(self) @DynamicGradMode() diff --git a/python/sglang/srt/multimodal/processors/internvl.py b/python/sglang/srt/multimodal/processors/internvl.py index e9a0753e4..955198730 100644 --- a/python/sglang/srt/multimodal/processors/internvl.py +++ b/python/sglang/srt/multimodal/processors/internvl.py @@ -19,6 +19,7 @@ from sglang.srt.multimodal.processors.base_processor import ( BaseMultiModalProcessorOutput, MultimodalSpecialTokens, ) +from sglang.srt.utils import get_device from sglang.srt.utils.video_decoder import VideoDecoderWrapper logger = logging.getLogger(__name__) @@ -434,7 +435,7 @@ class InternVLProcessor(BaseMultimodalProcessor): len(base_output.videos), ) - mean, std = self._get_normalize_tensors(device="cuda") + mean, std = self._get_normalize_tensors(device=get_device()) # ----- Images -> tiles ----- num_patches_list: List[int] = [] @@ -444,10 +445,11 @@ class InternVLProcessor(BaseMultimodalProcessor): if isinstance(image, Image.Image): img_np = np.array(image.convert("RGB")) tensor = ( - torch.from_numpy(img_np).permute(2, 0, 1).cuda().float() / 255.0 + torch.from_numpy(img_np).permute(2, 0, 1).to(get_device()).float() + / 255.0 ) else: - tensor = image.cuda() + tensor = image.to(get_device()) tensor = (tensor - mean) / std tiles = self.dynamic_preprocess( @@ -496,7 +498,11 @@ class InternVLProcessor(BaseMultimodalProcessor): for fi in frame_indices: img_np = vr[int(fi)] frame_t = ( - torch.from_numpy(img_np).permute(2, 0, 1).cuda().float() / 255.0 + torch.from_numpy(img_np) + .permute(2, 0, 1) + .to(get_device()) + .float() + / 255.0 ) frame_t = (frame_t - mean) / std @@ -568,14 +574,14 @@ class InternVLProcessor(BaseMultimodalProcessor): image_offsets = [] if image_tensor is not None: image_offsets = self.get_mm_items_offset( - input_ids=input_ids_tensor.to("cuda"), + input_ids=input_ids_tensor.to(get_device()), mm_token_id=self.img_context_token_id, ) video_offsets = [] if video_tensor is not None and self.video_token_id is not None: video_offsets = self.get_mm_items_offset( - input_ids=input_ids_tensor.to("cuda"), + input_ids=input_ids_tensor.to(get_device()), mm_token_id=self.video_token_id, ) @@ -633,7 +639,7 @@ class InternVLProcessor(BaseMultimodalProcessor): discard_alpha_channel=True, ) - mean, std = self._get_normalize_tensors(device="cuda") + mean, std = self._get_normalize_tensors(device=get_device()) num_patches_list: List[int] = [] pixel_values_list: List[torch.Tensor] = [] @@ -642,10 +648,11 @@ class InternVLProcessor(BaseMultimodalProcessor): if isinstance(image, Image.Image): img_np = np.array(image.convert("RGB")) tensor = ( - torch.from_numpy(img_np).permute(2, 0, 1).cuda().float() / 255.0 + torch.from_numpy(img_np).permute(2, 0, 1).to(get_device()).float() + / 255.0 ) else: - tensor = image.cuda() + tensor = image.to(get_device()) tensor = (tensor - mean) / std tiles = self.dynamic_preprocess( @@ -688,7 +695,7 @@ class InternVLProcessor(BaseMultimodalProcessor): image_offsets = [] if pixel_values is not None: image_offsets = self.get_mm_items_offset( - input_ids=input_ids_tensor.to("cuda"), + input_ids=input_ids_tensor.to(get_device()), mm_token_id=self.img_context_token_id, ) diff --git a/python/sglang/srt/server_args.py b/python/sglang/srt/server_args.py index ba8d1d61b..e1177cfcd 100644 --- a/python/sglang/srt/server_args.py +++ b/python/sglang/srt/server_args.py @@ -751,6 +751,7 @@ class ServerArgs: self._handle_hpu_backends() self._handle_cpu_backends() self._handle_npu_backends() + self._handle_xpu_backends() # Handle piecewise CUDA graph. self._handle_piecewise_cuda_graph() @@ -1027,6 +1028,15 @@ class ServerArgs: ) self.piecewise_cuda_graph_compiler = "eager" + def _handle_xpu_backends(self): + if self.device == "xpu": + if not self.disable_piecewise_cuda_graph: + logger.warning( + "XPU platform does not support piecewise CUDA graph, ignoring --disable-piecewise-cuda-graph" + " flag and disabling piecewise CUDA graph." + ) + self.disable_piecewise_cuda_graph = True + def _handle_piecewise_cuda_graph(self): # Skip auto-disable when enforce flag is set (for testing) if self.enforce_piecewise_cuda_graph: diff --git a/test/srt/run_suite.py b/test/srt/run_suite.py index dc1267195..3a505b670 100644 --- a/test/srt/run_suite.py +++ b/test/srt/run_suite.py @@ -71,10 +71,12 @@ suite_xeon = { } # Add Intel XPU tests +# NOTE: please sort the test cases alphabetically by the test file name suite_xpu = { "per-commit-xpu": [ - TestFile("xpu/test_intel_xpu_backend.py"), TestFile("xpu/test_deepseek_ocr.py"), + TestFile("xpu/test_internvl.py"), + TestFile("xpu/test_intel_xpu_backend.py"), ], } diff --git a/test/srt/xpu/test_internvl.py b/test/srt/xpu/test_internvl.py new file mode 100644 index 000000000..d2e2c3435 --- /dev/null +++ b/test/srt/xpu/test_internvl.py @@ -0,0 +1,147 @@ +""" +XPU tests for InternVL models (InternVL2.5-2B, InternVL3.5-2B). + +Uses the same structure as test_vision_openai_server_a.py: OpenAI /v1 chat API +and ImageOpenAITestMixin. An XPU-specific base injects --device xpu and +--attention-backend intel_xpu. + +Usage (pick module path to match your cwd): + + From test/srt/xpu: + python3 -m unittest test_internvl.TestInternVL25Server.test_single_image_chat_completion + python3 -m unittest test_internvl + + From test/srt: + python3 -m unittest xpu.test_internvl.TestInternVL25Server.test_single_image_chat_completion + python3 -m unittest xpu.test_internvl + + From repo root: + python3 -m unittest test.srt.xpu.test_internvl.TestInternVL25Server.test_single_image_chat_completion + python3 -m unittest test.srt.xpu.test_internvl +""" + +import os +import unittest + +from sglang.test.vlm_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + ImageOpenAITestMixin, + TestOpenAIMLLMServerBase, + kill_process_tree, + popen_launch_server, +) + +# XPU args injected into server launch for all InternVL XPU tests +XPU_ARGS = [ + "--device", + "xpu", + "--attention-backend", + "intel_xpu", +] + + +# Longer launch timeout for InternVL3.5 (can be slow to start on XPU) +INTERNVL35_LAUNCH_TIMEOUT = 900 + + +class InternVLXPUServerBase(TestOpenAIMLLMServerBase): + """Base for InternVL tests on XPU. Injects XPU args and sets SGLANG_USE_SGL_XPU.""" + + use_sgl_xpu = True # subclasses override for Triton backend + launch_timeout = None # subclasses can set to override (seconds) + + @classmethod + def setUpClass(cls): + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + + os.environ["SGLANG_USE_SGL_XPU"] = "1" if cls.use_sgl_xpu else "0" + + other_args = list(XPU_ARGS) + list(cls.extra_args) + if cls.trust_remote_code: + other_args.extend(cls.fixed_args) + else: + other_args.extend( + arg for arg in cls.fixed_args if arg != "--trust-remote-code" + ) + + timeout = ( + cls.launch_timeout + if cls.launch_timeout is not None + else DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH + ) + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=timeout, + api_key=cls.api_key, + other_args=other_args, + ) + cls.base_url += "/v1" + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + +class TestInternVL25Server(ImageOpenAITestMixin, InternVLXPUServerBase): + """InternVL2.5-2B on XPU with SGL XPU backend.""" + + model = "OpenGVLab/InternVL2_5-2B" + use_sgl_xpu = True + extra_args = [ + "--cuda-graph-max-bs=4", + ] + + def test_video_images_chat_completion(self): + # Video test uses 10 frames and exceeds max_prefill_tokens (23124 > 16384). + pass + + +class TestInternVL25TritonServer(ImageOpenAITestMixin, InternVLXPUServerBase): + """InternVL2.5-2B on XPU with Triton (non-SGL) backend.""" + + model = "OpenGVLab/InternVL2_5-2B" + use_sgl_xpu = False + extra_args = [ + "--cuda-graph-max-bs=4", + ] + + def test_video_images_chat_completion(self): + # Video test exceeds max_prefill_tokens on XPU with default limits. + pass + + +class TestInternVL35_2BServer(ImageOpenAITestMixin, InternVLXPUServerBase): + """InternVL3.5-2B on XPU with SGL XPU backend.""" + + model = "OpenGVLab/InternVL3_5-2B" + use_sgl_xpu = True + launch_timeout = INTERNVL35_LAUNCH_TIMEOUT + extra_args = [ + "--cuda-graph-max-bs=4", + ] + + def test_video_images_chat_completion(self): + # Video test exceeds max_prefill_tokens (23202 > 14588) on InternVL3.5. + pass + + +class TestInternVL35_2BTritonServer(ImageOpenAITestMixin, InternVLXPUServerBase): + """InternVL3.5-2B on XPU with Triton (non-SGL) backend.""" + + model = "OpenGVLab/InternVL3_5-2B" + use_sgl_xpu = False + launch_timeout = INTERNVL35_LAUNCH_TIMEOUT + extra_args = [ + "--cuda-graph-max-bs=4", + ] + + def test_video_images_chat_completion(self): + # Video test exceeds max_prefill_tokens on InternVL3.5. + pass + + +if __name__ == "__main__": + unittest.main()