[VLM] Support ViT Piecewise CUDA Graph for Qwen3-VL (#15320)

Co-authored-by: luoyuan.luo <luoyuan.luo@antgroup.com>
This commit is contained in:
Yuan Luo
2025-12-20 21:00:07 +08:00
committed by GitHub
co-authored by luoyuan.luo
parent 1f1f05a85e
commit 019517a356
6 changed files with 235 additions and 66 deletions
+2 -2
View File
@@ -294,7 +294,7 @@ class VisionTritonAttention(nn.Module):
Returns:
[b * s, h, head_size]
"""
if get_bool_env_var("SGLANG_VIT_ENABLE_CUDA_GRAPH") and self.tp_size == 1:
if get_bool_env_var("SGLANG_VIT_ENABLE_CUDA_GRAPH"):
if "output_ws" not in kwargs:
raise RuntimeError("output_ws should be prepared for cuda-graph mode")
@@ -363,7 +363,7 @@ class VisionFlash3Attention(nn.Module):
Returns:
[b * s, h, head_size]
"""
if get_bool_env_var("SGLANG_VIT_ENABLE_CUDA_GRAPH") and self.tp_size == 1:
if get_bool_env_var("SGLANG_VIT_ENABLE_CUDA_GRAPH"):
max_seqlen = cu_seqlens[1]
output = flash_attn_varlen_func(
q,