[VLM] Tiny: Unify VLM environment variables (#15572)
Co-authored-by: luoyuan.luo <luoyuan.luo@antgroup.com>
This commit is contained in:
@@ -11,6 +11,7 @@ import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from einops import rearrange
|
||||
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.dp_attention import get_attention_tp_rank, get_attention_tp_size
|
||||
from sglang.srt.utils import (
|
||||
get_bool_env_var,
|
||||
@@ -294,7 +295,7 @@ class VisionTritonAttention(nn.Module):
|
||||
Returns:
|
||||
[b * s, h, head_size]
|
||||
"""
|
||||
if get_bool_env_var("SGLANG_VIT_ENABLE_CUDA_GRAPH"):
|
||||
if envs.SGLANG_VIT_ENABLE_CUDA_GRAPH.get():
|
||||
if "output_ws" not in kwargs:
|
||||
raise RuntimeError("output_ws should be prepared for cuda-graph mode")
|
||||
|
||||
@@ -363,7 +364,7 @@ class VisionFlash3Attention(nn.Module):
|
||||
Returns:
|
||||
[b * s, h, head_size]
|
||||
"""
|
||||
if get_bool_env_var("SGLANG_VIT_ENABLE_CUDA_GRAPH"):
|
||||
if envs.SGLANG_VIT_ENABLE_CUDA_GRAPH.get():
|
||||
max_seqlen = cu_seqlens[1]
|
||||
output = flash_attn_varlen_func(
|
||||
q,
|
||||
|
||||
Reference in New Issue
Block a user