chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 11 Packages Projects Releases Wiki Activity

[Performance] Optimze the performance of Qwen25VL (#15640)

Browse Source 
Co-authored-by: Xinyuan Tong <115166877+JustinTong0323@users.noreply.github.com>
This commit is contained in:
Siyuan Chen
2026-01-03 15:15:36 +08:00
committed by GitHub
parent 5b4f790200
commit 9a414b164c
3 changed files with 20 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
python/sglang/srt/layers/rotary_embedding.py
View File

@@ -16,6 +16,7 @@ from sglang.srt.server_args import get_global_server_args
from sglang.srt.utils import (
cpu_has_amx_support,
get_bool_env_var,
get_compiler_backend,
is_cpu,
is_cuda,
is_hip,
@@ -2748,6 +2749,7 @@ def rotate_half(x):
return torch.cat((-x2, x1), dim=-1)
@torch.compile(dynamic=True, backend=get_compiler_backend())
def apply_rotary_pos_emb_native(
q: torch.Tensor,
k: torch.Tensor,

18
python/sglang/srt/model_executor/forward_batch_info.py
View File

@@ -668,15 +668,8 @@ class ForwardBatch:
self,
mm_input: MultimodalInputs,
seq_len: int,
device: torch.device,
) -> torch.Tensor:
if mm_input.mrope_position_delta.device.type != device:
# transfer mrope_position_delta to device when the first running,
# avoiding successvie host-to-device data transfer
mm_input.mrope_position_delta = mm_input.mrope_position_delta.to(
device, non_blocking=True
)
# doing below compute on cpu to avoid frequent small kernels
mrope_position_deltas = mm_input.mrope_position_delta.flatten()
mrope_positions = (
(mrope_position_deltas + seq_len - 1).unsqueeze(0).repeat(3, 1)
@@ -701,11 +694,10 @@ class ForwardBatch:
(3, 1),
self.seq_lens_cpu[batch_idx] - 1,
dtype=torch.int64,
device=model_runner.device,
)
else:
mrope_positions = self._expand_mrope_from_input(
mm_input, self.seq_lens_cpu[batch_idx], model_runner.device
mm_input, self.seq_lens_cpu[batch_idx]
)
mrope_positions_list[batch_idx] = mrope_positions
elif self.forward_mode.is_extend():
@@ -737,14 +729,14 @@ class ForwardBatch:
]
if mrope_positions.numel() == 0:
mrope_positions = self._expand_mrope_from_input(
mm_input, self.seq_lens[batch_idx], model_runner.device
mm_input, self.seq_lens_cpu[batch_idx]
)
mrope_positions_list[batch_idx] = mrope_positions
self.mrope_positions = torch.cat(
[pos.to(device=model_runner.device) for pos in mrope_positions_list],
[pos for pos in mrope_positions_list],
dim=1,
).to(dtype=torch.int64, device=model_runner.device)
).to(dtype=torch.int64, device=model_runner.device, non_blocking=True)
def get_max_chunk_capacity(self):
# Maximum number of tokens in each chunk

16
python/sglang/srt/models/qwen2_5_vl.py
View File

@@ -47,6 +47,7 @@ from sglang.srt.distributed import (
)
from sglang.srt.distributed.parallel_state import get_pp_group
from sglang.srt.environ import envs
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.attention.vision import VisionAttention
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import (
@@ -114,12 +115,21 @@ class Qwen2_5_VLMLP(nn.Module):
tp_size=self.tp_size,
tp_rank=self.tp_rank,
)
self.act = ACT2FN[hidden_act]
self.hidden_act = hidden_act
if self.hidden_act == "silu":
self.act = SiluAndMul()
else:
base_act = ACT2FN[self.hidden_act]
def _act_fn(x: torch.Tensor) -> torch.Tensor:
gate, up = x.chunk(2, dim=-1)
return base_act(gate) * up
self.act = _act_fn
def forward(self, x: torch.Tensor) -> torch.Tensor:
gate_up, _ = self.gate_up_proj(x)
gate, up = gate_up.chunk(2, dim=-1)
x = self.act(gate) * up
x = self.act(gate_up)
x_down, _ = self.down_proj(x)
return x_down