chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[Performance] Replace preprocess_video logic from GLM multimodal processor with transformer impl for speed up (up to 27% faster) and addressing OOM (up to 50x improvements) (#13487)

Browse Source
This commit is contained in:
Binyao Jiang
2025-11-24 18:17:13 -08:00
committed by GitHub
parent 4b45d556a7
commit de430b6745
3 changed files with 6 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
python/pyproject.toml
View File

@@ -63,6 +63,8 @@ dependencies = [
"timm==1.0.16",
"torch_memory_saver==0.0.9",
"torch==2.8.0",
"torchcodec==0.7.0 ; sys_platform != 'linux' or (sys_platform == 'linux' and platform_machine != 'aarch64' and platform_machine != 'arm64' and platform_machine != 'armv7l')", # torchcodec does not exist in those systems. If not provided, transformer will use torchvision instead by default.
"av ; sys_platform == 'linux' and (platform_machine == 'aarch64' or platform_machine == 'arm64' and platform_machine == 'armv7l')",
"torchaudio==2.8.0",
"torchvision",
"torchao==0.9.0",
@@ -89,7 +91,6 @@ diffusion = [
"moviepy>=2.0.0",
"cloudpickle",
"remote-pdb",
"torchcodec==0.5.0",
"st_attn ==0.0.7",
"vsa==0.0.4",
]

2
python/pyproject_xpu.toml
View File

@@ -16,6 +16,8 @@ classifiers = [
dependencies = [
"torch==2.9.0",
"torchcodec==0.8.0 ; sys_platform != 'linux' or (sys_platform == 'linux' and platform_machine != 'aarch64' and platform_machine != 'arm64' and platform_machine != 'armv7l')", # torchcodec does not exist in those systems. If not provided, transformer will use torchvision instead by default.
"av ; sys_platform == 'linux' and (platform_machine == 'aarch64' or platform_machine == 'arm64' and platform_machine == 'armv7l')",
"torchaudio==2.9.0",
"torchvision",
"sgl-kernel @ git+https://github.com/sgl-project/sgl-kernel-xpu.git",

45
python/sglang/srt/multimodal/processors/glm4v.py
View File

@@ -1,7 +1,5 @@
from typing import List, Union
from decord import VideoReader
from sglang.srt.layers.rotary_embedding import MRotaryEmbedding
from sglang.srt.models.glm4v import Glm4vForConditionalGeneration
from sglang.srt.models.glm4v_moe import Glm4vMoeForConditionalGeneration
@@ -46,36 +44,6 @@ class Glm4vImageProcessor(SGLangBaseProcessor):
video_token_id=self.IM_TOKEN_ID,
).build(_processor)
# adapted from https://github.com/huggingface/transformers/blob/369c99d0cea403b77bd0aef818527106453fd9fc/src/transformers/video_utils.py#L312
async def preprocess_video(self, vr: VideoReader):
"""
Preprocess video using VideoReader from Decord backend.
Args:
vr (VideoReader): VideoReader object from decord
Returns:
tuple: A tuple containing processed frames and metadata
"""
video_fps = vr.get_avg_fps()
total_num_frames = len(vr)
duration = total_num_frames / video_fps if video_fps else 0
# Extract all frames
indices = list(range(total_num_frames))
frames = vr.get_batch(indices).asnumpy()
# Return metadata as dict so transformers can properly create VideoMetadata objects
metadata = {
"total_num_frames": int(total_num_frames),
"fps": float(video_fps),
"duration": float(duration),
"video_backend": "decord",
"frames_indices": indices,
}
return frames, metadata
async def process_mm_data_async(
self,
image_data: List[Union[str, bytes]],
@@ -91,19 +59,10 @@ class Glm4vImageProcessor(SGLangBaseProcessor):
multimodal_tokens=self.mm_tokens,
)
video_metadata = None
if base_output.videos:
videos_processed = [
await self.preprocess_video(video) for video in base_output.videos
]
base_output.videos, video_metadata = map(list, zip(*videos_processed))
# transformer requires the video inputs to be under this format
base_output.videos = [base_output.videos]
video_metadata = [video_metadata]
base_output.videos = request_obj.video_data
mm_items, input_ids, ret = self.process_and_combine_mm_data(
base_output, self.mm_tokens, video_metadata=video_metadata
base_output, self.mm_tokens
)
input_ids = input_ids.flatten()