[Performance] Replace preprocess_video logic from GLM multimodal processor with transformer impl for speed up (up to 27% faster) and addressing OOM (up to 50x improvements) (#13487)

python/sglang/srt/multimodal/processors/glm4v.py

@@ -1,7 +1,5 @@
 from typing import List, Union
 
-from decord import VideoReader
-
 from sglang.srt.layers.rotary_embedding import MRotaryEmbedding
 from sglang.srt.models.glm4v import Glm4vForConditionalGeneration
 from sglang.srt.models.glm4v_moe import Glm4vMoeForConditionalGeneration
@@ -46,36 +44,6 @@ class Glm4vImageProcessor(SGLangBaseProcessor):
             video_token_id=self.IM_TOKEN_ID,
         ).build(_processor)
 
-    # adapted from https://github.com/huggingface/transformers/blob/369c99d0cea403b77bd0aef818527106453fd9fc/src/transformers/video_utils.py#L312
-    async def preprocess_video(self, vr: VideoReader):
-        """
-        Preprocess video using VideoReader from Decord backend.
-
-        Args:
-            vr (VideoReader): VideoReader object from decord
-
-        Returns:
-            tuple: A tuple containing processed frames and metadata
-        """
-        video_fps = vr.get_avg_fps()
-        total_num_frames = len(vr)
-        duration = total_num_frames / video_fps if video_fps else 0
-
-        # Extract all frames
-        indices = list(range(total_num_frames))
-        frames = vr.get_batch(indices).asnumpy()
-
-        # Return metadata as dict so transformers can properly create VideoMetadata objects
-        metadata = {
-            "total_num_frames": int(total_num_frames),
-            "fps": float(video_fps),
-            "duration": float(duration),
-            "video_backend": "decord",
-            "frames_indices": indices,
-        }
-
-        return frames, metadata
-
     async def process_mm_data_async(
         self,
         image_data: List[Union[str, bytes]],
@@ -91,19 +59,10 @@ class Glm4vImageProcessor(SGLangBaseProcessor):
             multimodal_tokens=self.mm_tokens,
         )
 
-        video_metadata = None
-
         if base_output.videos:
-            videos_processed = [
-                await self.preprocess_video(video) for video in base_output.videos
-            ]
-            base_output.videos, video_metadata = map(list, zip(*videos_processed))
-            # transformer requires the video inputs to be under this format
-            base_output.videos = [base_output.videos]
-            video_metadata = [video_metadata]
-
+            base_output.videos = request_obj.video_data
         mm_items, input_ids, ret = self.process_and_combine_mm_data(
-            base_output, self.mm_tokens, video_metadata=video_metadata
+            base_output, self.mm_tokens
         )
 
         input_ids = input_ids.flatten()