chore: add an unified server arg for multimodal inputs preprocess config(#12149)

Co-authored-by: bianfeng <bianfeng@pinduoduo.com>
Co-authored-by: Xinyuan Tong <115166877+JustinTong0323@users.noreply.github.com>

docs/advanced_features/server_arguments.md

@@ -156,6 +156,7 @@ Please consult the documentation below and [server_args.py](https://github.com/s
 | `--base-gpu-id` | The base GPU ID to start allocating GPUs from. Useful when running multiple instances on the same machine. | `0` | Type: int |
 | `--gpu-id-step` | The delta between consecutive GPU IDs that are used. For example, setting it to 2 will use GPU 0,2,4,... | `1` | Type: int |
 | `--sleep-on-idle` | Reduce CPU usage when sglang is idle. | `False` | bool flag (set to enable) |
+| `--mm-process-config` | A JSON string for multimodal preprocessing configuration. It can contain keys: `image`, `video`, `audio`. | `{}` |
 
 ## Logging
 | Argument | Description | Defaults | Options |

docs/supported_models/multimodal_language_models.md

@@ -101,3 +101,9 @@ For multimodal models, you can use the `--keep-mm-feature-on-device` flag to opt
 - **With `--keep-mm-feature-on-device`**: Feature tensors remain on GPU, reducing device-to-host copy overhead and improving latency, but consuming more GPU memory
 
 Use this flag when you have sufficient GPU memory and want to minimize latency for multimodal inference.
+
+### Multimodal Inputs Limitation
+
+- **Use `--mm-process-config '{"image":{"max_pixels":1048576},"video":{"fps":3,"max_pixels":602112,"max_frames":60}}'`**: To set `image`, `video`, and `audio` input limits.
+
+This can reduce GPU memory usage, improve inference speed, and help to avoid OOM, but may impact model performance, thus set a proper value based on your specific use case. Currently, only `qwen_vl` supports this config. Please refer to [qwen_vl processor](https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/multimodal/processors/qwen_vl.py) for understanding the meaning of each parameter.

python/sglang/srt/multimodal/processors/qwen_vl.py

@@ -145,37 +145,44 @@ def smart_nframes(
 async def preprocess_video(
     vr,
     image_factor: int = IMAGE_FACTOR,
-    # vr: VideoReader, image_factor: int = IMAGE_FACTOR
+    video_config: dict = {},
 ) -> torch.Tensor:
     entry_time = time.perf_counter()
-    ele = {}
+
     total_frames, video_fps = len(vr), vr.get_avg_fps()
-    nframes = smart_nframes({}, total_frames=total_frames, video_fps=video_fps)
+    nframes = smart_nframes(
+        video_config, total_frames=total_frames, video_fps=video_fps
+    )
     idx = np.linspace(0, total_frames - 1, num=nframes, dtype=np.int64)
     idx = np.unique(idx)
     video_np = vr.get_batch(idx).asnumpy()
     video = torch.from_numpy(video_np).pin_memory()
     video = video.permute(0, 3, 1, 2)  # Convert to TCHW format
+
     nframes, _, height, width = video.shape
-    min_pixels = ele.get("min_pixels", VIDEO_MIN_PIXELS)
-    total_pixels = ele.get("total_pixels", VIDEO_TOTAL_PIXELS)
+    min_pixels = video_config.get("min_pixels", VIDEO_MIN_PIXELS)
+    total_pixels = video_config.get("total_pixels", VIDEO_TOTAL_PIXELS)
     max_pixels = max(
-        min(VIDEO_MAX_PIXELS, total_pixels / nframes * FRAME_FACTOR),
+        min(
+            video_config.get("max_pixels", VIDEO_MAX_PIXELS),
+            total_pixels / nframes * FRAME_FACTOR,
+        ),
         int(min_pixels * 1.05),
     )
 
     get_batch_time = time.perf_counter()
 
-    max_pixels_supposed = ele.get("max_pixels", max_pixels)
+    max_pixels_supposed = video_config.get("max_pixels", max_pixels)
+
     if max_pixels_supposed > max_pixels:
         logger.warning(
             f"The given max_pixels[{max_pixels_supposed}] exceeds limit[{max_pixels}]."
         )
     max_pixels = min(max_pixels_supposed, max_pixels)
-    if "resized_height" in ele and "resized_width" in ele:
+    if "resized_height" in video_config and "resized_width" in video_config:
         resized_height, resized_width = smart_resize(
-            ele["resized_height"],
-            ele["resized_width"],
+            video_config["resized_height"],
+            video_config["resized_width"],
             factor=image_factor,
         )
     else:
@@ -236,6 +243,9 @@ class QwenVLImageProcessor(SGLangBaseProcessor):
         self.audio_start_token_id = getattr(hf_config, "audio_start_token_id", None)
         self.audio_token_id = getattr(hf_config, "audio_token_id", None)
 
+        self.image_config = server_args.mm_process_config.get("image", {})
+        self.video_config = server_args.mm_process_config.get("video", {})
+
         self.mm_tokens = MultimodalSpecialTokens(
             image_token="<|vision_start|><|image_pad|><|vision_end|>",
             image_token_id=hf_config.image_token_id,
@@ -269,7 +279,8 @@ class QwenVLImageProcessor(SGLangBaseProcessor):
         video_metadata = None
         if base_output.videos:
             videos_processed = [
-                await preprocess_video(video) for video in base_output.videos
+                await preprocess_video(video, video_config=self.video_config)
+                for video in base_output.videos
             ]
             base_output.videos, video_metadata = map(list, zip(*videos_processed))
 

python/sglang/srt/server_args.py

@@ -297,6 +297,7 @@ class ServerArgs:
     base_gpu_id: int = 0
     gpu_id_step: int = 1
     sleep_on_idle: bool = False
+    mm_process_config: Optional[Dict[str, Any]] = None
 
     # Logging
     log_level: str = "info"
@@ -682,6 +683,8 @@ class ServerArgs:
             self.device = get_device()
         if self.random_seed is None:
             self.random_seed = random.randint(0, 1 << 30)
+        if self.mm_process_config is None:
+            self.mm_process_config = {}
 
     def _handle_gpu_memory_settings(self, gpu_mem):
         """
@@ -2351,6 +2354,12 @@ class ServerArgs:
             action="store_true",
             help="Reduce CPU usage when sglang is idle.",
         )
+        parser.add_argument(
+            "--mm-process-config",
+            type=json.loads,
+            default=ServerArgs.mm_process_config,
+            help="Multimodal preprocessing config, a json config contains keys: `image`, `video`, `audio`",
+        )
 
         # Logging
         parser.add_argument(