From 0a346d3bd9fb6df3a0621eb849b60d9b9cc0e2ba Mon Sep 17 00:00:00 2001 From: Xinyuan Tong <115166877+JustinTong0323@users.noreply.github.com> Date: Sat, 20 Dec 2025 18:44:47 +0000 Subject: [PATCH] feat: Add limit-mm-data-per-request argument to server arguments (#15418) Signed-off-by: Xinyuan Tong --- docs/advanced_features/server_arguments.md | 1 + .../sglang/srt/managers/tokenizer_manager.py | 23 +++++++++---- python/sglang/srt/server_args.py | 33 ++++++++++++++++++- 3 files changed, 50 insertions(+), 7 deletions(-) diff --git a/docs/advanced_features/server_arguments.md b/docs/advanced_features/server_arguments.md index faac0ab49..5bfe2fcdd 100644 --- a/docs/advanced_features/server_arguments.md +++ b/docs/advanced_features/server_arguments.md @@ -93,6 +93,7 @@ Please consult the documentation below and [server_args.py](https://github.com/s | `--context-length` | The model's maximum context length. Defaults to None (will use the value from the model's config.json instead). | `None` | Type: int | | `--is-embedding` | Whether to use a CausalLM as an embedding model. | `False` | bool flag (set to enable) | | `--enable-multimodal` | Enable the multimodal functionality for the served model. If the model being served is not multimodal, nothing will happen | `None` | bool flag (set to enable) | +| `--limit-mm-data-per-request` | Limit the number of multimodal inputs per request. e.g. '{"image": 1, "video": 1, "audio": 1}' | `None` | Type: JSON / Dict | | `--revision` | The specific model version to use. It can be a branch name, a tag name, or a commit id. If unspecified, will use the default version. | `None` | Type: str | | `--model-impl` | Which implementation of the model to use. * "auto" will try to use the SGLang implementation if it exists and fall back to the Transformers implementation if no SGLang implementation is available. * "sglang" will use the SGLang model implementation. * "transformers" will use the Transformers model implementation. | `auto` | Type: str | diff --git a/python/sglang/srt/managers/tokenizer_manager.py b/python/sglang/srt/managers/tokenizer_manager.py index 2d46471de..6c65530bb 100644 --- a/python/sglang/srt/managers/tokenizer_manager.py +++ b/python/sglang/srt/managers/tokenizer_manager.py @@ -31,7 +31,6 @@ from http import HTTPStatus from typing import Any, Awaitable, Dict, List, Optional, Tuple, Union import fastapi -import orjson import uvloop import zmq import zmq.asyncio @@ -184,11 +183,7 @@ class TokenizerManager(TokenizerCommunicatorMixin, TokenizerManagerMultiItemMixi self.enable_metrics = server_args.enable_metrics self.log_requests = server_args.log_requests self.log_requests_level = server_args.log_requests_level - self.preferred_sampling_params = ( - orjson.loads(server_args.preferred_sampling_params) - if server_args.preferred_sampling_params - else None - ) + self.preferred_sampling_params = server_args.preferred_sampling_params self.crash_dump_folder = server_args.crash_dump_folder self.enable_trace = server_args.enable_trace @@ -628,6 +623,7 @@ class TokenizerManager(TokenizerCommunicatorMixin, TokenizerManagerMultiItemMixi obj.image_data = [obj.image_data] if obj.audio_data is not None and not isinstance(obj.audio_data, list): obj.audio_data = [obj.audio_data] + self._validate_mm_limits(obj) mm_inputs = None @@ -748,6 +744,21 @@ class TokenizerManager(TokenizerCommunicatorMixin, TokenizerManagerMultiItemMixi "Please set `--enable-custom-logit-processor` to enable this feature." ) + def _validate_mm_limits( + self, obj: Union[GenerateReqInput, EmbeddingReqInput] + ) -> None: + if not self.server_args.limit_mm_data_per_request: + return + + for modality, limit in self.server_args.limit_mm_data_per_request.items(): + data = getattr(obj, f"{modality}_data", None) + if data: + count = len(data) if isinstance(data, list) else 1 + if count > limit: + raise ValueError( + f"{modality.capitalize()} count {count} exceeds limit {limit} per request." + ) + def _validate_for_matryoshka_dim(self, obj: EmbeddingReqInput) -> None: """Validate the request for Matryoshka dim if it has the field set.""" if obj.dimensions is None: diff --git a/python/sglang/srt/server_args.py b/python/sglang/srt/server_args.py index 56ac53a5d..553cb1c49 100644 --- a/python/sglang/srt/server_args.py +++ b/python/sglang/srt/server_args.py @@ -264,6 +264,7 @@ class ServerArgs: context_length: Optional[int] = None is_embedding: bool = False enable_multimodal: Optional[bool] = None + limit_mm_data_per_request: Optional[Union[str, Dict[str, int]]] = None revision: Optional[str] = None model_impl: str = "auto" @@ -2367,6 +2368,29 @@ class ServerArgs: self.disable_cuda_graph = True self.skip_server_warmup = True + # Validate limit_mm_per_prompt modalities + if self.limit_mm_data_per_request: + if isinstance(self.limit_mm_data_per_request, str): + self.limit_mm_data_per_request = json.loads( + self.limit_mm_data_per_request + ) + + if isinstance(self.limit_mm_data_per_request, dict): + allowed_modalities = {"image", "video", "audio"} + for modality in self.limit_mm_data_per_request.keys(): + if modality not in allowed_modalities: + raise ValueError( + f"Invalid modality '{modality}' in --limit-mm-data-per-request." + f"Allowed modalities are: {list(allowed_modalities)}" + ) + + # Validate preferred_sampling_params + if self.preferred_sampling_params: + if isinstance(self.preferred_sampling_params, str): + self.preferred_sampling_params = json.loads( + self.preferred_sampling_params + ) + @staticmethod def add_cli_args(parser: argparse.ArgumentParser): @@ -2455,6 +2479,13 @@ class ServerArgs: action="store_true", help="Enable the multimodal functionality for the served model. If the model being served is not multimodal, nothing will happen", ) + parser.add_argument( + "--limit-mm-data-per-request", + type=json.loads, + default=ServerArgs.limit_mm_data_per_request, + help="Limit the number of multimodal inputs per request. " + 'e.g. \'{"image": 1, "video": 1, "audio": 1}\'', + ) parser.add_argument( "--revision", type=str, @@ -3135,7 +3166,7 @@ class ServerArgs: ) parser.add_argument( "--preferred-sampling-params", - type=str, + type=json.loads, help="json-formatted sampling settings that will be returned in /get_model_info", )