From f0021c0dc8f978ebe6fa595560dcd61511c18dce Mon Sep 17 00:00:00 2001 From: Yuan Luo Date: Sat, 15 Nov 2025 14:43:44 +0800 Subject: [PATCH] Add feature flag for mm inputs processing optimization (#13278) Co-authored-by: luoyuan.luo --- python/sglang/srt/managers/scheduler.py | 10 ++++++++-- python/sglang/srt/server_args.py | 7 +++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index fa48e9d96..b0dd93e7f 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -1239,6 +1239,12 @@ class Scheduler( return image_inputs + def _get_multimodal_inputs(self, mm_inputs_dict: dict): + if self.server_args.enable_broadcast_mm_inputs_process: + return self._process_and_broadcast_mm_inputs(mm_inputs_dict) + else: + return MultimodalInputs.from_dict(mm_inputs_dict) + def handle_generate_request( self, recv_req: TokenizedGenerateReqInput, @@ -1320,7 +1326,7 @@ class Scheduler( # Handle multimodal inputs if recv_req.mm_inputs is not None: - image_inputs = self._process_and_broadcast_mm_inputs(recv_req.mm_inputs) + image_inputs = self._get_multimodal_inputs(recv_req.mm_inputs) # The following steps are already fast, execute locally on each rank. # Expand a single image token into multiple dummy tokens for receiving image embeddings @@ -1555,7 +1561,7 @@ class Scheduler( # Handle multimodal inputs if recv_req.image_inputs is not None: - image_inputs = self._process_and_broadcast_mm_inputs(recv_req.image_inputs) + image_inputs = self._get_multimodal_inputs(recv_req.image_inputs) # Expand a single image token into multiple dummy tokens for receiving image embeddings req.origin_input_ids = self.pad_input_ids_func( req.origin_input_ids, image_inputs diff --git a/python/sglang/srt/server_args.py b/python/sglang/srt/server_args.py index f5cfd59ac..fe7fc1a6b 100644 --- a/python/sglang/srt/server_args.py +++ b/python/sglang/srt/server_args.py @@ -565,6 +565,7 @@ class ServerArgs: # For Multi-Modal mm_max_concurrent_calls: int = 32 mm_per_request_timeout: float = 10.0 + enable_broadcast_mm_inputs_process: bool = False # For checkpoint decryption decrypted_config_file: Optional[str] = None @@ -3656,6 +3657,12 @@ class ServerArgs: default=ServerArgs.mm_per_request_timeout, help="The timeout for each multi-modal request in seconds.", ) + parser.add_argument( + "--enable-broadcast-mm-inputs-process", + action="store_true", + default=ServerArgs.enable_broadcast_mm_inputs_process, + help="Enable broadcast mm-inputs process in scheduler.", + ) # For checkpoint decryption parser.add_argument(