diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index fcacdf865..6bac100b1 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -137,6 +137,7 @@ from sglang.srt.managers.io_struct import ( UpdateWeightsFromTensorReqInput, ) from sglang.srt.managers.mm_utils import init_mm_embedding_cache, unwrap_shm_features +from sglang.srt.managers.multimodal_processor import get_mm_processor, import_processors from sglang.srt.managers.overlap_utils import FutureMap from sglang.srt.managers.prefill_delayer import ( PrefillDelayer, @@ -513,6 +514,24 @@ class Scheduler( revision=server_args.revision, ) + # Load multimodal processor for M-RoPE fallback computation. + self._mm_processor = None + if self.model_config.is_multimodal and self.processor is not None: + try: + import_processors("sglang.srt.multimodal.processors") + self._mm_processor = get_mm_processor( + self.model_config.hf_config, + server_args, + self.processor, + "default", + skip_mm_pool=True, + ) + except Exception: + logger.warning( + "Failed to load multimodal processor in scheduler; " + "M-RoPE fallback will not be available." + ) + # Set reasoning_parser and think_end_id if --reasoning_parser is enabled if self.server_args.reasoning_parser and self.tokenizer: reasoning_parser = ReasoningParser( @@ -1605,6 +1624,23 @@ class Scheduler( else: return MultimodalInputs.from_dict(mm_inputs_dict) + def _maybe_compute_mrope_positions(self, req) -> None: + """Compute M-RoPE positions when they are missing (e.g. gRPC preprocessed path).""" + if self._mm_processor is None: + return + mm = req.multimodal_inputs + if mm is None or mm.mrope_positions is not None: + return + + mrope_positions, mrope_position_delta = ( + self._mm_processor.compute_mrope_positions( + req.origin_input_ids, mm.mm_items + ) + ) + if mrope_positions is not None: + mm.mrope_positions = mrope_positions + mm.mrope_position_delta = mrope_position_delta + def _maybe_clear_mm_inputs(self, batch: ScheduleBatch) -> None: for req in batch.reqs: if not req.finished() or not (mm_inputs := req.multimodal_inputs): @@ -1736,6 +1772,7 @@ class Scheduler( req.origin_input_ids, image_inputs ) req.extend_image_inputs(image_inputs) + self._maybe_compute_mrope_positions(req) if len(req.origin_input_ids) >= self.max_req_input_len: req.set_finish_with_abort( @@ -1989,6 +2026,7 @@ class Scheduler( ) req.extend_image_inputs(image_inputs) + self._maybe_compute_mrope_positions(req) if len(req.origin_input_ids) >= self.max_req_input_len: req.set_finish_with_abort( diff --git a/python/sglang/srt/multimodal/processors/base_processor.py b/python/sglang/srt/multimodal/processors/base_processor.py index 0121c9755..859c225be 100644 --- a/python/sglang/srt/multimodal/processors/base_processor.py +++ b/python/sglang/srt/multimodal/processors/base_processor.py @@ -252,6 +252,14 @@ class BaseMultimodalProcessor(ABC): MM_ITEM_MEMORY_POOL_RECYCLE_INTERVAL, ) + def compute_mrope_positions(self, input_ids, mm_items): + """Compute M-RoPE positions from expanded input_ids and multimodal items. + + Returns (mrope_positions, mrope_position_delta) or (None, None) if the + model does not use M-RoPE. + """ + return None, None + @property def spatial_merge_size(self): return self.hf_config.vision_config.spatial_merge_size diff --git a/python/sglang/srt/multimodal/processors/ernie45_vl.py b/python/sglang/srt/multimodal/processors/ernie45_vl.py index 8b3633c59..b96ffb249 100644 --- a/python/sglang/srt/multimodal/processors/ernie45_vl.py +++ b/python/sglang/srt/multimodal/processors/ernie45_vl.py @@ -357,6 +357,24 @@ class Ernie4_5_VLImageProcessor(SGLangBaseProcessor): return result + def compute_mrope_positions(self, input_ids, mm_items): + image_grid_thw = None + video_grid_thw = None + for item in mm_items: + if "image_grid_thw" in item.model_specific_data: + image_grid_thw = item.model_specific_data["image_grid_thw"] + if "video_grid_thw" in item.model_specific_data: + video_grid_thw = item.model_specific_data["video_grid_thw"] + + input_ids_tensor = torch.tensor(input_ids, dtype=torch.long).unsqueeze(0) + mrope_positions, mrope_position_delta = MRotaryEmbedding.get_rope_index_ernie45( + input_ids=input_ids_tensor, + hf_config=self.hf_config, + image_grid_thw=image_grid_thw, + video_grid_thw=video_grid_thw, + ) + return mrope_positions.squeeze(1), mrope_position_delta + async def process_mm_data_async( self, image_data: List[Union[str, bytes]], diff --git a/python/sglang/srt/multimodal/processors/glm4v.py b/python/sglang/srt/multimodal/processors/glm4v.py index 33cce6fe2..8169a764c 100644 --- a/python/sglang/srt/multimodal/processors/glm4v.py +++ b/python/sglang/srt/multimodal/processors/glm4v.py @@ -59,6 +59,28 @@ class Glm4vImageProcessor(SGLangBaseProcessor): video_token_id=self.IM_TOKEN_ID, ).build(_processor) + def compute_mrope_positions(self, input_ids, mm_items): + image_grid_thw = None + video_grid_thw = None + for item in mm_items: + if "image_grid_thw" in item.model_specific_data: + image_grid_thw = item.model_specific_data["image_grid_thw"] + if "video_grid_thw" in item.model_specific_data: + video_grid_thw = item.model_specific_data["video_grid_thw"] + + import torch + + input_ids_tensor = torch.tensor(input_ids, dtype=torch.long).unsqueeze(0) + attention_mask = torch.ones_like(input_ids_tensor) + mrope_positions, mrope_position_delta = MRotaryEmbedding.get_rope_index_glm4v( + input_ids=input_ids_tensor, + hf_config=self.hf_config, + image_grid_thw=image_grid_thw, + video_grid_thw=video_grid_thw, + attention_mask=attention_mask, + ) + return mrope_positions.squeeze(1), mrope_position_delta + async def process_mm_data_async( self, image_data: List[Union[str, bytes]], diff --git a/python/sglang/srt/multimodal/processors/qwen_vl.py b/python/sglang/srt/multimodal/processors/qwen_vl.py index e2849acdd..2e1159dcd 100644 --- a/python/sglang/srt/multimodal/processors/qwen_vl.py +++ b/python/sglang/srt/multimodal/processors/qwen_vl.py @@ -369,6 +369,31 @@ class QwenVLImageProcessor(SGLangBaseProcessor): return input_ids, offsets, modality_list + def compute_mrope_positions(self, input_ids, mm_items): + image_grid_thw = None + video_grid_thw = None + for item in mm_items: + if "image_grid_thw" in item.model_specific_data: + image_grid_thw = item.model_specific_data["image_grid_thw"] + if "video_grid_thw" in item.model_specific_data: + video_grid_thw = item.model_specific_data["video_grid_thw"] + + input_ids_tensor = torch.tensor(input_ids, dtype=torch.long).unsqueeze(0) + mrope_positions, mrope_position_delta = MRotaryEmbedding.get_rope_index( + spatial_merge_size=self.hf_config.vision_config.spatial_merge_size, + image_token_id=self.mm_tokens.image_token_id, + video_token_id=self.mm_tokens.video_token_id, + vision_start_token_id=self.vision_start_token_id, + model_type=self.model_type, + tokens_per_second=getattr( + self.hf_config.vision_config, "tokens_per_second", None + ), + input_ids=input_ids_tensor, + image_grid_thw=image_grid_thw, + video_grid_thw=video_grid_thw, + ) + return mrope_positions.squeeze(1), mrope_position_delta + def get_mm_data(self, prompt, embeddings, **kwargs): img_grid_thw = kwargs.get("img_grid_thw", None) video_grid_thw = kwargs.get("video_grid_thw", None)