From 42f4b7276c3b5888dec4807ce26df8a5e87e54cc Mon Sep 17 00:00:00 2001 From: Baizhou Zhang Date: Thu, 19 Mar 2026 18:03:04 -0700 Subject: [PATCH] Revert "feat(mm)(grpc): compute M-RoPE positions for preprocessed VL inputs" (#20956) --- python/sglang/srt/managers/scheduler.py | 38 ------------------- .../multimodal/processors/base_processor.py | 8 ---- .../srt/multimodal/processors/ernie45_vl.py | 18 --------- .../sglang/srt/multimodal/processors/glm4v.py | 22 ----------- .../srt/multimodal/processors/qwen_vl.py | 25 ------------ 5 files changed, 111 deletions(-) diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index ae87216e9..da1a9ed58 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -141,7 +141,6 @@ from sglang.srt.managers.io_struct import ( UpdateWeightsFromTensorReqInput, ) from sglang.srt.managers.mm_utils import init_mm_embedding_cache, unwrap_shm_features -from sglang.srt.managers.multimodal_processor import get_mm_processor, import_processors from sglang.srt.managers.overlap_utils import FutureMap from sglang.srt.managers.prefill_delayer import ( PrefillDelayer, @@ -518,24 +517,6 @@ class Scheduler( revision=server_args.revision, ) - # Load multimodal processor for M-RoPE fallback computation. - self._mm_processor = None - if self.model_config.is_multimodal and self.processor is not None: - try: - import_processors("sglang.srt.multimodal.processors") - self._mm_processor = get_mm_processor( - self.model_config.hf_config, - server_args, - self.processor, - "default", - skip_mm_pool=True, - ) - except Exception: - logger.warning( - "Failed to load multimodal processor in scheduler; " - "M-RoPE fallback will not be available." - ) - # Set reasoning_parser and think_end_id if --reasoning_parser is enabled if self.server_args.reasoning_parser and self.tokenizer: reasoning_parser = ReasoningParser( @@ -1628,23 +1609,6 @@ class Scheduler( else: return MultimodalInputs.from_dict(mm_inputs_dict) - def _maybe_compute_mrope_positions(self, req) -> None: - """Compute M-RoPE positions when they are missing (e.g. gRPC preprocessed path).""" - if self._mm_processor is None: - return - mm = req.multimodal_inputs - if mm is None or mm.mrope_positions is not None: - return - - mrope_positions, mrope_position_delta = ( - self._mm_processor.compute_mrope_positions( - req.origin_input_ids, mm.mm_items - ) - ) - if mrope_positions is not None: - mm.mrope_positions = mrope_positions - mm.mrope_position_delta = mrope_position_delta - def _maybe_clear_mm_inputs(self, batch: ScheduleBatch) -> None: for req in batch.reqs: if not req.finished() or not (mm_inputs := req.multimodal_inputs): @@ -1776,7 +1740,6 @@ class Scheduler( req.origin_input_ids, image_inputs ) req.extend_image_inputs(image_inputs) - self._maybe_compute_mrope_positions(req) if len(req.origin_input_ids) >= self.max_req_input_len: req.set_finish_with_abort( @@ -2030,7 +1993,6 @@ class Scheduler( ) req.extend_image_inputs(image_inputs) - self._maybe_compute_mrope_positions(req) if len(req.origin_input_ids) >= self.max_req_input_len: req.set_finish_with_abort( diff --git a/python/sglang/srt/multimodal/processors/base_processor.py b/python/sglang/srt/multimodal/processors/base_processor.py index 859c225be..0121c9755 100644 --- a/python/sglang/srt/multimodal/processors/base_processor.py +++ b/python/sglang/srt/multimodal/processors/base_processor.py @@ -252,14 +252,6 @@ class BaseMultimodalProcessor(ABC): MM_ITEM_MEMORY_POOL_RECYCLE_INTERVAL, ) - def compute_mrope_positions(self, input_ids, mm_items): - """Compute M-RoPE positions from expanded input_ids and multimodal items. - - Returns (mrope_positions, mrope_position_delta) or (None, None) if the - model does not use M-RoPE. - """ - return None, None - @property def spatial_merge_size(self): return self.hf_config.vision_config.spatial_merge_size diff --git a/python/sglang/srt/multimodal/processors/ernie45_vl.py b/python/sglang/srt/multimodal/processors/ernie45_vl.py index b96ffb249..8b3633c59 100644 --- a/python/sglang/srt/multimodal/processors/ernie45_vl.py +++ b/python/sglang/srt/multimodal/processors/ernie45_vl.py @@ -357,24 +357,6 @@ class Ernie4_5_VLImageProcessor(SGLangBaseProcessor): return result - def compute_mrope_positions(self, input_ids, mm_items): - image_grid_thw = None - video_grid_thw = None - for item in mm_items: - if "image_grid_thw" in item.model_specific_data: - image_grid_thw = item.model_specific_data["image_grid_thw"] - if "video_grid_thw" in item.model_specific_data: - video_grid_thw = item.model_specific_data["video_grid_thw"] - - input_ids_tensor = torch.tensor(input_ids, dtype=torch.long).unsqueeze(0) - mrope_positions, mrope_position_delta = MRotaryEmbedding.get_rope_index_ernie45( - input_ids=input_ids_tensor, - hf_config=self.hf_config, - image_grid_thw=image_grid_thw, - video_grid_thw=video_grid_thw, - ) - return mrope_positions.squeeze(1), mrope_position_delta - async def process_mm_data_async( self, image_data: List[Union[str, bytes]], diff --git a/python/sglang/srt/multimodal/processors/glm4v.py b/python/sglang/srt/multimodal/processors/glm4v.py index 8169a764c..33cce6fe2 100644 --- a/python/sglang/srt/multimodal/processors/glm4v.py +++ b/python/sglang/srt/multimodal/processors/glm4v.py @@ -59,28 +59,6 @@ class Glm4vImageProcessor(SGLangBaseProcessor): video_token_id=self.IM_TOKEN_ID, ).build(_processor) - def compute_mrope_positions(self, input_ids, mm_items): - image_grid_thw = None - video_grid_thw = None - for item in mm_items: - if "image_grid_thw" in item.model_specific_data: - image_grid_thw = item.model_specific_data["image_grid_thw"] - if "video_grid_thw" in item.model_specific_data: - video_grid_thw = item.model_specific_data["video_grid_thw"] - - import torch - - input_ids_tensor = torch.tensor(input_ids, dtype=torch.long).unsqueeze(0) - attention_mask = torch.ones_like(input_ids_tensor) - mrope_positions, mrope_position_delta = MRotaryEmbedding.get_rope_index_glm4v( - input_ids=input_ids_tensor, - hf_config=self.hf_config, - image_grid_thw=image_grid_thw, - video_grid_thw=video_grid_thw, - attention_mask=attention_mask, - ) - return mrope_positions.squeeze(1), mrope_position_delta - async def process_mm_data_async( self, image_data: List[Union[str, bytes]], diff --git a/python/sglang/srt/multimodal/processors/qwen_vl.py b/python/sglang/srt/multimodal/processors/qwen_vl.py index 2e1159dcd..e2849acdd 100644 --- a/python/sglang/srt/multimodal/processors/qwen_vl.py +++ b/python/sglang/srt/multimodal/processors/qwen_vl.py @@ -369,31 +369,6 @@ class QwenVLImageProcessor(SGLangBaseProcessor): return input_ids, offsets, modality_list - def compute_mrope_positions(self, input_ids, mm_items): - image_grid_thw = None - video_grid_thw = None - for item in mm_items: - if "image_grid_thw" in item.model_specific_data: - image_grid_thw = item.model_specific_data["image_grid_thw"] - if "video_grid_thw" in item.model_specific_data: - video_grid_thw = item.model_specific_data["video_grid_thw"] - - input_ids_tensor = torch.tensor(input_ids, dtype=torch.long).unsqueeze(0) - mrope_positions, mrope_position_delta = MRotaryEmbedding.get_rope_index( - spatial_merge_size=self.hf_config.vision_config.spatial_merge_size, - image_token_id=self.mm_tokens.image_token_id, - video_token_id=self.mm_tokens.video_token_id, - vision_start_token_id=self.vision_start_token_id, - model_type=self.model_type, - tokens_per_second=getattr( - self.hf_config.vision_config, "tokens_per_second", None - ), - input_ids=input_ids_tensor, - image_grid_thw=image_grid_thw, - video_grid_thw=video_grid_thw, - ) - return mrope_positions.squeeze(1), mrope_position_delta - def get_mm_data(self, prompt, embeddings, **kwargs): img_grid_thw = kwargs.get("img_grid_thw", None) video_grid_thw = kwargs.get("video_grid_thw", None)