From 0b3b3e9a69c4a4dfc7a38162b39f07329f4a2721 Mon Sep 17 00:00:00 2001 From: ash-sigh Date: Mon, 27 Oct 2025 04:06:09 +0800 Subject: [PATCH] transfer mrope_position_delta to device when first running (#11047) --- .../sglang/srt/model_executor/forward_batch_info.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/python/sglang/srt/model_executor/forward_batch_info.py b/python/sglang/srt/model_executor/forward_batch_info.py index de14a5d36..82ea20fe1 100644 --- a/python/sglang/srt/model_executor/forward_batch_info.py +++ b/python/sglang/srt/model_executor/forward_batch_info.py @@ -575,9 +575,15 @@ class ForwardBatch: device=model_runner.device, ) else: - mrope_position_deltas = mm_input.mrope_position_delta.flatten().to( - model_runner.device, non_blocking=True - ) + if mm_input.mrope_position_delta.device.type != model_runner.device: + # transfer mrope_position_delta to device when the first running, + # avoiding successvie host-to-device data transfer + mm_input.mrope_position_delta = ( + mm_input.mrope_position_delta.to( + model_runner.device, non_blocking=True + ) + ) + mrope_position_deltas = mm_input.mrope_position_delta.flatten() mrope_positions_list[batch_idx] = ( (mrope_position_deltas + self.seq_lens[batch_idx] - 1) .unsqueeze(0)