From 53846746bf76082855ad067912d5e1e9ab544515 Mon Sep 17 00:00:00 2001 From: Yuan Luo Date: Wed, 7 Jan 2026 11:30:35 +0800 Subject: [PATCH] [VLM] Fix CUDA IPC OOM (#16118) Co-authored-by: luoyuan.luo --- python/sglang/srt/managers/schedule_batch.py | 3 +++ python/sglang/srt/multimodal/processors/base_processor.py | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/python/sglang/srt/managers/schedule_batch.py b/python/sglang/srt/managers/schedule_batch.py index d423e61d7..50288f3e2 100644 --- a/python/sglang/srt/managers/schedule_batch.py +++ b/python/sglang/srt/managers/schedule_batch.py @@ -1574,6 +1574,9 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin): mm_item.feature = pixel_values.reconstruct_on_target_device( torch.cuda.current_device() ) + # The reference by CudaIpcTensorTransportProxy was cut off, + # proactively delete to avoid slow gc. + del pixel_values self.multimodal_inputs = multimodal_inputs self.token_type_ids = token_type_ids_tensor self.seq_lens_sum = sum(seq_lens) diff --git a/python/sglang/srt/multimodal/processors/base_processor.py b/python/sglang/srt/multimodal/processors/base_processor.py index 0fa304431..306ef5d19 100644 --- a/python/sglang/srt/multimodal/processors/base_processor.py +++ b/python/sglang/srt/multimodal/processors/base_processor.py @@ -880,6 +880,8 @@ class BaseMultimodalProcessor(ABC): info_data=item.feature, sync_buffer_meta=sync_flag, ) + elif not self.server_args.keep_mm_feature_on_device: + item.feature = item.feature.cpu() elif ( isinstance(item.precomputed_embeddings, torch.Tensor) and item.precomputed_embeddings.is_cuda @@ -900,5 +902,7 @@ class BaseMultimodalProcessor(ABC): info_data=item.precomputed_embeddings, sync_buffer_meta=sync_flag, ) + elif not self.server_args.keep_mm_feature_on_device: + item.precomputed_embeddings = item.precomputed_embeddings.cpu() return all_collected_items, input_ids, ret