From 0c2993eed03a39f8b927a9c40b440b848408fd3c Mon Sep 17 00:00:00 2001 From: cen121212 Date: Thu, 22 Jan 2026 09:10:08 +0800 Subject: [PATCH] Optimize Qwen3-VL video memory usage (#16366) --- python/sglang/srt/models/qwen3_vl.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/python/sglang/srt/models/qwen3_vl.py b/python/sglang/srt/models/qwen3_vl.py index 079f45843..93661fc78 100644 --- a/python/sglang/srt/models/qwen3_vl.py +++ b/python/sglang/srt/models/qwen3_vl.py @@ -852,10 +852,18 @@ class Qwen3VLForConditionalGeneration(nn.Module): return torch.cat(all_chunk_embeds, dim=0) def get_video_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor: + for item in items: + item.feature = item.feature.to(self.visual.device) # in qwen-vl, last dim is the same pixel_values = torch.cat([item.feature for item in items], dim=0).type( self.visual.dtype ) + # Memory optimization for item.feature: + # 1. item.feature is released when request finished + # 2. High concurrency may cause device OOM due to delayed release + # 3. Fix: Offload item.feature to CPU, move to device only when needed + for item in items: + item.feature = item.feature.to("cpu") video_grid_thw = torch.concat([item.video_grid_thw for item in items], dim=0) assert pixel_values.dim() == 2, pixel_values.dim() assert video_grid_thw.dim() == 2, video_grid_thw.dim()