[VLM] Boost Memory Pool based CUDA IPC (#14123)

Co-authored-by: luoyuan.luo <luoyuan.luo@antgroup.com>
This commit is contained in:
Yuan Luo
2025-12-01 17:17:46 +08:00
committed by GitHub
co-authored by luoyuan.luo
parent 7ce8faae28
commit 630a693081
3 changed files with 74 additions and 17 deletions
@@ -23,6 +23,7 @@ from sglang.srt.utils import (
)
from sglang.srt.utils.cuda_ipc_transport_utils import (
MM_FEATURE_CACHE_SIZE,
MM_ITEM_MEMORY_POOL_RECYCLE_INTERVAL,
CudaIpcTensorTransportProxy,
MmItemMemoryPool,
)
@@ -225,7 +226,10 @@ class BaseMultimodalProcessor(ABC):
]
if SGL_USE_CUDA_IPC:
self.cudaipc_mmfeature_pool = MmItemMemoryPool(MM_FEATURE_CACHE_SIZE)
self.cudaipc_mmfeature_pool = MmItemMemoryPool(
MM_FEATURE_CACHE_SIZE,
MM_ITEM_MEMORY_POOL_RECYCLE_INTERVAL,
)
def process_mm_data(
self, input_text, images=None, videos=None, audios=None, **kwargs