[VLM] Boost Memory Pool based CUDA IPC (#14123)
Co-authored-by: luoyuan.luo <luoyuan.luo@antgroup.com>
This commit is contained in:
@@ -23,6 +23,7 @@ from sglang.srt.utils import (
|
||||
)
|
||||
from sglang.srt.utils.cuda_ipc_transport_utils import (
|
||||
MM_FEATURE_CACHE_SIZE,
|
||||
MM_ITEM_MEMORY_POOL_RECYCLE_INTERVAL,
|
||||
CudaIpcTensorTransportProxy,
|
||||
MmItemMemoryPool,
|
||||
)
|
||||
@@ -225,7 +226,10 @@ class BaseMultimodalProcessor(ABC):
|
||||
]
|
||||
|
||||
if SGL_USE_CUDA_IPC:
|
||||
self.cudaipc_mmfeature_pool = MmItemMemoryPool(MM_FEATURE_CACHE_SIZE)
|
||||
self.cudaipc_mmfeature_pool = MmItemMemoryPool(
|
||||
MM_FEATURE_CACHE_SIZE,
|
||||
MM_ITEM_MEMORY_POOL_RECYCLE_INTERVAL,
|
||||
)
|
||||
|
||||
def process_mm_data(
|
||||
self, input_text, images=None, videos=None, audios=None, **kwargs
|
||||
|
||||
Reference in New Issue
Block a user