[Fix] memory leak by overlap + retract (#11981)
Co-authored-by: Liangsheng Yin <lsyincs@gmail.com>
This commit is contained in:
@@ -4,6 +4,7 @@ import time
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from sglang.srt.disaggregation.utils import DisaggregationMode
|
||||
from sglang.srt.managers.schedule_batch import ScheduleBatch
|
||||
from sglang.srt.mem_cache.mamba_radix_cache import MambaRadixCache
|
||||
from sglang.srt.mem_cache.swa_radix_cache import SWARadixCache
|
||||
|
||||
@@ -65,6 +66,58 @@ class SchedulerRuntimeCheckerMixin:
|
||||
token_msg = f"{self.max_total_num_tokens=}, {available_size=}, {evictable_size=}, {protected_size=}\n"
|
||||
return memory_leak, token_msg
|
||||
|
||||
def _check_runtime_mem_leak(self: Scheduler):
|
||||
current_batch: ScheduleBatch = self.last_batch
|
||||
|
||||
if current_batch is None:
|
||||
return
|
||||
|
||||
_, _, available_size, evictable_size = self._get_token_info()
|
||||
protected_size = self.tree_cache.protected_size()
|
||||
|
||||
extend_size = 0
|
||||
for i, req in enumerate(current_batch.reqs):
|
||||
seq_len = len(req.origin_input_ids) + len(req.output_ids)
|
||||
fill_len = len(req.fill_ids) if req.fill_ids is not None else 0
|
||||
prefix_len = (
|
||||
len(req.prefix_indices) if req.prefix_indices is not None else 0
|
||||
)
|
||||
|
||||
if current_batch.forward_mode.is_decode():
|
||||
if req.finished():
|
||||
unreleased_len = 1
|
||||
else:
|
||||
unreleased_len = seq_len - prefix_len
|
||||
else:
|
||||
unreleased_len = fill_len - prefix_len
|
||||
|
||||
extend_size += unreleased_len
|
||||
|
||||
if (
|
||||
current_batch.forward_mode.is_extend()
|
||||
and self.running_batch is not None
|
||||
and not self.running_batch.is_empty()
|
||||
and self.running_batch.forward_mode.is_decode()
|
||||
):
|
||||
for i, req in enumerate(self.running_batch.reqs):
|
||||
seq_len = len(req.origin_input_ids) + len(req.output_ids)
|
||||
prefix_len = (
|
||||
len(req.prefix_indices) if req.prefix_indices is not None else 0
|
||||
)
|
||||
|
||||
if req.finished():
|
||||
unreleased_len = 0
|
||||
else:
|
||||
unreleased_len = seq_len - prefix_len - 1
|
||||
|
||||
extend_size += unreleased_len
|
||||
|
||||
total_tokens = available_size + evictable_size + protected_size + extend_size
|
||||
|
||||
assert (
|
||||
total_tokens == self.max_total_num_tokens
|
||||
), f"Mem Leak Detected! {total_tokens=} vs {self.max_total_num_tokens=}"
|
||||
|
||||
def _check_req_pool(self: Scheduler):
|
||||
if self.disaggregation_mode == DisaggregationMode.DECODE:
|
||||
req_total_size = (
|
||||
|
||||
Reference in New Issue
Block a user