[Fix] memory leak by overlap + retract (#11981)

Co-authored-by: Liangsheng Yin <lsyincs@gmail.com>
This commit is contained in:
cctry
2025-10-23 22:59:23 +08:00
committed by GitHub
co-authored by Liangsheng Yin
parent 6c18addb6f
commit b0b4f71679
9 changed files with 132 additions and 25 deletions
@@ -4,6 +4,7 @@ import time
from typing import TYPE_CHECKING
from sglang.srt.disaggregation.utils import DisaggregationMode
from sglang.srt.managers.schedule_batch import ScheduleBatch
from sglang.srt.mem_cache.mamba_radix_cache import MambaRadixCache
from sglang.srt.mem_cache.swa_radix_cache import SWARadixCache
@@ -65,6 +66,58 @@ class SchedulerRuntimeCheckerMixin:
token_msg = f"{self.max_total_num_tokens=}, {available_size=}, {evictable_size=}, {protected_size=}\n"
return memory_leak, token_msg
def _check_runtime_mem_leak(self: Scheduler):
current_batch: ScheduleBatch = self.last_batch
if current_batch is None:
return
_, _, available_size, evictable_size = self._get_token_info()
protected_size = self.tree_cache.protected_size()
extend_size = 0
for i, req in enumerate(current_batch.reqs):
seq_len = len(req.origin_input_ids) + len(req.output_ids)
fill_len = len(req.fill_ids) if req.fill_ids is not None else 0
prefix_len = (
len(req.prefix_indices) if req.prefix_indices is not None else 0
)
if current_batch.forward_mode.is_decode():
if req.finished():
unreleased_len = 1
else:
unreleased_len = seq_len - prefix_len
else:
unreleased_len = fill_len - prefix_len
extend_size += unreleased_len
if (
current_batch.forward_mode.is_extend()
and self.running_batch is not None
and not self.running_batch.is_empty()
and self.running_batch.forward_mode.is_decode()
):
for i, req in enumerate(self.running_batch.reqs):
seq_len = len(req.origin_input_ids) + len(req.output_ids)
prefix_len = (
len(req.prefix_indices) if req.prefix_indices is not None else 0
)
if req.finished():
unreleased_len = 0
else:
unreleased_len = seq_len - prefix_len - 1
extend_size += unreleased_len
total_tokens = available_size + evictable_size + protected_size + extend_size
assert (
total_tokens == self.max_total_num_tokens
), f"Mem Leak Detected! {total_tokens=} vs {self.max_total_num_tokens=}"
def _check_req_pool(self: Scheduler):
if self.disaggregation_mode == DisaggregationMode.DECODE:
req_total_size = (