Fix streaming session with paged KV cache (SWA/MLA) (#20070)

Co-authored-by: Yilong Zhao <74357408+happierpig@users.noreply.github.com>
Co-authored-by: Aurick Qiao <6137920+aurickq@users.noreply.github.com>
This commit is contained in:
Liangsheng Yin
2026-03-08 03:00:32 -07:00
committed by GitHub
co-authored by Yilong Zhao Aurick Qiao
parent 230fb55899
commit 36b557d2c9
5 changed files with 65 additions and 12 deletions
@@ -26,6 +26,16 @@ class SchedulerRuntimeCheckerMixin:
return self.tree_cache.session_held_tokens()
return 0
def _session_held_full_tokens(self: Scheduler) -> int:
if isinstance(self.tree_cache, SessionAwareCache):
return self.tree_cache.session_held_full_tokens()
return 0
def _session_held_swa_tokens(self: Scheduler) -> int:
if isinstance(self.tree_cache, SessionAwareCache):
return self.tree_cache.session_held_swa_tokens()
return 0
def _session_held_req_count(self: Scheduler) -> int:
if isinstance(self.tree_cache, SessionAwareCache):
return self.tree_cache.session_held_req_count()
@@ -104,11 +114,20 @@ class SchedulerRuntimeCheckerMixin:
swa_available_size,
swa_evictable_size,
) = self._get_swa_token_info()
session_held = self._session_held_tokens()
memory_leak = (full_num_used - session_held) != 0 or swa_num_used != 0
session_held_full = self._session_held_full_tokens()
session_held_swa = self._session_held_swa_tokens()
# Streaming sessions hold tree locks during idle, so tree-protected
# tokens must be accounted for alongside session-held tokens.
full_protected = self.tree_cache.full_protected_size()
swa_protected = self.tree_cache.swa_protected_size()
full_leaked = full_num_used - full_protected - session_held_full
swa_leaked = swa_num_used - swa_protected - session_held_swa
memory_leak = full_leaked != 0 or swa_leaked != 0
token_msg = (
f"{self.full_tokens_per_layer=}, {full_available_size=}, {full_evictable_size=}, {self.tree_cache.full_protected_size()=}\n"
f"{self.swa_tokens_per_layer=}, {swa_available_size=}, {swa_evictable_size=}, {self.tree_cache.swa_protected_size()=}, {session_held=}\n"
f"{full_leaked=}, {swa_leaked=}\n"
f"{self.full_tokens_per_layer=}, {full_available_size=}, {full_evictable_size=}, {full_protected=}, {session_held_full=}\n"
f"{self.swa_tokens_per_layer=}, {swa_available_size=}, {swa_evictable_size=}, {swa_protected=}, {session_held_swa=}\n"
)
return memory_leak, token_msg