[Session] Gate streaming sessions with --enable-streaming-session and spec v2 guard (#19531)

This commit is contained in:
Liangsheng Yin
2026-02-27 18:14:55 -08:00
committed by GitHub
parent b5a8e4179e
commit e08ef06758
6 changed files with 62 additions and 17 deletions
@@ -8,6 +8,7 @@ from typing import TYPE_CHECKING
from sglang.srt.disaggregation.utils import DisaggregationMode
from sglang.srt.environ import envs
from sglang.srt.managers.schedule_batch import ScheduleBatch
from sglang.srt.mem_cache.session_aware_cache import SessionAwareCache
from sglang.srt.utils.common import ceil_align, raise_error_or_warn
from sglang.srt.utils.request_logger import disable_request_logging
from sglang.srt.utils.watchdog import WatchdogRaw
@@ -19,6 +20,16 @@ logger = logging.getLogger(__name__)
class SchedulerRuntimeCheckerMixin:
def _session_held_tokens(self: Scheduler) -> int:
if isinstance(self.tree_cache, SessionAwareCache):
return self.tree_cache.session_held_tokens()
return 0
def _session_held_req_count(self: Scheduler) -> int:
if isinstance(self.tree_cache, SessionAwareCache):
return self.tree_cache.session_held_req_count()
return 0
def _get_token_info(self: Scheduler):
available_size = self.token_to_kv_pool_allocator.available_size()
evictable_size = self.tree_cache.evictable_size()
@@ -92,7 +103,7 @@ class SchedulerRuntimeCheckerMixin:
swa_available_size,
swa_evictable_size,
) = self._get_swa_token_info()
session_held = self.tree_cache.session_held_tokens()
session_held = self._session_held_tokens()
memory_leak = (full_num_used - session_held) != 0 or swa_num_used != 0
token_msg = (
f"{self.full_tokens_per_layer=}, {full_available_size=}, {full_evictable_size=}, {self.tree_cache.full_protected_size()=}\n"
@@ -111,7 +122,7 @@ class SchedulerRuntimeCheckerMixin:
mamba_available_size,
mamba_evictable_size,
) = self._get_mamba_token_info()
session_held = self.tree_cache.session_held_tokens()
session_held = self._session_held_tokens()
memory_leak = (
full_num_used != self.tree_cache.full_protected_size() + session_held
or mamba_num_used != self.tree_cache.mamba_protected_size()
@@ -152,7 +163,7 @@ class SchedulerRuntimeCheckerMixin:
def _check_radix_cache_memory(self: Scheduler):
_, _, available_size, evictable_size = self._get_token_info()
protected_size = self.tree_cache.protected_size()
session_held = self.tree_cache.session_held_tokens()
session_held = self._session_held_tokens()
memory_leak = (available_size + evictable_size) != (
self.max_total_num_tokens - protected_size - session_held
)
@@ -204,7 +215,7 @@ class SchedulerRuntimeCheckerMixin:
log_msg = f"[Mem Check (BUSY)] {available_size=}, {evictable_size=}, {protected_size=}, {uncached_size=}"
logger.info(log_msg)
session_held = self.tree_cache.session_held_tokens()
session_held = self._session_held_tokens()
total_tokens = (
available_size
+ evictable_size
@@ -224,7 +235,7 @@ class SchedulerRuntimeCheckerMixin:
else:
req_total_size = self.req_to_token_pool.size
session_req_count = self.tree_cache.session_held_req_count()
session_req_count = self._session_held_req_count()
if len(self.req_to_token_pool.free_slots) + session_req_count != req_total_size:
msg = (
"req_to_token_pool memory leak detected!"