[Session] Gate streaming sessions with --enable-streaming-session and spec v2 guard (#19531)
This commit is contained in:
@@ -8,6 +8,7 @@ from typing import TYPE_CHECKING
|
||||
from sglang.srt.disaggregation.utils import DisaggregationMode
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.managers.schedule_batch import ScheduleBatch
|
||||
from sglang.srt.mem_cache.session_aware_cache import SessionAwareCache
|
||||
from sglang.srt.utils.common import ceil_align, raise_error_or_warn
|
||||
from sglang.srt.utils.request_logger import disable_request_logging
|
||||
from sglang.srt.utils.watchdog import WatchdogRaw
|
||||
@@ -19,6 +20,16 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SchedulerRuntimeCheckerMixin:
|
||||
def _session_held_tokens(self: Scheduler) -> int:
|
||||
if isinstance(self.tree_cache, SessionAwareCache):
|
||||
return self.tree_cache.session_held_tokens()
|
||||
return 0
|
||||
|
||||
def _session_held_req_count(self: Scheduler) -> int:
|
||||
if isinstance(self.tree_cache, SessionAwareCache):
|
||||
return self.tree_cache.session_held_req_count()
|
||||
return 0
|
||||
|
||||
def _get_token_info(self: Scheduler):
|
||||
available_size = self.token_to_kv_pool_allocator.available_size()
|
||||
evictable_size = self.tree_cache.evictable_size()
|
||||
@@ -92,7 +103,7 @@ class SchedulerRuntimeCheckerMixin:
|
||||
swa_available_size,
|
||||
swa_evictable_size,
|
||||
) = self._get_swa_token_info()
|
||||
session_held = self.tree_cache.session_held_tokens()
|
||||
session_held = self._session_held_tokens()
|
||||
memory_leak = (full_num_used - session_held) != 0 or swa_num_used != 0
|
||||
token_msg = (
|
||||
f"{self.full_tokens_per_layer=}, {full_available_size=}, {full_evictable_size=}, {self.tree_cache.full_protected_size()=}\n"
|
||||
@@ -111,7 +122,7 @@ class SchedulerRuntimeCheckerMixin:
|
||||
mamba_available_size,
|
||||
mamba_evictable_size,
|
||||
) = self._get_mamba_token_info()
|
||||
session_held = self.tree_cache.session_held_tokens()
|
||||
session_held = self._session_held_tokens()
|
||||
memory_leak = (
|
||||
full_num_used != self.tree_cache.full_protected_size() + session_held
|
||||
or mamba_num_used != self.tree_cache.mamba_protected_size()
|
||||
@@ -152,7 +163,7 @@ class SchedulerRuntimeCheckerMixin:
|
||||
def _check_radix_cache_memory(self: Scheduler):
|
||||
_, _, available_size, evictable_size = self._get_token_info()
|
||||
protected_size = self.tree_cache.protected_size()
|
||||
session_held = self.tree_cache.session_held_tokens()
|
||||
session_held = self._session_held_tokens()
|
||||
memory_leak = (available_size + evictable_size) != (
|
||||
self.max_total_num_tokens - protected_size - session_held
|
||||
)
|
||||
@@ -204,7 +215,7 @@ class SchedulerRuntimeCheckerMixin:
|
||||
log_msg = f"[Mem Check (BUSY)] {available_size=}, {evictable_size=}, {protected_size=}, {uncached_size=}"
|
||||
logger.info(log_msg)
|
||||
|
||||
session_held = self.tree_cache.session_held_tokens()
|
||||
session_held = self._session_held_tokens()
|
||||
total_tokens = (
|
||||
available_size
|
||||
+ evictable_size
|
||||
@@ -224,7 +235,7 @@ class SchedulerRuntimeCheckerMixin:
|
||||
else:
|
||||
req_total_size = self.req_to_token_pool.size
|
||||
|
||||
session_req_count = self.tree_cache.session_held_req_count()
|
||||
session_req_count = self._session_held_req_count()
|
||||
if len(self.req_to_token_pool.free_slots) + session_req_count != req_total_size:
|
||||
msg = (
|
||||
"req_to_token_pool memory leak detected!"
|
||||
|
||||
Reference in New Issue
Block a user