Refactor prefix cache type checking (#17028)
This commit is contained in:
@@ -8,8 +8,6 @@ from typing import TYPE_CHECKING
|
||||
from sglang.srt.disaggregation.utils import DisaggregationMode
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.managers.schedule_batch import ScheduleBatch
|
||||
from sglang.srt.mem_cache.mamba_radix_cache import MambaRadixCache
|
||||
from sglang.srt.mem_cache.swa_radix_cache import SWARadixCache
|
||||
from sglang.srt.utils.common import ceil_align, raise_error_or_warn
|
||||
from sglang.srt.utils.request_logger import disable_request_logging
|
||||
from sglang.srt.utils.watchdog import WatchdogRaw
|
||||
@@ -29,14 +27,16 @@ class SchedulerRuntimeCheckerMixin:
|
||||
return num_used, token_usage, available_size, evictable_size
|
||||
|
||||
def _get_mamba_token_info(self: Scheduler):
|
||||
is_radix_tree = isinstance(self.tree_cache, MambaRadixCache)
|
||||
is_mamba_radix_cache = (
|
||||
self.tree_cache.supports_mamba() and self.tree_cache.is_tree_cache()
|
||||
)
|
||||
full_available_size = self.token_to_kv_pool_allocator.available_size()
|
||||
full_evictable_size = (
|
||||
self.tree_cache.full_evictable_size() if is_radix_tree else 0
|
||||
self.tree_cache.full_evictable_size() if is_mamba_radix_cache else 0
|
||||
)
|
||||
mamba_available_size = self.req_to_token_pool.mamba_pool.available_size()
|
||||
mamba_evictable_size = (
|
||||
self.tree_cache.mamba_evictable_size() if is_radix_tree else 0
|
||||
self.tree_cache.mamba_evictable_size() if is_mamba_radix_cache else 0
|
||||
)
|
||||
full_num_used = self.token_to_kv_pool_allocator.size - (
|
||||
full_available_size + full_evictable_size
|
||||
@@ -234,7 +234,7 @@ class SchedulerRuntimeCheckerMixin:
|
||||
def check_memory(self: Scheduler):
|
||||
if self.is_hybrid_swa:
|
||||
memory_leak, token_msg = self._check_hybrid_memory()
|
||||
elif self.is_hybrid_ssm and isinstance(self.tree_cache, MambaRadixCache):
|
||||
elif self.is_hybrid_ssm and self.tree_cache.supports_mamba():
|
||||
memory_leak, token_msg = self._check_mamba_memory()
|
||||
else:
|
||||
memory_leak, token_msg = self._check_radix_cache_memory()
|
||||
@@ -307,8 +307,10 @@ class SchedulerRuntimeCheckerMixin:
|
||||
self._publish_kv_events()
|
||||
|
||||
def check_tree_cache(self: Scheduler):
|
||||
if (self.is_hybrid_swa and isinstance(self.tree_cache, SWARadixCache)) or (
|
||||
self.is_hybrid_ssm and isinstance(self.tree_cache, MambaRadixCache)
|
||||
if (
|
||||
self.tree_cache.is_tree_cache()
|
||||
and (self.is_hybrid_swa and self.tree_cache.supports_swa())
|
||||
or (self.is_hybrid_ssm and self.tree_cache.supports_mamba())
|
||||
):
|
||||
self.tree_cache.sanity_check()
|
||||
|
||||
@@ -341,9 +343,7 @@ def create_scheduler_watchdog(
|
||||
return ""
|
||||
if scheduler.is_hybrid_swa:
|
||||
_, info_msg = scheduler._check_hybrid_memory()
|
||||
elif scheduler.is_hybrid_ssm and isinstance(
|
||||
scheduler.tree_cache, MambaRadixCache
|
||||
):
|
||||
elif scheduler.is_hybrid_ssm and scheduler.tree_cache.supports_mamba():
|
||||
_, info_msg = scheduler._check_mamba_memory()
|
||||
else:
|
||||
_, info_msg = scheduler._check_radix_cache_memory()
|
||||
|
||||
Reference in New Issue
Block a user