Refactor prefix cache type checking (#17028)

This commit is contained in:
Ke Bao
2026-01-15 11:28:13 +08:00
committed by GitHub
parent 6b065298b5
commit 7f8a58fffb
12 changed files with 71 additions and 36 deletions
@@ -8,8 +8,6 @@ from typing import TYPE_CHECKING
from sglang.srt.disaggregation.utils import DisaggregationMode
from sglang.srt.environ import envs
from sglang.srt.managers.schedule_batch import ScheduleBatch
from sglang.srt.mem_cache.mamba_radix_cache import MambaRadixCache
from sglang.srt.mem_cache.swa_radix_cache import SWARadixCache
from sglang.srt.utils.common import ceil_align, raise_error_or_warn
from sglang.srt.utils.request_logger import disable_request_logging
from sglang.srt.utils.watchdog import WatchdogRaw
@@ -29,14 +27,16 @@ class SchedulerRuntimeCheckerMixin:
return num_used, token_usage, available_size, evictable_size
def _get_mamba_token_info(self: Scheduler):
is_radix_tree = isinstance(self.tree_cache, MambaRadixCache)
is_mamba_radix_cache = (
self.tree_cache.supports_mamba() and self.tree_cache.is_tree_cache()
)
full_available_size = self.token_to_kv_pool_allocator.available_size()
full_evictable_size = (
self.tree_cache.full_evictable_size() if is_radix_tree else 0
self.tree_cache.full_evictable_size() if is_mamba_radix_cache else 0
)
mamba_available_size = self.req_to_token_pool.mamba_pool.available_size()
mamba_evictable_size = (
self.tree_cache.mamba_evictable_size() if is_radix_tree else 0
self.tree_cache.mamba_evictable_size() if is_mamba_radix_cache else 0
)
full_num_used = self.token_to_kv_pool_allocator.size - (
full_available_size + full_evictable_size
@@ -234,7 +234,7 @@ class SchedulerRuntimeCheckerMixin:
def check_memory(self: Scheduler):
if self.is_hybrid_swa:
memory_leak, token_msg = self._check_hybrid_memory()
elif self.is_hybrid_ssm and isinstance(self.tree_cache, MambaRadixCache):
elif self.is_hybrid_ssm and self.tree_cache.supports_mamba():
memory_leak, token_msg = self._check_mamba_memory()
else:
memory_leak, token_msg = self._check_radix_cache_memory()
@@ -307,8 +307,10 @@ class SchedulerRuntimeCheckerMixin:
self._publish_kv_events()
def check_tree_cache(self: Scheduler):
if (self.is_hybrid_swa and isinstance(self.tree_cache, SWARadixCache)) or (
self.is_hybrid_ssm and isinstance(self.tree_cache, MambaRadixCache)
if (
self.tree_cache.is_tree_cache()
and (self.is_hybrid_swa and self.tree_cache.supports_swa())
or (self.is_hybrid_ssm and self.tree_cache.supports_mamba())
):
self.tree_cache.sanity_check()
@@ -341,9 +343,7 @@ def create_scheduler_watchdog(
return ""
if scheduler.is_hybrid_swa:
_, info_msg = scheduler._check_hybrid_memory()
elif scheduler.is_hybrid_ssm and isinstance(
scheduler.tree_cache, MambaRadixCache
):
elif scheduler.is_hybrid_ssm and scheduler.tree_cache.supports_mamba():
_, info_msg = scheduler._check_mamba_memory()
else:
_, info_msg = scheduler._check_radix_cache_memory()