Enable RadixCache for Mamba2 models (#13584)

This commit is contained in:
roikoren755
2025-12-05 18:23:58 +08:00
committed by GitHub
parent 35ba6fe19e
commit 2ce121a1c3
6 changed files with 29 additions and 19 deletions
@@ -204,7 +204,7 @@ class SchedulerRuntimeCheckerMixin:
def check_memory(self: Scheduler):
if self.is_hybrid_swa:
memory_leak, token_msg = self._check_hybrid_memory()
elif self.is_hybrid_gdn and isinstance(self.tree_cache, MambaRadixCache):
elif self.is_ssm_model and isinstance(self.tree_cache, MambaRadixCache):
memory_leak, token_msg = self._check_mamba_memory()
else:
memory_leak, token_msg = self._check_radix_cache_memory()
@@ -239,7 +239,7 @@ class SchedulerRuntimeCheckerMixin:
) = self._get_swa_token_info()
num_used = max(full_num_used, swa_num_used)
token_usage = max(full_token_usage, swa_token_usage)
elif self.is_hybrid_gdn:
elif self.is_ssm_model:
(
num_used,
_,
@@ -278,7 +278,7 @@ class SchedulerRuntimeCheckerMixin:
def check_tree_cache(self: Scheduler):
if (self.is_hybrid_swa and isinstance(self.tree_cache, SWARadixCache)) or (
self.is_hybrid_gdn and isinstance(self.tree_cache, MambaRadixCache)
self.is_ssm_model and isinstance(self.tree_cache, MambaRadixCache)
):
self.tree_cache.sanity_check()
@@ -322,7 +322,7 @@ class SchedulerRuntimeCheckerMixin:
# Print batch size and memory pool info to check whether there are de-sync issues.
if self.is_hybrid_swa:
_, info_msg = self._check_hybrid_memory()
elif self.is_hybrid_gdn and isinstance(self.tree_cache, MambaRadixCache):
elif self.is_ssm_model and isinstance(self.tree_cache, MambaRadixCache):
_, info_msg = self._check_mamba_memory()
else:
_, info_msg = self._check_radix_cache_memory()