Enable RadixCache for Mamba2 models (#13584)
This commit is contained in:
@@ -204,7 +204,7 @@ class SchedulerRuntimeCheckerMixin:
|
||||
def check_memory(self: Scheduler):
|
||||
if self.is_hybrid_swa:
|
||||
memory_leak, token_msg = self._check_hybrid_memory()
|
||||
elif self.is_hybrid_gdn and isinstance(self.tree_cache, MambaRadixCache):
|
||||
elif self.is_ssm_model and isinstance(self.tree_cache, MambaRadixCache):
|
||||
memory_leak, token_msg = self._check_mamba_memory()
|
||||
else:
|
||||
memory_leak, token_msg = self._check_radix_cache_memory()
|
||||
@@ -239,7 +239,7 @@ class SchedulerRuntimeCheckerMixin:
|
||||
) = self._get_swa_token_info()
|
||||
num_used = max(full_num_used, swa_num_used)
|
||||
token_usage = max(full_token_usage, swa_token_usage)
|
||||
elif self.is_hybrid_gdn:
|
||||
elif self.is_ssm_model:
|
||||
(
|
||||
num_used,
|
||||
_,
|
||||
@@ -278,7 +278,7 @@ class SchedulerRuntimeCheckerMixin:
|
||||
|
||||
def check_tree_cache(self: Scheduler):
|
||||
if (self.is_hybrid_swa and isinstance(self.tree_cache, SWARadixCache)) or (
|
||||
self.is_hybrid_gdn and isinstance(self.tree_cache, MambaRadixCache)
|
||||
self.is_ssm_model and isinstance(self.tree_cache, MambaRadixCache)
|
||||
):
|
||||
self.tree_cache.sanity_check()
|
||||
|
||||
@@ -322,7 +322,7 @@ class SchedulerRuntimeCheckerMixin:
|
||||
# Print batch size and memory pool info to check whether there are de-sync issues.
|
||||
if self.is_hybrid_swa:
|
||||
_, info_msg = self._check_hybrid_memory()
|
||||
elif self.is_hybrid_gdn and isinstance(self.tree_cache, MambaRadixCache):
|
||||
elif self.is_ssm_model and isinstance(self.tree_cache, MambaRadixCache):
|
||||
_, info_msg = self._check_mamba_memory()
|
||||
else:
|
||||
_, info_msg = self._check_radix_cache_memory()
|
||||
|
||||
Reference in New Issue
Block a user