Fix crash after flush cache (#12107)
Co-authored-by: fzyzcjy <5236035+fzyzcjy@users.noreply.github.com>
This commit is contained in:
@@ -2305,13 +2305,30 @@ class Scheduler(
|
||||
if_success = False
|
||||
return ClearHiCacheReqOutput(success=if_success)
|
||||
|
||||
def flush_cache(self):
|
||||
"""Flush the memory pool and cache."""
|
||||
if (
|
||||
def _is_no_request(self):
|
||||
no_request = (
|
||||
len(self.waiting_queue) == 0
|
||||
and self.running_batch.is_empty()
|
||||
and (self.last_batch is None or self.last_batch.is_empty())
|
||||
and (self.cur_batch is None or self.cur_batch.is_empty())
|
||||
and (not self.enable_overlap or len(self.result_queue) == 0)
|
||||
and (self.pp_size == 1 or all(x.is_empty() for x in self.running_mbs))
|
||||
):
|
||||
)
|
||||
if self.disaggregation_mode == DisaggregationMode.PREFILL:
|
||||
no_request &= (
|
||||
len(self.disagg_prefill_bootstrap_queue.queue) == 0
|
||||
and len(self.disagg_prefill_inflight_queue) == 0
|
||||
)
|
||||
if self.disaggregation_mode == DisaggregationMode.DECODE:
|
||||
no_request &= (
|
||||
len(self.disagg_decode_prealloc_queue.queue) == 0
|
||||
and len(self.disagg_decode_transfer_queue.queue) == 0
|
||||
)
|
||||
return no_request
|
||||
|
||||
def flush_cache(self):
|
||||
"""Flush the memory pool and cache."""
|
||||
if self._is_no_request():
|
||||
self.cur_batch = None
|
||||
self.last_batch = None
|
||||
self.tree_cache.reset()
|
||||
|
||||
@@ -533,6 +533,10 @@ class RadixCache(BasePrefixCache):
|
||||
self.protected_size_ -= len(node.key)
|
||||
delta += len(node.key)
|
||||
node.lock_ref -= 1
|
||||
if node.parent is None:
|
||||
assert (
|
||||
node is self.root_node
|
||||
), f"This request holds the node from another tree"
|
||||
node = node.parent
|
||||
return delta
|
||||
|
||||
|
||||
Reference in New Issue
Block a user