Refactor / Unify event loop across PD-Disagg, Overlap, DP-Attn cases (#12839)
Co-authored-by: cctry <17473714+cctry@users.noreply.github.com>
This commit is contained in:
@@ -217,6 +217,17 @@ class SchedulerRuntimeCheckerMixin:
|
||||
self.tree_cache.sanity_check()
|
||||
|
||||
def self_check_during_idle(self: Scheduler):
|
||||
if self.disaggregation_mode == DisaggregationMode.DECODE:
|
||||
queue_size = (
|
||||
len(self.waiting_queue)
|
||||
+ len(self.disagg_decode_transfer_queue.queue)
|
||||
+ len(self.disagg_decode_prealloc_queue.queue)
|
||||
)
|
||||
if self.server_args.disaggregation_decode_enable_offload_kvcache:
|
||||
queue_size += len(self.decode_offload_manager.ongoing_offload)
|
||||
if queue_size:
|
||||
return
|
||||
|
||||
self.check_memory()
|
||||
self.check_tree_cache()
|
||||
self.new_token_ratio = self.init_new_token_ratio
|
||||
|
||||
Reference in New Issue
Block a user