Improve engine customization interface (#15635)
This commit is contained in:
@@ -136,7 +136,7 @@ class SchedulerPPMixin:
|
||||
|
||||
# When the server is idle, self-check and re-init some states
|
||||
if server_is_idle:
|
||||
self.check_during_pp_idle()
|
||||
self.self_check_during_idle()
|
||||
|
||||
@DynamicGradMode()
|
||||
def event_loop_pp_disagg_prefill(self: Scheduler):
|
||||
@@ -312,7 +312,7 @@ class SchedulerPPMixin:
|
||||
|
||||
# When the server is idle, self-check and re-init some states
|
||||
if server_is_idle and len(self.disagg_prefill_inflight_queue) == 0:
|
||||
self.check_during_pp_idle()
|
||||
self.self_check_during_idle()
|
||||
|
||||
@DynamicGradMode()
|
||||
def event_loop_pp_disagg_decode(self: Scheduler):
|
||||
@@ -501,7 +501,7 @@ class SchedulerPPMixin:
|
||||
queue_size += len(self.decode_offload_manager.ongoing_offload)
|
||||
|
||||
if server_is_idle and queue_size == 0:
|
||||
self.check_during_pp_idle()
|
||||
self.self_check_during_idle()
|
||||
|
||||
def init_pp_loop_state(self: Scheduler):
|
||||
self.pp_loop_size: int = self.pp_size + self.server_args.pp_async_batch_depth
|
||||
@@ -700,12 +700,6 @@ class SchedulerPPMixin:
|
||||
|
||||
return predicted_size
|
||||
|
||||
def check_during_pp_idle(self: Scheduler):
|
||||
self.check_memory()
|
||||
self.check_tree_cache()
|
||||
self.new_token_ratio = self.init_new_token_ratio
|
||||
self.maybe_sleep_on_idle()
|
||||
|
||||
def process_bootstrapped_queue(
|
||||
self: Scheduler, bootstrapped_rids: Optional[List[str]]
|
||||
):
|
||||
|
||||
Reference in New Issue
Block a user