Add metrics for having prefill and decode in different ranks (#15752)
This commit is contained in:
@@ -716,6 +716,7 @@ class Scheduler(
|
||||
self.last_batch: Optional[ScheduleBatch] = None
|
||||
self.forward_ct = 0
|
||||
self.last_prefill_tokens = 0
|
||||
self.last_prefill_cache_tokens = 0
|
||||
self.return_health_check_ct = 0
|
||||
self.num_retracted_reqs: int = 0
|
||||
self.num_paused_reqs: int = 0
|
||||
@@ -1830,6 +1831,8 @@ class Scheduler(
|
||||
if ret:
|
||||
trace_event_batch("schedule", ret.reqs)
|
||||
|
||||
self.log_prefill_stats_late(ret)
|
||||
|
||||
return ret
|
||||
|
||||
def get_num_allocatable_reqs(self, running_bs):
|
||||
|
||||
Reference in New Issue
Block a user