Add metrics for having prefill and decode in different ranks (#15752)

This commit is contained in:
fzyzcjy
2025-12-24 21:35:35 +08:00
committed by GitHub
parent b3b818fd86
commit fd4a558e71
9 changed files with 238 additions and 64 deletions
+3
View File
@@ -716,6 +716,7 @@ class Scheduler(
self.last_batch: Optional[ScheduleBatch] = None
self.forward_ct = 0
self.last_prefill_tokens = 0
self.last_prefill_cache_tokens = 0
self.return_health_check_ct = 0
self.num_retracted_reqs: int = 0
self.num_paused_reqs: int = 0
@@ -1830,6 +1831,8 @@ class Scheduler(
if ret:
trace_event_batch("schedule", ret.reqs)
self.log_prefill_stats_late(ret)
return ret
def get_num_allocatable_reqs(self, running_bs):