Add metrics for having prefill and decode in different ranks (#15752)

This commit is contained in:
fzyzcjy
2025-12-24 21:35:35 +08:00
committed by GitHub
parent b3b818fd86
commit fd4a558e71
9 changed files with 238 additions and 64 deletions
@@ -445,6 +445,10 @@ class SchedulerOutputProcessorMixin:
and self.forward_ct_decode % self.server_args.decode_log_interval == 0
):
self.log_decode_stats(can_run_cuda_graph, running_batch=batch)
if self.enable_metrics:
self.log_decode_stats_every_iteration(
batch, num_accepted_tokens=result.num_accepted_tokens
)
def _mamba_prefix_cache_update(
self, req: Req, batch: ScheduleBatch, result: GenerationBatchResult, i: int