Add metrics for having prefill and decode in different ranks (#15752)
This commit is contained in:
@@ -445,6 +445,10 @@ class SchedulerOutputProcessorMixin:
|
||||
and self.forward_ct_decode % self.server_args.decode_log_interval == 0
|
||||
):
|
||||
self.log_decode_stats(can_run_cuda_graph, running_batch=batch)
|
||||
if self.enable_metrics:
|
||||
self.log_decode_stats_every_iteration(
|
||||
batch, num_accepted_tokens=result.num_accepted_tokens
|
||||
)
|
||||
|
||||
def _mamba_prefix_cache_update(
|
||||
self, req: Req, batch: ScheduleBatch, result: GenerationBatchResult, i: int
|
||||
|
||||
Reference in New Issue
Block a user