Add request-level timestamp for when prefill finishes (#14860)

This commit is contained in:
Scott Lee
2025-12-17 13:34:46 -08:00
committed by GitHub
parent 4128d4f5cb
commit 169a75dfb8
7 changed files with 42 additions and 6 deletions
@@ -425,6 +425,9 @@ class SchedulerDisaggregationPrefillMixin:
zip(batch.reqs, next_token_ids, strict=True)
):
if req.is_chunked <= 0:
if req.time_stats.prefill_finished_ts == 0.0:
req.time_stats.prefill_finished_ts = time.time()
# There is no output_ids for prefill
req.output_ids.append(next_token_id)
self.tree_cache.cache_unfinished_req(req) # update the tree and lock