Add request-level timestamp for when prefill finishes (#14860)
This commit is contained in:
@@ -425,6 +425,9 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
zip(batch.reqs, next_token_ids, strict=True)
|
||||
):
|
||||
if req.is_chunked <= 0:
|
||||
if req.time_stats.prefill_finished_ts == 0.0:
|
||||
req.time_stats.prefill_finished_ts = time.time()
|
||||
|
||||
# There is no output_ids for prefill
|
||||
req.output_ids.append(next_token_id)
|
||||
self.tree_cache.cache_unfinished_req(req) # update the tree and lock
|
||||
|
||||
Reference in New Issue
Block a user