fix: set first_token_time before computing decode_throughput for single-batch completions (#19984)

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Kangyan-Zhou
2026-03-06 16:11:41 -08:00
committed by GitHub
parent 5c8e28698c
commit 7a12255b6e

View File

@@ -1612,6 +1612,11 @@ class TokenizerManager(TokenizerCommunicatorMixin, TokenizerManagerMultiItemMixi
state.finished = recv_obj.finished_reasons[i] is not None
if state.finished:
# Ensure first_token_time is set before computing decode_throughput.
# Without this, requests that finish on their first output batch
# would have first_token_time=0.0, producing bogus throughput.
if state.time_stats.first_token_time == 0.0:
state.time_stats.set_first_token_time()
state.time_stats.trace_ctx.trace_set_root_attrs(
self.convert_to_span_attrs(state, recv_obj, i)
)