From 7105bf3782ca92ee58402c08815165bd71db022a Mon Sep 17 00:00:00 2001 From: Liangsheng Yin Date: Sun, 8 Mar 2026 15:18:51 -0700 Subject: [PATCH] [Bug] Fix missing TTFT histogram for single-batch requests (#20122) Co-authored-by: Claude Opus 4.6 (1M context) --- python/sglang/srt/managers/tokenizer_manager.py | 16 +++++++++------- test/registered/metrics/test_priority_metrics.py | 1 - 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/python/sglang/srt/managers/tokenizer_manager.py b/python/sglang/srt/managers/tokenizer_manager.py index ac8512fbc..e78869f6b 100644 --- a/python/sglang/srt/managers/tokenizer_manager.py +++ b/python/sglang/srt/managers/tokenizer_manager.py @@ -134,6 +134,7 @@ class ReqState: obj: Union[GenerateReqInput, EmbeddingReqInput] time_stats: APIServerReqTimeStats last_completion_tokens: int = 1 + ttft_observed: bool = False # For streaming output last_output_offset: int = 0 @@ -1611,12 +1612,13 @@ class TokenizerManager(TokenizerCommunicatorMixin, TokenizerManagerMultiItemMixi } state.finished = recv_obj.finished_reasons[i] is not None + + # Set first_token_time on the first output batch. + # This is the single write point for first_token_time. + if state.time_stats.first_token_time == 0.0: + state.time_stats.set_first_token_time() + if state.finished: - # Ensure first_token_time is set before computing decode_throughput. - # Without this, requests that finish on their first output batch - # would have first_token_time=0.0, producing bogus throughput. - if state.time_stats.first_token_time == 0.0: - state.time_stats.set_first_token_time() state.time_stats.trace_ctx.trace_set_root_attrs( self.convert_to_span_attrs(state, recv_obj, i) ) @@ -1927,10 +1929,10 @@ class TokenizerManager(TokenizerCommunicatorMixin, TokenizerManagerMultiItemMixi if priority is not None: labels["priority"] = str(priority) if ( - state.time_stats.first_token_time == 0.0 + not state.ttft_observed and self.disaggregation_mode != DisaggregationMode.PREFILL ): - state.time_stats.set_first_token_time() + state.ttft_observed = True state.last_completion_tokens = completion_tokens self.metrics_collector.observe_time_to_first_token( labels, state.time_stats.get_first_token_latency() diff --git a/test/registered/metrics/test_priority_metrics.py b/test/registered/metrics/test_priority_metrics.py index 1fc5cc37b..b25caea41 100644 --- a/test/registered/metrics/test_priority_metrics.py +++ b/test/registered/metrics/test_priority_metrics.py @@ -19,7 +19,6 @@ from sglang.test.test_utils import ( register_cuda_ci( est_time=60, suite="stage-b-test-small-1-gpu", - disabled="Flaky: no TTFT histogram samples. See https://github.com/sgl-project/sglang/actions/runs/22787468608/job/66118130037", ) register_amd_ci(est_time=60, suite="stage-b-test-small-1-gpu-amd")