chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 11 Packages Projects Releases Wiki Activity

Add timing metrics for requests (#12646)

Browse Source 
Co-authored-by: Scott Lee <scottjlee@users.noreply.github.com>
This commit is contained in:
yinghui
2025-11-05 23:07:16 -08:00
committed by GitHub
co-authored by Scott Lee
parent fd3034da75
commit 58095cb00a
9 changed files with 334 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/managers/scheduler_output_processor_mixin.py
+92 -30
View File
@@ -275,6 +275,7 @@ class SchedulerOutputProcessorMixin:
next_token_ids[i * stride : i * stride + accept_lens[i]]
)
req.spec_verify_ct += 1
req.spec_accepted_tokens += accept_lens[i] - 1
return predict_tokens
@@ -760,6 +761,11 @@ class SchedulerOutputProcessorMixin:
retraction_counts = []
output_hidden_states = None
queue_times = []
forward_entry_times = []
prefill_delays = []
prefill_latencies = []
if return_logprob:
input_token_logprobs_val = []
input_token_logprobs_idx = []
@@ -860,6 +866,28 @@ class SchedulerOutputProcessorMixin:
cached_tokens.append(req.cached_tokens)
retraction_counts.append(req.retraction_count)
queue_times.append(req.time_stats.get_queueing_time())
forward_entry_times.append(req.time_stats.forward_entry_time)
if req.time_stats.prefill_start_time > 0.0:
prefill_delays.append(
req.time_stats.prefill_start_time
- req.time_stats.forward_entry_time
)
else:
prefill_delays.append(None)
if (
req.time_stats.prefill_start_time > 0.0
and req.time_stats.prefill_end_time > 0.0
):
prefill_latencies.append(
req.time_stats.prefill_end_time
- req.time_stats.prefill_start_time
)
else:
prefill_latencies.append(None)
if not self.spec_algorithm.is_none():
spec_verify_ct.append(req.spec_verify_ct)
spec_accepted_tokens.append(req.spec_accepted_tokens)
@@ -951,31 +979,35 @@ class SchedulerOutputProcessorMixin:
self.send_to_detokenizer.send_output(
BatchTokenIDOutput(
finished_reasons,
decoded_texts,
decode_ids_list,
read_offsets,
output_ids,
skip_special_tokens,
spaces_between_special_tokens,
no_stop_trim,
prompt_tokens,
completion_tokens,
cached_tokens,
spec_verify_ct,
spec_accepted_tokens,
input_token_logprobs_val,
input_token_logprobs_idx,
output_token_logprobs_val,
output_token_logprobs_idx,
input_top_logprobs_val,
input_top_logprobs_idx,
output_top_logprobs_val,
output_top_logprobs_idx,
input_token_ids_logprobs_val,
input_token_ids_logprobs_idx,
output_token_ids_logprobs_val,
output_token_ids_logprobs_idx,
spec_verify_ct=spec_verify_ct,
spec_accepted_tokens=spec_accepted_tokens,
queue_time=queue_times,
forward_entry_time=forward_entry_times,
prefill_delay=prefill_delays,
prefill_latency=prefill_latencies,
finished_reasons=finished_reasons,
decoded_texts=decoded_texts,
decode_ids=decode_ids_list,
read_offsets=read_offsets,
output_ids=output_ids,
skip_special_tokens=skip_special_tokens,
spaces_between_special_tokens=spaces_between_special_tokens,
no_stop_trim=no_stop_trim,
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
cached_tokens=cached_tokens,
input_token_logprobs_val=input_token_logprobs_val,
input_token_logprobs_idx=input_token_logprobs_idx,
output_token_logprobs_val=output_token_logprobs_val,
output_token_logprobs_idx=output_token_logprobs_idx,
input_top_logprobs_val=input_top_logprobs_val,
input_top_logprobs_idx=input_top_logprobs_idx,
output_top_logprobs_val=output_top_logprobs_val,
output_top_logprobs_idx=output_top_logprobs_idx,
input_token_ids_logprobs_val=input_token_ids_logprobs_val,
input_token_ids_logprobs_idx=input_token_ids_logprobs_idx,
output_token_ids_logprobs_val=output_token_ids_logprobs_val,
output_token_ids_logprobs_idx=output_token_ids_logprobs_idx,
output_token_entropy_val=None,
output_hidden_states=output_hidden_states,
rids=rids,
@@ -994,6 +1026,10 @@ class SchedulerOutputProcessorMixin:
embeddings = []
prompt_tokens = []
cached_tokens = []
queue_times = []
forward_entry_times = []
prefill_delays = []
prefill_latencies = []
retraction_counts = []
for req in reqs:
if req.finished():
@@ -1003,17 +1039,43 @@ class SchedulerOutputProcessorMixin:
embeddings.append(req.embedding)
prompt_tokens.append(len(req.origin_input_ids))
cached_tokens.append(req.cached_tokens)
queue_times.append(req.time_stats.get_queueing_time())
forward_entry_times.append(req.time_stats.forward_entry_time)
if req.time_stats.prefill_start_time > 0.0:
prefill_delays.append(
req.time_stats.prefill_start_time
- req.time_stats.forward_entry_time
)
else:
prefill_delays.append(None)
if (
req.time_stats.prefill_start_time > 0.0
and req.time_stats.prefill_end_time > 0.0
):
prefill_latencies.append(
req.time_stats.prefill_end_time
- req.time_stats.prefill_start_time
)
else:
prefill_latencies.append(None)
retraction_counts.append(req.retraction_count)
self.send_to_detokenizer.send_output(
BatchEmbeddingOutput(
finished_reasons,
embeddings,
prompt_tokens,
cached_tokens,
rids=rids,
queue_time=queue_times,
forward_entry_time=forward_entry_times,
prefill_delay=prefill_delays,
prefill_latency=prefill_latencies,
finished_reasons=finished_reasons,
embeddings=embeddings,
prompt_tokens=prompt_tokens,
cached_tokens=cached_tokens,
http_worker_ipcs=http_worker_ipcs,
placeholder_tokens_idx=None,
placeholder_tokens_val=None,
retraction_counts=retraction_counts,
rids=rids,
)
)