Add metrics for speculative decoding (acceptance rate, average acceptance length) (#11144)

This commit is contained in:
Scott Lee
2025-10-10 00:46:44 -07:00
committed by GitHub
parent f19613e6c3
commit 0babd48736
9 changed files with 74 additions and 1 deletions
@@ -634,6 +634,7 @@ class SchedulerOutputProcessorMixin:
completion_tokens = []
cached_tokens = []
spec_verify_ct = []
spec_accepted_tokens = []
output_hidden_states = None
if return_logprob:
@@ -725,6 +726,7 @@ class SchedulerOutputProcessorMixin:
if not self.spec_algorithm.is_none():
spec_verify_ct.append(req.spec_verify_ct)
spec_accepted_tokens.append(req.spec_accepted_tokens)
if return_logprob:
if (
@@ -825,6 +827,7 @@ class SchedulerOutputProcessorMixin:
completion_tokens,
cached_tokens,
spec_verify_ct,
spec_accepted_tokens,
input_token_logprobs_val,
input_token_logprobs_idx,
output_token_logprobs_val,