Add metrics for speculative decoding (acceptance rate, average acceptance length) (#11144)
This commit is contained in:
@@ -634,6 +634,7 @@ class SchedulerOutputProcessorMixin:
|
||||
completion_tokens = []
|
||||
cached_tokens = []
|
||||
spec_verify_ct = []
|
||||
spec_accepted_tokens = []
|
||||
output_hidden_states = None
|
||||
|
||||
if return_logprob:
|
||||
@@ -725,6 +726,7 @@ class SchedulerOutputProcessorMixin:
|
||||
|
||||
if not self.spec_algorithm.is_none():
|
||||
spec_verify_ct.append(req.spec_verify_ct)
|
||||
spec_accepted_tokens.append(req.spec_accepted_tokens)
|
||||
|
||||
if return_logprob:
|
||||
if (
|
||||
@@ -825,6 +827,7 @@ class SchedulerOutputProcessorMixin:
|
||||
completion_tokens,
|
||||
cached_tokens,
|
||||
spec_verify_ct,
|
||||
spec_accepted_tokens,
|
||||
input_token_logprobs_val,
|
||||
input_token_logprobs_idx,
|
||||
output_token_logprobs_val,
|
||||
|
||||
Reference in New Issue
Block a user