Add metrics for speculative decoding (acceptance rate, average acceptance length) (#11441)
This commit is contained in:
@@ -711,6 +711,7 @@ class SchedulerOutputProcessorMixin:
|
||||
completion_tokens = []
|
||||
cached_tokens = []
|
||||
spec_verify_ct = []
|
||||
spec_accepted_tokens = []
|
||||
output_hidden_states = None
|
||||
|
||||
if return_logprob:
|
||||
@@ -808,6 +809,7 @@ class SchedulerOutputProcessorMixin:
|
||||
|
||||
if not self.spec_algorithm.is_none():
|
||||
spec_verify_ct.append(req.spec_verify_ct)
|
||||
spec_accepted_tokens.append(req.spec_accepted_tokens)
|
||||
|
||||
if return_logprob:
|
||||
if (
|
||||
@@ -908,6 +910,7 @@ class SchedulerOutputProcessorMixin:
|
||||
completion_tokens,
|
||||
cached_tokens,
|
||||
spec_verify_ct,
|
||||
spec_accepted_tokens,
|
||||
input_token_logprobs_val,
|
||||
input_token_logprobs_idx,
|
||||
output_token_logprobs_val,
|
||||
|
||||
Reference in New Issue
Block a user