Add metrics for speculative decoding (acceptance rate, average acceptance length) (#11144)

2025-10-10 00:46:44 -07:00
parent f19613e6c3
commit 0babd48736
9 changed files with 74 additions and 1 deletions
@@ -378,6 +378,13 @@ class EagleVerifyInput(SpecInput):
                    unfinished_accept_index.append(accept_index[i])
            req.spec_verify_ct += 1

+        # For each request, accumulate # of accepted tokens for this verify pass.
+        accept_length_this_pass = (accept_index != -1).sum(dim=1) - 1
+        for i, (req, accepted_count) in enumerate(
+            zip(batch.reqs, accept_length_this_pass.tolist())
+        ):
+            req.spec_accepted_tokens += accepted_count
+
        if has_finished:
            accept_length = (accept_index != -1).sum(dim=1) - 1