Add metrics for speculative decoding (acceptance rate, average acceptance length) (#11441)

2025-10-13 11:24:27 -07:00
parent f4aa78801e
commit b6fb5d7666
9 changed files with 70 additions and 1 deletions
@@ -384,6 +384,9 @@ class EagleVerifyInput(SpecInput, EagleVerifyInputV2Mixin):
                else:
                    unfinished_accept_index.append(accept_index[i])
            req.spec_verify_ct += 1
+            req.spec_accepted_tokens += (
+                sum(1 for idx in accept_index_row if idx != -1) - 1
+            )

        if has_finished:
            accept_length = (accept_index != -1).sum(dim=1) - 1