feat: Naive support Spec V2 + Constrained Decoding (#13425)
Signed-off-by: Ubospica <ubospica@gmail.com> Co-authored-by: Liangsheng Yin <lsyincs@gmail.com>
This commit is contained in:
co-authored by
Liangsheng Yin
parent
25758647b1
commit
6350042696
@@ -395,10 +395,16 @@ class SchedulerOutputProcessorMixin:
|
||||
logits_output.hidden_states[i].cpu().clone().tolist()
|
||||
)
|
||||
|
||||
if req.grammar is not None and batch.spec_algorithm.is_none():
|
||||
if req.grammar is not None:
|
||||
# FIXME: this try-except block is for handling unexpected xgrammar issue.
|
||||
try:
|
||||
req.grammar.accept_token(next_token_id)
|
||||
if batch.spec_algorithm.is_none():
|
||||
# Normal decode: single token
|
||||
req.grammar.accept_token(next_token_id)
|
||||
elif batch.is_v2_eagle:
|
||||
# Speculative decode: next_token_id is a list of accepted tokens
|
||||
for token_id in next_token_id:
|
||||
req.grammar.accept_token(token_id)
|
||||
except ValueError as e:
|
||||
# Grammar accept_token can raise ValueError if the token is not in the grammar.
|
||||
# This can happen if the grammar is not set correctly or the token is invalid.
|
||||
|
||||
Reference in New Issue
Block a user