Fix the output of hidden states after HTTP requests (#4269)

This commit is contained in:
Qiaolin Yu
2025-03-13 14:54:06 -07:00
committed by GitHub
parent 5fe79605a8
commit 85d2365d33
5 changed files with 47 additions and 9 deletions
@@ -111,6 +111,7 @@ class SchedulerOutputProcessorMixin:
]
.cpu()
.clone()
.tolist()
)
if req.grammar is not None:
@@ -245,7 +246,9 @@ class SchedulerOutputProcessorMixin:
)
if req.return_hidden_states and logits_output.hidden_states is not None:
req.hidden_states.append(logits_output.hidden_states[i].cpu().clone())
req.hidden_states.append(
logits_output.hidden_states[i].cpu().clone().tolist()
)
if req.grammar is not None and batch.spec_algorithm.is_none():
req.grammar.accept_token(next_token_id)