Fix the output of hidden states after HTTP requests (#4269)
This commit is contained in:
@@ -111,6 +111,7 @@ class SchedulerOutputProcessorMixin:
|
||||
]
|
||||
.cpu()
|
||||
.clone()
|
||||
.tolist()
|
||||
)
|
||||
|
||||
if req.grammar is not None:
|
||||
@@ -245,7 +246,9 @@ class SchedulerOutputProcessorMixin:
|
||||
)
|
||||
|
||||
if req.return_hidden_states and logits_output.hidden_states is not None:
|
||||
req.hidden_states.append(logits_output.hidden_states[i].cpu().clone())
|
||||
req.hidden_states.append(
|
||||
logits_output.hidden_states[i].cpu().clone().tolist()
|
||||
)
|
||||
|
||||
if req.grammar is not None and batch.spec_algorithm.is_none():
|
||||
req.grammar.accept_token(next_token_id)
|
||||
|
||||
Reference in New Issue
Block a user