[Fix] Slice input_embeds to extend_input_len in prepare_for_extend (#20376)
This commit is contained in:
committed by
GitHub
parent
7458407437
commit
7c773ddb0a
@@ -1528,8 +1528,11 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
|
||||
# If input_embeds are available, store them
|
||||
if req.input_embeds is not None:
|
||||
# If req.input_embeds is already a list, append its content directly
|
||||
input_embeds.extend(req.input_embeds) # Use extend to avoid nesting
|
||||
# Slice to match extend_input_len — PrefillAdder truncates
|
||||
# fill_ids/extend_input_len on chunk overflow but not input_embeds.
|
||||
input_embeds.extend(
|
||||
req.input_embeds[pre_len : pre_len + req.extend_input_len]
|
||||
)
|
||||
|
||||
multimodal_inputs.append(req.multimodal_inputs)
|
||||
|
||||
@@ -1973,6 +1976,9 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
def prepare_for_decode(self):
|
||||
self.forward_mode = ForwardMode.DECODE
|
||||
bs = len(self.reqs)
|
||||
# Decode embeds the last output token via embed_tokens; clear the stale
|
||||
# prefill-time tensor so it doesn't leak into ForwardBatch.
|
||||
self.input_embeds = None
|
||||
|
||||
if self.is_spec_v2:
|
||||
# TODO(spec-v2): all spec v2 should go through this path
|
||||
|
||||
@@ -2336,6 +2336,8 @@ class Scheduler(
|
||||
self.is_mixed_chunk
|
||||
and not self.running_batch.is_empty()
|
||||
and not (new_batch.return_logprob or self.running_batch.return_logprob)
|
||||
# mix_with_running cats input_ids but not input_embeds — shapes would mismatch
|
||||
and new_batch.input_embeds is None
|
||||
):
|
||||
# TODO (lianmin): support return_logprob + mixed chunked prefill
|
||||
self.running_batch.filter_batch(v1_spec_info_filtered=True)
|
||||
|
||||
Reference in New Issue
Block a user