[Fix] Slice input_embeds to extend_input_len in prepare_for_extend (#20376)

This commit is contained in:
Kit Fraser-Taliente
2026-03-15 00:07:05 -07:00
committed by GitHub
parent 7458407437
commit 7c773ddb0a
3 changed files with 198 additions and 2 deletions

View File

@@ -1528,8 +1528,11 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
# If input_embeds are available, store them
if req.input_embeds is not None:
# If req.input_embeds is already a list, append its content directly
input_embeds.extend(req.input_embeds) # Use extend to avoid nesting
# Slice to match extend_input_len — PrefillAdder truncates
# fill_ids/extend_input_len on chunk overflow but not input_embeds.
input_embeds.extend(
req.input_embeds[pre_len : pre_len + req.extend_input_len]
)
multimodal_inputs.append(req.multimodal_inputs)
@@ -1973,6 +1976,9 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
def prepare_for_decode(self):
self.forward_mode = ForwardMode.DECODE
bs = len(self.reqs)
# Decode embeds the last output token via embed_tokens; clear the stale
# prefill-time tensor so it doesn't leak into ForwardBatch.
self.input_embeds = None
if self.is_spec_v2:
# TODO(spec-v2): all spec v2 should go through this path

View File

@@ -2336,6 +2336,8 @@ class Scheduler(
self.is_mixed_chunk
and not self.running_batch.is_empty()
and not (new_batch.return_logprob or self.running_batch.return_logprob)
# mix_with_running cats input_ids but not input_embeds — shapes would mismatch
and new_batch.input_embeds is None
):
# TODO (lianmin): support return_logprob + mixed chunked prefill
self.running_batch.filter_batch(v1_spec_info_filtered=True)