[Qwen3.5] Enable MTP spec_v2 and add test for nvidia/Qwen3.5-397B-A17B-NVFP4 (#19391)

This commit is contained in:
hlu1
2026-03-04 14:01:25 -08:00
committed by GitHub
parent 0ee9d3c8e9
commit 9457c049e1
8 changed files with 252 additions and 16 deletions
@@ -465,6 +465,7 @@ class EagleDraftWorker(BaseDraftWorker):
batch: ModelWorkerBatch,
target_hidden_states: torch.Tensor,
next_token_ids: torch.Tensor,
mm_input_embeds: Optional[torch.Tensor] = None,
):
"""
Run draft model extend to correctly fill the KV cache.
@@ -498,6 +499,8 @@ class EagleDraftWorker(BaseDraftWorker):
# Run forward
forward_batch = ForwardBatch.init_new(batch, self.draft_runner)
if mm_input_embeds is not None:
forward_batch.mm_input_embeds = mm_input_embeds
logits_output = self.draft_runner.forward(forward_batch).logits_output
# Update spec_info for the next draft step
@@ -668,6 +671,7 @@ class EAGLEWorkerV2(BaseSpecWorker):
model_worker_batch,
batch_output.logits_output.hidden_states,
batch_output.next_token_ids,
batch_output.logits_output.mm_input_embeds,
)
)
return batch_output