[spec-overlap] bugfix for pd disaggregation and npu (#14088)

Co-authored-by: Even Zhou <even.y.zhou@outlook.com>
This commit is contained in:
liupeng374
2025-12-01 22:58:20 +08:00
committed by GitHub
co-authored by Even Zhou
parent e55731b6e8
commit 2e8f54e61e
9 changed files with 32 additions and 38 deletions
@@ -386,8 +386,8 @@ class EagleDraftWorker(BaseDraftWorker):
spec_info.hidden_states = hidden_states
# Run forward
logits_output = self.draft_runner.model.forward(
forward_batch.input_ids, forward_batch.positions, forward_batch
logits_output, _ = self.draft_runner.forward(
forward_batch, skip_attn_backend_init=True
)
if self.server_args.enable_nan_detection:
detect_nan(logits_output)
+9 -20
View File
@@ -117,26 +117,15 @@ def assign_req_to_token_pool_func(
out_cache_loc: torch.Tensor,
batch_size: int,
):
if _is_cuda or _is_hip:
assign_req_to_token_pool[(batch_size,)](
req_pool_indices,
req_to_token,
start_offset,
end_offset,
out_cache_loc,
req_to_token.shape[1],
next_power_of_2(batch_size),
)
elif _is_npu:
import sgl_kernel_npu # noqa: F401
torch.ops.npu.cache_loc_assign(
req_pool_indices,
req_to_token,
start_offset.to(torch.int64),
end_offset.to(torch.int64),
out_cache_loc,
)
assign_req_to_token_pool[(batch_size,)](
req_pool_indices,
req_to_token,
start_offset,
end_offset,
out_cache_loc,
req_to_token.shape[1],
next_power_of_2(batch_size),
)
@triton.jit