[spec-overlap] bugfix for pd disaggregation and npu (#14088)
Co-authored-by: Even Zhou <even.y.zhou@outlook.com>
This commit is contained in:
@@ -386,8 +386,8 @@ class EagleDraftWorker(BaseDraftWorker):
|
||||
spec_info.hidden_states = hidden_states
|
||||
|
||||
# Run forward
|
||||
logits_output = self.draft_runner.model.forward(
|
||||
forward_batch.input_ids, forward_batch.positions, forward_batch
|
||||
logits_output, _ = self.draft_runner.forward(
|
||||
forward_batch, skip_attn_backend_init=True
|
||||
)
|
||||
if self.server_args.enable_nan_detection:
|
||||
detect_nan(logits_output)
|
||||
|
||||
@@ -117,26 +117,15 @@ def assign_req_to_token_pool_func(
|
||||
out_cache_loc: torch.Tensor,
|
||||
batch_size: int,
|
||||
):
|
||||
if _is_cuda or _is_hip:
|
||||
assign_req_to_token_pool[(batch_size,)](
|
||||
req_pool_indices,
|
||||
req_to_token,
|
||||
start_offset,
|
||||
end_offset,
|
||||
out_cache_loc,
|
||||
req_to_token.shape[1],
|
||||
next_power_of_2(batch_size),
|
||||
)
|
||||
elif _is_npu:
|
||||
import sgl_kernel_npu # noqa: F401
|
||||
|
||||
torch.ops.npu.cache_loc_assign(
|
||||
req_pool_indices,
|
||||
req_to_token,
|
||||
start_offset.to(torch.int64),
|
||||
end_offset.to(torch.int64),
|
||||
out_cache_loc,
|
||||
)
|
||||
assign_req_to_token_pool[(batch_size,)](
|
||||
req_pool_indices,
|
||||
req_to_token,
|
||||
start_offset,
|
||||
end_offset,
|
||||
out_cache_loc,
|
||||
req_to_token.shape[1],
|
||||
next_power_of_2(batch_size),
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
|
||||
Reference in New Issue
Block a user