Maintain seq_lens_sum to make more FlashInfer operations non-blocking (#1741)
This commit is contained in:
@@ -127,6 +127,7 @@ class FlashInferAttnBackend(AttentionBackend):
|
||||
self.indices_updater_decode.update(
|
||||
forward_batch.req_pool_indices,
|
||||
forward_batch.seq_lens,
|
||||
forward_batch.seq_lens_sum,
|
||||
)
|
||||
self.forward_metadata = (self.decode_wrappers,)
|
||||
else:
|
||||
@@ -134,10 +135,7 @@ class FlashInferAttnBackend(AttentionBackend):
|
||||
|
||||
# Some heuristics to check whether to use ragged forward
|
||||
use_ragged = False
|
||||
if (
|
||||
torch.sum(forward_batch.seq_lens).item() >= 4096
|
||||
and self.num_wrappers == 1
|
||||
):
|
||||
if forward_batch.extend_num_tokens >= 4096 and self.num_wrappers == 1:
|
||||
use_ragged = True
|
||||
|
||||
extend_no_prefix = not torch.any(forward_batch.extend_prefix_lens).item()
|
||||
@@ -181,15 +179,25 @@ class FlashInferAttnBackend(AttentionBackend):
|
||||
)
|
||||
)
|
||||
|
||||
self.indices_updater_decode.update(req_pool_indices, seq_lens, decode_wrappers)
|
||||
seq_lens_sum = seq_lens.sum().item()
|
||||
self.indices_updater_decode.update(
|
||||
req_pool_indices, seq_lens, seq_lens_sum, decode_wrappers
|
||||
)
|
||||
self.cuda_graph_metadata[bs] = decode_wrappers
|
||||
self.forward_metadata = (decode_wrappers,)
|
||||
|
||||
def init_forward_metadata_replay_cuda_graph(
|
||||
self, bs: int, req_pool_indices: torch.Tensor, seq_lens: torch.Tensor
|
||||
self,
|
||||
bs: int,
|
||||
req_pool_indices: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
seq_lens_sum: int,
|
||||
):
|
||||
self.indices_updater_decode.update(
|
||||
req_pool_indices[:bs], seq_lens[:bs], self.cuda_graph_metadata[bs]
|
||||
req_pool_indices[:bs],
|
||||
seq_lens[:bs],
|
||||
seq_lens_sum,
|
||||
self.cuda_graph_metadata[bs],
|
||||
)
|
||||
|
||||
def get_cuda_graph_seq_len_fill_value(self):
|
||||
@@ -305,13 +313,30 @@ class FlashInferIndicesUpdaterDecode:
|
||||
assert attn_backend.num_wrappers == 1
|
||||
self.update = self.update_single_wrapper
|
||||
|
||||
def update_single_wrapper(self, req_pool_indices, seq_lens, decode_wrappers=None):
|
||||
def update_single_wrapper(
|
||||
self,
|
||||
req_pool_indices: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
seq_lens_sum: int,
|
||||
decode_wrappers=None,
|
||||
):
|
||||
decode_wrappers = decode_wrappers or self.decode_wrappers
|
||||
self.call_begin_forward(
|
||||
decode_wrappers[0], req_pool_indices, seq_lens, self.kv_indptr[0], None
|
||||
decode_wrappers[0],
|
||||
req_pool_indices,
|
||||
seq_lens,
|
||||
seq_lens_sum,
|
||||
self.kv_indptr[0],
|
||||
None,
|
||||
)
|
||||
|
||||
def update_sliding_window(self, req_pool_indices, seq_lens, decode_wrappers=None):
|
||||
def update_sliding_window(
|
||||
self,
|
||||
req_pool_indices: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
seq_lens_sum: int,
|
||||
decode_wrappers=None,
|
||||
):
|
||||
decode_wrappers = decode_wrappers or self.decode_wrappers
|
||||
|
||||
for wrapper_id in range(2):
|
||||
@@ -331,6 +356,7 @@ class FlashInferIndicesUpdaterDecode:
|
||||
decode_wrappers[wrapper_id],
|
||||
req_pool_indices,
|
||||
paged_kernel_lens,
|
||||
seq_lens_sum,
|
||||
self.kv_indptr[wrapper_id],
|
||||
kv_start_idx,
|
||||
)
|
||||
@@ -339,13 +365,18 @@ class FlashInferIndicesUpdaterDecode:
|
||||
raise NotImplementedError()
|
||||
|
||||
def call_begin_forward(
|
||||
self, wrapper, req_pool_indices, paged_kernel_lens, kv_indptr, kv_start_idx
|
||||
self,
|
||||
wrapper,
|
||||
req_pool_indices,
|
||||
paged_kernel_lens,
|
||||
seq_lens_sum,
|
||||
kv_indptr,
|
||||
kv_start_idx,
|
||||
):
|
||||
bs = len(req_pool_indices)
|
||||
kv_indptr = kv_indptr[: bs + 1]
|
||||
# TODO: optimize the blocking call on kv_indptr[-1]
|
||||
kv_indptr[1:] = torch.cumsum(paged_kernel_lens, dim=0)
|
||||
kv_indices = torch.empty(kv_indptr[-1], dtype=torch.int32, device="cuda")
|
||||
kv_indices = torch.empty(seq_lens_sum, dtype=torch.int32, device="cuda")
|
||||
|
||||
create_flashinfer_kv_indices_triton[(bs,)](
|
||||
self.req_to_token,
|
||||
|
||||
Reference in New Issue
Block a user