Maintain seq_lens_sum to make more FlashInfer operations non-blocking (#1741)

This commit is contained in:
Lianmin Zheng
2024-10-21 01:43:16 -07:00
committed by GitHub
parent cf470fea32
commit 09603c6dc9
8 changed files with 98 additions and 43 deletions
@@ -127,6 +127,7 @@ class FlashInferAttnBackend(AttentionBackend):
self.indices_updater_decode.update(
forward_batch.req_pool_indices,
forward_batch.seq_lens,
forward_batch.seq_lens_sum,
)
self.forward_metadata = (self.decode_wrappers,)
else:
@@ -134,10 +135,7 @@ class FlashInferAttnBackend(AttentionBackend):
# Some heuristics to check whether to use ragged forward
use_ragged = False
if (
torch.sum(forward_batch.seq_lens).item() >= 4096
and self.num_wrappers == 1
):
if forward_batch.extend_num_tokens >= 4096 and self.num_wrappers == 1:
use_ragged = True
extend_no_prefix = not torch.any(forward_batch.extend_prefix_lens).item()
@@ -181,15 +179,25 @@ class FlashInferAttnBackend(AttentionBackend):
)
)
self.indices_updater_decode.update(req_pool_indices, seq_lens, decode_wrappers)
seq_lens_sum = seq_lens.sum().item()
self.indices_updater_decode.update(
req_pool_indices, seq_lens, seq_lens_sum, decode_wrappers
)
self.cuda_graph_metadata[bs] = decode_wrappers
self.forward_metadata = (decode_wrappers,)
def init_forward_metadata_replay_cuda_graph(
self, bs: int, req_pool_indices: torch.Tensor, seq_lens: torch.Tensor
self,
bs: int,
req_pool_indices: torch.Tensor,
seq_lens: torch.Tensor,
seq_lens_sum: int,
):
self.indices_updater_decode.update(
req_pool_indices[:bs], seq_lens[:bs], self.cuda_graph_metadata[bs]
req_pool_indices[:bs],
seq_lens[:bs],
seq_lens_sum,
self.cuda_graph_metadata[bs],
)
def get_cuda_graph_seq_len_fill_value(self):
@@ -305,13 +313,30 @@ class FlashInferIndicesUpdaterDecode:
assert attn_backend.num_wrappers == 1
self.update = self.update_single_wrapper
def update_single_wrapper(self, req_pool_indices, seq_lens, decode_wrappers=None):
def update_single_wrapper(
self,
req_pool_indices: torch.Tensor,
seq_lens: torch.Tensor,
seq_lens_sum: int,
decode_wrappers=None,
):
decode_wrappers = decode_wrappers or self.decode_wrappers
self.call_begin_forward(
decode_wrappers[0], req_pool_indices, seq_lens, self.kv_indptr[0], None
decode_wrappers[0],
req_pool_indices,
seq_lens,
seq_lens_sum,
self.kv_indptr[0],
None,
)
def update_sliding_window(self, req_pool_indices, seq_lens, decode_wrappers=None):
def update_sliding_window(
self,
req_pool_indices: torch.Tensor,
seq_lens: torch.Tensor,
seq_lens_sum: int,
decode_wrappers=None,
):
decode_wrappers = decode_wrappers or self.decode_wrappers
for wrapper_id in range(2):
@@ -331,6 +356,7 @@ class FlashInferIndicesUpdaterDecode:
decode_wrappers[wrapper_id],
req_pool_indices,
paged_kernel_lens,
seq_lens_sum,
self.kv_indptr[wrapper_id],
kv_start_idx,
)
@@ -339,13 +365,18 @@ class FlashInferIndicesUpdaterDecode:
raise NotImplementedError()
def call_begin_forward(
self, wrapper, req_pool_indices, paged_kernel_lens, kv_indptr, kv_start_idx
self,
wrapper,
req_pool_indices,
paged_kernel_lens,
seq_lens_sum,
kv_indptr,
kv_start_idx,
):
bs = len(req_pool_indices)
kv_indptr = kv_indptr[: bs + 1]
# TODO: optimize the blocking call on kv_indptr[-1]
kv_indptr[1:] = torch.cumsum(paged_kernel_lens, dim=0)
kv_indices = torch.empty(kv_indptr[-1], dtype=torch.int32, device="cuda")
kv_indices = torch.empty(seq_lens_sum, dtype=torch.int32, device="cuda")
create_flashinfer_kv_indices_triton[(bs,)](
self.req_to_token,