[DeepSeek v3.2][Bugfix] get_index_k_scale_buffer support cp (#18280)
This commit is contained in:
@@ -138,6 +138,8 @@ class NSAMetadata:
|
||||
indexer_k_start_end: Optional[Tuple[torch.Tensor, torch.Tensor]] = None
|
||||
# seq lens for each batch.
|
||||
indexer_seq_lens_cpu: Optional[torch.Tensor] = None
|
||||
# seq lens for each batch.
|
||||
indexer_seq_lens: Optional[torch.Tensor] = None
|
||||
# batch index for each token.
|
||||
token_to_batch_idx: Optional[torch.Tensor] = None
|
||||
|
||||
@@ -194,6 +196,9 @@ class NSAIndexerMetadata(BaseIndexerMetadata):
|
||||
def get_indexer_kvcache_range(self) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
return self.attn_metadata.indexer_k_start_end
|
||||
|
||||
def get_indexer_seq_len(self) -> torch.Tensor:
|
||||
return self.attn_metadata.indexer_seq_lens
|
||||
|
||||
def get_indexer_seq_len_cpu(self) -> torch.Tensor:
|
||||
return self.attn_metadata.indexer_seq_lens_cpu
|
||||
|
||||
@@ -404,6 +409,7 @@ class NativeSparseAttnBackend(
|
||||
bs_idx_cpu = None
|
||||
# seq_len_cpu of selected sequences
|
||||
indexer_seq_lens_cpu = forward_batch.seq_lens_cpu
|
||||
indexer_seq_lens = forward_batch.seq_lens
|
||||
|
||||
if forward_batch.forward_mode.is_decode_or_idle():
|
||||
extend_seq_lens_cpu = [1] * batch_size
|
||||
@@ -504,6 +510,7 @@ class NativeSparseAttnBackend(
|
||||
)
|
||||
)
|
||||
indexer_seq_lens_cpu = indexer_seq_lens_cpu[bs_idx_cpu]
|
||||
indexer_seq_lens = indexer_seq_lens[bs_idx]
|
||||
cache_seqlens_int32 = cache_seqlens_int32[bs_idx]
|
||||
cu_seqlens_k = compute_cu_seqlens(cache_seqlens_int32)
|
||||
max_seqlen_k = (
|
||||
@@ -641,6 +648,7 @@ class NativeSparseAttnBackend(
|
||||
topk_indices_offset=topk_indices_offset,
|
||||
indexer_k_start_end=indexer_k_start_end,
|
||||
indexer_seq_lens_cpu=indexer_seq_lens_cpu,
|
||||
indexer_seq_lens=indexer_seq_lens,
|
||||
token_to_batch_idx=token_to_batch_idx,
|
||||
)
|
||||
self.forward_metadata = metadata
|
||||
|
||||
Reference in New Issue
Block a user