Enable index partial-current reuse without replaying prefix materialize

The index path now mirrors the target MLA partial-current contract: prefetched or synchronously materialized prefix pages are composed with valid current index K/scale rows in slot-dense page buffers. Current-only batches keep the compact current-index fast path, while partial cache-hit batches share one composed dense index buffer across the in-seq prev/next topk pair.\n\nThe prefetch consume path remaps through the slot page inverse instead of treating the slot-dense buffer as physical-pool capacity, and current-index quantization uses valid extend rows so padded out_cache_loc does not disable reuse.\n\nConstraint: CP shared KV remains page-slot based; padding rows must stay invisible to attention/index semantics\nConstraint: Draft/EAGLE partial-current reuse remains guarded by should_reuse_current_extend_kv\nRejected: Replace prefix all-reduce with all-gather | NCCL all-gather still uses SM and would require an additional compose/scatter step\nConfidence: medium\nScope-risk: moderate\nDirective: Do not reintroduce current-only gating for index reuse; partial target cache hits must compose prefix + valid current rows\nTested: Local py_compile for touched Python files\nTested: g0034 sglang-glm5-dev-2 PYTHONPATH=python python -m pytest -q test/registered/unit/mem_cache/test_cp_shared_kv_runtime.py -> 77 passed, 5 warnings, 2 subtests passed\nNot-tested: Full ETE traffic with latest commit; CUDA perf impact of index partial-current prefetch under production load
This commit is contained in:
laoyao0822
2026-05-31 02:31:09 +08:00
parent 251a48fb0a
commit 3c14b1f127
5 changed files with 604 additions and 14 deletions

View File

@@ -91,3 +91,36 @@ PYTHONPATH=python python3 -m pytest \
test/registered/unit/layers/test_nsa_cp_utils.py \
test/registered/unit/mem_cache/test_cp_shared_kv_layout.py -q
```
## 2026-05-31 update: index partial/current reuse
The index path now follows the same target-model partial-current contract as MLA:
```text
page-aligned cached prefix index pages
+ valid current index K/scale rows copied into their slot-dense suffix pages
+ page-tail slack left zero and invisible through valid seq/page-table lengths
```
Implementation notes:
- `Indexer._can_reuse_current_index_kv()` uses `should_reuse_current_extend_kv()` instead of the old current-only gate, so target cache-hit suffixes can reuse freshly computed current index K/scale.
- Current-only batches still use the compact `current_index_kv` path and avoid page-table materialization.
- Partial cache-hit batches materialize the prefix once for the prev/next in-seq split pair, then pass the composed dense index buffer to both topk calls.
- If the async index prefetcher has a ready prefix handle, `CpSharedKVIndexPrefetcher.consume_prefix_with_current()` consumes it and only fills current rows into the suffix page slots.
- If no prefetch handle exists, `materialize_prefix_and_reuse_current_index_page_slots()` synchronously materializes only prefix slots, all-reduces that prefix range, and fills current rows locally.
- `forward_cuda()` quantizes only `valid_current_rows = extend_seq_lens_cpu[0]` current rows. This preserves current reuse when `out_cache_loc` is physically padded for page alignment.
Debug contract correction:
- The prefetch consume path must remap `logical_pages` through the slot `page_inverse` directly. It must not validate these logical pages against `dense_page_buffer.shape[0]` as if the dense buffer were the physical pool capacity; that debug-only check can reject valid high logical page ids that are intentionally packed into a small slot-dense buffer.
Verification:
```bash
# g0034 / sglang-glm5-dev-2
cd /sgl-workspace/sglang-tai
PYTHONPATH=python python -m pytest -q \
test/registered/unit/mem_cache/test_cp_shared_kv_runtime.py
# 77 passed, 5 warnings, 2 subtests passed
```

View File

@@ -19,6 +19,7 @@ from sglang.srt.layers.attention.nsa.cp_shared_kv_runtime import (
cp_shared_kv_mla_prefetch_should_log_layer,
filter_locs_mappable_to_physical_pool,
filter_pages_mappable_to_physical_pool,
fill_current_index_page_slots,
fill_current_kv_page_slots_and_remap_locs,
get_or_build_shared_paged_buffer_slot_remap,
get_or_build_shared_token_kv_slot_remap,
@@ -1469,6 +1470,104 @@ class CpSharedKVIndexPrefetcher:
)
return dense_page_buffer, dense_pages
def consume_prefix_with_current(
self,
*,
layer_id: int,
logical_pages: torch.Tensor,
current_index_k: torch.Tensor,
current_index_scale: torch.Tensor,
current_locs: torch.Tensor,
page_size: int,
index_head_dim: int,
) -> Optional[tuple[torch.Tensor, torch.Tensor]]:
if self.disabled:
self._log_layer(
layer_id,
"index_consume_prefix_current_skip reason=disabled layer=%s",
layer_id,
)
return None
self._log_layer(
layer_id,
"index_consume_prefix_current_enter layer=%s prefix_pages=%s "
"total_slots=%s handles=%s",
layer_id,
self.prefix_pages,
self.total_slots,
_debug_handle_keys(layer_id, self.handles),
)
handle = self.handles.get(layer_id)
if handle is None:
self._log_layer(
layer_id,
"index_consume_prefix_current_miss layer=%s",
layer_id,
)
return None
if handle.event is None:
self.launch_pending_reduce()
handle = self.handles.get(layer_id)
if handle is None or handle.event is None:
self._log_layer(
layer_id,
"index_consume_prefix_current_miss "
"reason=prefix_reduce_not_ready layer=%s",
layer_id,
)
return None
handle = self.handles.pop(layer_id)
if self.pending_attention_handle is handle:
self.pending_attention_handle = None
if handle.layer_id != layer_id:
self.disabled = True
self._log_layer(
layer_id,
"index_consume_prefix_current_skip reason=layer_mismatch "
"expected=%s actual=%s",
layer_id,
handle.layer_id,
)
return None
consume_cpu = _cpu_timing_start()
wait_cpu = _cpu_timing_start()
torch.cuda.current_stream().wait_event(handle.event)
wait_ms = _cpu_timing_ms(wait_cpu)
dense_page_buffer = handle.dense_page_buffer
remap_cpu = _cpu_timing_start()
dense_pages = remap_logical_pages_to_slot_dense_pages(
logical_pages,
page_inverse=self.page_inverse,
)
dense_page_buffer = fill_current_index_page_slots(
dense_page_buffer=dense_page_buffer,
current_index_k=current_index_k,
current_index_scale=current_index_scale,
current_locs=current_locs,
page_inverse=self.page_inverse,
page_size=page_size,
index_head_dim=index_head_dim,
)
remap_ms = _cpu_timing_ms(remap_cpu)
total_ms = _cpu_timing_ms(consume_cpu)
self._log_layer(
layer_id,
"index_consume_prefix_current_hit layer=%s prefix_pages=%s "
"dense_pages=%s current_rows=%s total_ms=%.3f wait_ms=%.3f "
"remap_ms=%.3f",
layer_id,
self.prefix_pages,
int(dense_page_buffer.shape[0]),
int(current_index_k.shape[0]),
total_ms,
wait_ms,
remap_ms,
)
return dense_page_buffer, dense_pages
def start_next_layer_prefix(
self,
*,

View File

@@ -813,6 +813,103 @@ def fill_current_kv_page_slots_and_remap_locs(
return dense_kv_cache, mixed_locs, current_mask
def fill_current_index_page_slots(
*,
dense_page_buffer: torch.Tensor,
current_index_k: torch.Tensor,
current_index_scale: torch.Tensor,
current_locs: torch.Tensor,
page_inverse: torch.Tensor,
page_size: int,
index_head_dim: int,
) -> torch.Tensor:
"""Fill current index K/scale rows into a slot-dense page buffer.
The dense page buffer already has one row per logical page slot and page 0 is
the dummy page. Current rows are copied into the row/offset selected by their
logical token loc; tail-page slack remains zero and is invisible because the
index kernels still receive the valid sequence length separately.
"""
current_locs = current_locs.reshape(-1)
current_rows = int(current_locs.numel())
if current_rows == 0:
return dense_page_buffer
if (
int(current_index_k.shape[0]) < current_rows
or int(current_index_scale.shape[0]) < current_rows
):
raise ValueError(
"CP shared KV index current-slot fill got fewer current rows than "
f"locs: k_rows={int(current_index_k.shape[0])} "
f"scale_rows={int(current_index_scale.shape[0])} locs={current_rows}"
)
current_index_k = current_index_k[:current_rows].contiguous()
current_index_scale = current_index_scale[:current_rows].contiguous()
k_bytes = current_index_k.reshape(current_rows, -1).view(torch.uint8)
if int(k_bytes.shape[1]) != int(index_head_dim):
raise ValueError(
"CP shared KV index current-slot fill got unexpected K width: "
f"k_bytes_per_token={int(k_bytes.shape[1])} index_head_dim={index_head_dim}"
)
scale_bytes = current_index_scale.reshape(current_rows, -1).view(torch.uint8)
scale_bytes = scale_bytes.reshape(current_rows, -1)
page_stride = int(dense_page_buffer.shape[1])
scale_offset = int(page_size) * int(index_head_dim)
if scale_offset + int(page_size) * int(scale_bytes.shape[1]) > page_stride:
raise ValueError(
"CP shared KV index current-slot fill got incompatible page buffer: "
f"page_stride={page_stride} scale_offset={scale_offset} "
f"scale_bytes_per_token={int(scale_bytes.shape[1])} page_size={page_size}"
)
current_pages = torch.div(current_locs, page_size, rounding_mode="floor")
valid_pages = (current_locs >= 0) & (current_pages >= 0) & (
current_pages < int(page_inverse.numel())
)
safe_pages = torch.clamp(
current_pages,
min=0,
max=max(int(page_inverse.numel()) - 1, 0),
)
dense_pages = page_inverse[safe_pages.to(torch.long)].to(torch.long)
valid_rows = valid_pages & (dense_pages > 0)
if not torch.any(valid_rows):
return dense_page_buffer
valid_dense_pages = dense_pages[valid_rows]
valid_offsets = torch.remainder(current_locs[valid_rows], page_size).to(torch.long)
flat_buffer = dense_page_buffer.reshape(-1)
k_cols = torch.arange(
int(index_head_dim),
dtype=torch.long,
device=dense_page_buffer.device,
)
k_indices = (
valid_dense_pages[:, None] * page_stride
+ valid_offsets[:, None] * int(index_head_dim)
+ k_cols[None, :]
)
flat_buffer[k_indices.reshape(-1)] = k_bytes[valid_rows].reshape(-1)
s_cols = torch.arange(
int(scale_bytes.shape[1]),
dtype=torch.long,
device=dense_page_buffer.device,
)
s_indices = (
valid_dense_pages[:, None] * page_stride
+ scale_offset
+ valid_offsets[:, None] * int(scale_bytes.shape[1])
+ s_cols[None, :]
)
flat_buffer[s_indices.reshape(-1)] = scale_bytes[valid_rows].reshape(-1)
return dense_page_buffer
def _copy_tai_dense_slot_range_body(
*,
tai_dense_kv_cache: torch.Tensor,
@@ -2058,6 +2155,62 @@ def materialize_prefix_and_reuse_current_kv_page_slots(
return mixed_kv_cache, mixed_locs
def materialize_prefix_and_reuse_current_index_page_slots(
*,
page_buffer: torch.Tensor,
current_index_k: torch.Tensor,
current_index_scale: torch.Tensor,
current_locs: torch.Tensor,
slot_remap: SharedPagedBufferSlotRemap,
layout: CpSharedKVLayout,
page_size: int,
index_head_dim: int,
prefix_pages: int,
layer_id: int | None = None,
nvtx_source: str = "index.partial_current_sync",
) -> tuple[torch.Tensor, torch.Tensor]:
"""Synchronously compose prefix index materialization with current index rows."""
total_slots = int(slot_remap.slot_logical_pages.numel())
if prefix_pages < 0 or prefix_pages > total_slots:
raise ValueError(
"Invalid CP shared KV index partial-current prefix range: "
f"prefix_pages={prefix_pages} total_slots={total_slots}"
)
dense_page_buffer = page_buffer.new_zeros(
(slot_remap.dense_num_pages, *page_buffer.shape[1:])
)
materialize_local_paged_buffer_page_slots_into(
page_buffer=page_buffer,
dense_page_buffer=dense_page_buffer,
slot_logical_pages=slot_remap.slot_logical_pages,
layout=layout,
start_slot=0,
end_slot=prefix_pages,
)
prefix_rows = slot_range_to_page_slice(0, prefix_pages)
_all_reduce_materialized_buffer_range(
dense_page_buffer,
layout.cp_size,
prefix_rows.start,
prefix_rows.stop,
nvtx_source=nvtx_source,
nvtx_layer_id=layer_id,
nvtx_cp_rank=layout.cp_rank,
)
dense_page_buffer = fill_current_index_page_slots(
dense_page_buffer=dense_page_buffer,
current_index_k=current_index_k,
current_index_scale=current_index_scale,
current_locs=current_locs,
page_inverse=slot_remap.page_inverse,
page_size=page_size,
index_head_dim=index_head_dim,
)
return dense_page_buffer, slot_remap.dense_pages
def slot_range_to_token_slice(
page_size: int,
start_slot: int,

View File

@@ -17,7 +17,6 @@ from sglang.srt.layers.attention.nsa import index_buf_accessor
from sglang.srt.layers.attention.nsa.cp_shared_kv_runtime import (
cp_shared_kv_debug_enabled,
cp_shared_kv_debug_log,
cp_shared_kv_current_reuse_enabled,
cp_shared_kv_mla_prefetch_log,
cp_shared_kv_mla_prefetch_log_enabled,
cp_shared_kv_mla_prefetch_should_log_layer,
@@ -26,7 +25,9 @@ from sglang.srt.layers.attention.nsa.cp_shared_kv_runtime import (
get_or_build_shared_paged_buffer_slot_remap,
is_current_only_extend_batch,
log_cp_draft_shared_kv_debug,
materialize_prefix_and_reuse_current_index_page_slots,
materialize_shared_paged_buffer,
should_reuse_current_extend_kv,
tensor_debug_checksum,
tensor_debug_summary,
try_tai_prepare_cp_mqa_index,
@@ -302,6 +303,7 @@ class Indexer(MultiPlatformOp):
forward_batch: ForwardBatch,
layer_id: int,
logical_page_table: torch.Tensor,
current_index_kv: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
) -> tuple[torch.Tensor, torch.Tensor]:
index_buffer = forward_batch.token_to_kv_pool.get_index_k_with_scale_buffer(
layer_id=layer_id
@@ -314,6 +316,100 @@ class Indexer(MultiPlatformOp):
index_prefetcher = getattr(
forward_batch, "cp_shared_kv_index_prefetcher", None
)
if current_index_kv is not None:
page_size = int(forward_batch.token_to_kv_pool.page_size)
prefix_lens_cpu = getattr(forward_batch, "extend_prefix_lens_cpu", None)
extend_lens_cpu = getattr(forward_batch, "extend_seq_lens_cpu", None)
prefix_lens = (
[int(x) for x in prefix_lens_cpu]
if prefix_lens_cpu is not None
else None
)
extend_lens = (
[int(x) for x in extend_lens_cpu]
if extend_lens_cpu is not None
else None
)
if (
prefix_lens_cpu is None
or len(prefix_lens_cpu) != 1
or int(prefix_lens_cpu[0]) <= 0
or int(prefix_lens_cpu[0]) % page_size != 0
):
raise RuntimeError(
"[CP_SHARED_KV_FAIL_FAST][index_partial_current_sync] "
"CP shared KV index partial-current compose requires one "
"positive page-aligned prefix. "
f"cp_rank={layout.cp_rank} layer_id={layer_id} "
f"prefix_lens={prefix_lens} extend_lens={extend_lens} "
f"logical_page_table_shape={tuple(logical_page_table.shape)} "
f"page_size={page_size}"
)
current_locs = forward_batch.out_cache_loc
if extend_lens_cpu is not None and len(extend_lens_cpu) == 1:
valid_current_rows = int(extend_lens_cpu[0])
if (
valid_current_rows > 0
and valid_current_rows < int(current_locs.numel())
and valid_current_rows <= int(current_index_kv[0].shape[0])
and valid_current_rows <= int(current_index_kv[1].shape[0])
):
current_locs = current_locs[:valid_current_rows]
current_index_kv = (
current_index_kv[0][:valid_current_rows],
current_index_kv[1][:valid_current_rows],
)
prefix_pages = int(prefix_lens_cpu[0]) // page_size
if index_prefetcher is not None:
prefetched = index_prefetcher.consume_prefix_with_current(
layer_id=layer_id,
logical_pages=logical_page_table,
current_index_k=current_index_kv[0],
current_index_scale=current_index_kv[1],
current_locs=current_locs,
page_size=page_size,
index_head_dim=forward_batch.token_to_kv_pool.index_head_dim,
)
if prefetched is not None:
return prefetched
slot_remap = get_or_build_shared_paged_buffer_slot_remap(
forward_batch,
page_buffer=index_buffer,
logical_pages=logical_page_table,
layout=layout,
)
materialized, dense_pages = (
materialize_prefix_and_reuse_current_index_page_slots(
page_buffer=index_buffer,
current_index_k=current_index_kv[0],
current_index_scale=current_index_kv[1],
current_locs=current_locs,
slot_remap=slot_remap,
layout=layout,
page_size=page_size,
index_head_dim=forward_batch.token_to_kv_pool.index_head_dim,
prefix_pages=prefix_pages,
layer_id=layer_id,
)
)
if (
cp_shared_kv_mla_prefetch_log_enabled()
and cp_shared_kv_mla_prefetch_should_log_layer(layer_id)
):
cp_shared_kv_mla_prefetch_log(
"index_partial_current_sync_compose cp_rank=%s layer=%s "
"prefix_lens=%s extend_lens=%s prefix_pages=%s "
"current_rows=%s dense_pages=%s",
layout.cp_rank,
layer_id,
prefix_lens,
extend_lens,
prefix_pages,
int(current_index_kv[0].shape[0]),
int(materialized.shape[0]),
)
return materialized, dense_pages
if index_prefetcher is not None:
prefetched = index_prefetcher.consume(
layer_id=layer_id,
@@ -461,12 +557,11 @@ class Indexer(MultiPlatformOp):
def _can_reuse_current_index_kv(self, forward_batch: ForwardBatch) -> bool:
return (
cp_shared_kv_current_reuse_enabled()
and forward_batch.uses_cp_shared_kv
forward_batch.uses_cp_shared_kv
and self.nsa_enable_prefill_cp
and forward_batch.nsa_cp_metadata is not None
and is_nsa_prefill_cp_in_seq_split()
and is_current_only_extend_batch(forward_batch)
and should_reuse_current_extend_kv(forward_batch)
and forward_batch.hisparse_coordinator is None
and _is_cuda
and not _is_fp8_fnuz
@@ -1283,7 +1378,21 @@ class Indexer(MultiPlatformOp):
shared_index_buffer = None
shared_block_tables = None
if current_index_kv is None:
current_index_kv_for_topk = current_index_kv
if current_index_kv is not None and not is_current_only_extend_batch(
forward_batch
):
current_index_kv_for_topk = None
shared_block_tables = metadata.get_page_table_64()
shared_index_buffer, shared_block_tables = (
self._maybe_materialize_shared_index_buffer(
forward_batch,
layer_id,
shared_block_tables,
current_index_kv=current_index_kv,
)
)
elif current_index_kv is None:
shared_block_tables = metadata.get_page_table_64()
shared_index_buffer, shared_block_tables = (
self._maybe_materialize_shared_index_buffer(
@@ -1301,7 +1410,7 @@ class Indexer(MultiPlatformOp):
metadata,
kv_len_prev,
actual_seq_q_prev,
current_index_kv=current_index_kv,
current_index_kv=current_index_kv_for_topk,
shared_index_buffer=shared_index_buffer,
shared_block_tables=shared_block_tables,
actual_seq_q_tensor=forward_batch.nsa_cp_metadata.actual_seq_q_prev_tensor,
@@ -1316,7 +1425,7 @@ class Indexer(MultiPlatformOp):
metadata,
kv_len_next,
actual_seq_q_next,
current_index_kv=current_index_kv,
current_index_kv=current_index_kv_for_topk,
shared_index_buffer=shared_index_buffer,
shared_block_tables=shared_block_tables,
actual_seq_q_tensor=forward_batch.nsa_cp_metadata.actual_seq_q_next_tensor,
@@ -1674,9 +1783,18 @@ class Indexer(MultiPlatformOp):
current_index_kv = None
if self._can_reuse_current_index_kv(forward_batch):
if key.shape[0] == forward_batch.out_cache_loc.numel():
extend_seq_lens_cpu = getattr(forward_batch, "extend_seq_lens_cpu", None)
valid_current_rows = int(forward_batch.out_cache_loc.numel())
if extend_seq_lens_cpu is not None and len(extend_seq_lens_cpu) == 1:
valid_current_rows = min(
int(extend_seq_lens_cpu[0]),
valid_current_rows,
)
if key.shape[0] >= valid_current_rows:
current_k_fp8, current_k_scale = act_quant(
key.contiguous(), self.block_size, self.scale_fmt
key[:valid_current_rows].contiguous(),
self.block_size,
self.scale_fmt,
)
current_index_kv = (
current_k_fp8.contiguous(),
@@ -1690,14 +1808,25 @@ class Indexer(MultiPlatformOp):
if forward_batch.cp_shared_kv_layout is not None
else None,
layer_id,
tensor_debug_summary(forward_batch.out_cache_loc),
tensor_debug_summary(
forward_batch.out_cache_loc[:valid_current_rows]
),
tensor_debug_checksum(current_index_kv[0]),
tensor_debug_checksum(current_index_kv[1]),
)
elif cp_shared_kv_debug_enabled():
raise RuntimeError(
"CP shared KV current index reuse shape mismatch: "
f"key_tokens={key.shape[0]} out_cache_loc={forward_batch.out_cache_loc.numel()}"
else:
_log_cp_shared_kv_index_prefetch_fallback(
"current_reuse_shape_mismatch",
"NSA index current reuse skipped because key has fewer rows "
"than valid current locs. cp_rank=%s layer=%s key_tokens=%s "
"valid_current_rows=%s out_cache_loc=%s",
forward_batch.cp_shared_kv_layout.cp_rank
if forward_batch.cp_shared_kv_layout is not None
else None,
layer_id,
int(key.shape[0]),
valid_current_rows,
int(forward_batch.out_cache_loc.numel()),
)
if _is_cuda or _is_hip:

View File

@@ -1021,6 +1021,182 @@ class TestCpSharedKVRuntimeHelpers(unittest.TestCase):
self.assertLess(valid_tokens, padded_pages * page_size)
self.assertEqual(valid_tokens, 10)
def test_index_partial_current_sync_compose_fills_current_page_slots(self):
from sglang.srt.layers.attention.nsa import cp_shared_kv_runtime as runtime
from sglang.srt.mem_cache.cp_shared_kv_layout import CpSharedKVLayout
page_size = 4
index_head_dim = 4
scale_bytes = 4
page_bytes = page_size * index_head_dim + page_size * scale_bytes
layout = CpSharedKVLayout(page_size=page_size, cp_size=1, cp_rank=0)
page_buffer = torch.zeros((6, page_bytes), dtype=torch.uint8)
page_buffer[1] = torch.arange(10, 10 + page_bytes, dtype=torch.uint8)
page_buffer[2] = 99
logical_pages = torch.tensor([[1, 2]], dtype=torch.int64)
slot_remap = runtime.build_shared_paged_buffer_slot_remap(
page_buffer,
logical_pages,
layout,
)
current_k = torch.tensor(
[[1, 2, 3, 4], [5, 6, 7, 8]],
dtype=torch.uint8,
)
current_scale = torch.tensor([[1.25], [2.5]], dtype=torch.float32)
with patch.object(
runtime, "_all_reduce_materialized_buffer_range", _identity_all_reduce
):
dense_page_buffer, dense_pages = (
runtime.materialize_prefix_and_reuse_current_index_page_slots(
page_buffer=page_buffer,
current_index_k=current_k,
current_index_scale=current_scale,
current_locs=torch.tensor([8, 9], dtype=torch.int64),
slot_remap=slot_remap,
layout=layout,
page_size=page_size,
index_head_dim=index_head_dim,
prefix_pages=1,
layer_id=2,
)
)
scale_offset = page_size * index_head_dim
self.assertEqual(dense_pages.tolist(), [[1, 2]])
self.assertTrue(torch.equal(dense_page_buffer[1], page_buffer[1]))
self.assertTrue(torch.equal(dense_page_buffer[2, 0:4], current_k[0]))
self.assertTrue(torch.equal(dense_page_buffer[2, 4:8], current_k[1]))
self.assertTrue(
torch.equal(
dense_page_buffer[2, 8:scale_offset],
torch.zeros(8, dtype=torch.uint8),
)
)
self.assertTrue(
torch.equal(
dense_page_buffer[2, scale_offset : scale_offset + 4],
current_scale[0].view(torch.uint8),
)
)
self.assertTrue(
torch.equal(
dense_page_buffer[2, scale_offset + 4 : scale_offset + 8],
current_scale[1].view(torch.uint8),
)
)
self.assertTrue(
torch.equal(
dense_page_buffer[2, scale_offset + 8 :],
torch.zeros(8, dtype=torch.uint8),
)
)
def test_index_prefetch_partial_current_compose_fills_current_page_slots(self):
from sglang.srt.environ import envs
from sglang.srt.layers.attention.nsa import cp_shared_kv_prefetch as prefetch
from sglang.srt.mem_cache.cp_shared_kv_layout import CpSharedKVLayout
class FakeCurrentStream:
def __init__(self):
self.events = []
def wait_event(self, event):
self.events.append(event)
page_size = 4
index_head_dim = 4
scale_bytes = 4
page_bytes = page_size * index_head_dim + page_size * scale_bytes
dense_page_buffer = torch.zeros((3, page_bytes), dtype=torch.uint8)
dense_page_buffer[1] = torch.arange(10, 10 + page_bytes, dtype=torch.uint8)
fake_event = object()
prefetcher = prefetch.CpSharedKVIndexPrefetcher(
layout=CpSharedKVLayout(page_size=page_size, cp_size=1, cp_rank=0),
prefix_pages=1,
slot_logical_pages=torch.tensor([1, 20], dtype=torch.int64),
page_inverse=torch.tensor(
[-1, 1] + [-1] * 18 + [2],
dtype=torch.int64,
),
dense_num_pages=3,
stream=object(),
)
handle = prefetch.CpSharedKVIndexPrefetchHandle(
layer_id=1,
dense_page_buffer=dense_page_buffer,
prefix_rows=slice(1, 2),
event=fake_event,
)
prefetcher.handles[1] = handle
prefetcher.pending_attention_handle = handle
current_k = torch.tensor(
[[11, 12, 13, 14], [15, 16, 17, 18]],
dtype=torch.uint8,
)
current_scale = torch.tensor([[3.25], [4.5]], dtype=torch.float32)
with envs.SGLANG_DEBUG_CP_SHARED_KV.override(True), patch.object(
prefetch.torch.cuda,
"current_stream",
return_value=FakeCurrentStream(),
):
mixed_buffer, dense_pages = prefetcher.consume_prefix_with_current(
layer_id=1,
logical_pages=torch.tensor([[1, 20]], dtype=torch.int64),
current_index_k=current_k,
current_index_scale=current_scale,
current_locs=torch.tensor([80, 81], dtype=torch.int64),
page_size=page_size,
index_head_dim=index_head_dim,
)
scale_offset = page_size * index_head_dim
self.assertEqual(dense_pages.tolist(), [[1, 2]])
self.assertTrue(torch.equal(mixed_buffer[2, 0:4], current_k[0]))
self.assertTrue(torch.equal(mixed_buffer[2, 4:8], current_k[1]))
self.assertTrue(
torch.equal(
mixed_buffer[2, scale_offset : scale_offset + 4],
current_scale[0].view(torch.uint8),
)
)
self.assertEqual(prefetcher.handles, {})
self.assertIsNone(prefetcher.pending_attention_handle)
def test_index_current_reuse_gate_uses_partial_current_contract(self):
from pathlib import Path
source = (
Path(__file__).resolve().parents[4]
/ "python/sglang/srt/layers/attention/nsa/nsa_indexer.py"
).read_text()
start = source.index(" def _can_reuse_current_index_kv")
end = source.index(" @contextlib.contextmanager", start)
method_source = source[start:end]
self.assertIn("should_reuse_current_extend_kv(forward_batch)", method_source)
self.assertNotIn("is_current_only_extend_batch(forward_batch)", method_source)
def test_index_current_reuse_prepare_accepts_padded_out_cache_loc(self):
from pathlib import Path
source = (
Path(__file__).resolve().parents[4]
/ "python/sglang/srt/layers/attention/nsa/nsa_indexer.py"
).read_text()
start = source.index(" if self._can_reuse_current_index_kv")
end = source.index(" if _is_cuda or _is_hip:", start)
prepare_source = source[start:end]
self.assertIn("valid_current_rows", prepare_source)
self.assertIn("key[:valid_current_rows]", prepare_source)
self.assertNotIn(
"key.shape[0] == forward_batch.out_cache_loc.numel()",
prepare_source,
)
def test_materialize_local_token_kv_pages(self):
from sglang.srt.layers.attention.nsa.cp_shared_kv_runtime import (
build_dense_page_remap,