Preserve request-slot identity for CP cache-hit reuse

Warm-cache bs>1 requests can carry duplicate logical page ids across request rows. A batch-global page inverse collapses those request slots and can alias current/prefix KV across requests.

The runtime now carries compact row-scoped sorted page descriptors and remaps flattened logical locs with request row ids where needed, while retaining the legacy global inverse for no-row-context paths.

Constraint: Avoid the rejected dense [batch, logical_page_capacity] inverse because bs up to 10 and long contexts make that memory cost unacceptable
Rejected: Keep global page_inverse for bs>1 duplicate pages | it is lossy and matches the GSM8K warm-cache corruption shape
Rejected: Allocate page_inverse_by_row | correctness-safe but too much GPU memory for production
Confidence: medium
Scope-risk: moderate
Directive: Any future TAI materialize fast path for bs>1 duplicate pages must consume row-scoped descriptors or an equivalent request-row key
Tested: Local py_compile for touched runtime files; remote py_compile; remote pytest test_cp_shared_kv_runtime.py, test_nsa_cp_utils.py, test_cp_shared_kv_layout.py => 263 passed, 5 warnings, 2 subtests passed
Not-tested: Full GSM8K ETE after this cleanup pass
This commit is contained in:
laoyao0822
2026-06-08 20:56:31 +08:00
parent fb5ccaff26
commit b58513cba4
4 changed files with 912 additions and 23 deletions
@@ -1374,6 +1374,246 @@ class TestCpSharedKVRuntimeHelpers(unittest.TestCase):
)
)
@unittest.skipIf(not torch.cuda.is_available(), "CUDA is required")
def test_tai_current_slot_fill_matches_torch_for_parallel20_tiny_fp8_rows(
self,
):
from sglang.srt.environ import envs
from sglang.srt.layers.attention.nsa import cp_shared_kv_runtime as runtime
kernels = runtime._load_tai_materialize_kernels()
if kernels is None or not hasattr(
kernels, "fill_current_token_kv_page_slots_and_remap_locs"
):
self.skipTest("TAI current-slot fill kernel is not installed")
device = torch.device("cuda", torch.cuda.current_device())
page_size = 64
prefix_lens = [704, 704, 640, 704, 704]
extend_lens = [29, 9, 60, 9, 17]
pages_per_request = max(
(prefix + extend + page_size - 1) // page_size
for prefix, extend in zip(prefix_lens, extend_lens)
)
rows = []
logical_locs = []
current_locs = []
next_page = 1
for prefix_len, extend_len in zip(prefix_lens, extend_lens):
prefix_pages = prefix_len // page_size
current_pages = (prefix_len + extend_len + page_size - 1) // page_size
req_pages = []
for _ in range(current_pages):
req_pages.append(next_page)
next_page += 1
req_pages.extend([0] * (pages_per_request - len(req_pages)))
rows.append(req_pages)
for page_idx in range(prefix_pages):
page = req_pages[page_idx]
logical_locs.extend(
page * page_size + offset for offset in range(page_size)
)
cur_page = req_pages[prefix_pages]
cur = [cur_page * page_size + offset for offset in range(extend_len)]
logical_locs.extend(cur)
current_locs.extend(cur)
remap_logical_pages = torch.tensor(rows, device=device, dtype=torch.long)
slot_logical_pages, _ = runtime.build_slot_page_remap(remap_logical_pages)
page_inverse = runtime.build_slot_page_inverse(
slot_logical_pages, logical_page_capacity=next_page * page_size
)
logical_locs_t = torch.tensor(logical_locs, device=device, dtype=torch.long)
current_locs_t = torch.tensor(current_locs, device=device, dtype=torch.long)
materialized_locs = runtime.remap_logical_locs_to_slot_dense_locs(
logical_locs_t, page_inverse=page_inverse, page_size=page_size
)
dense_rows = (int(slot_logical_pages.numel()) + 1) * page_size
# Use production FP8 MLA row width. uint8 makes byte-copy exactness visible
# without depending on fp8 value quantization in the unit test.
row_width = 656
base_current = torch.arange(
int(current_locs_t.numel()) * row_width,
device=device,
dtype=torch.int64,
)
current_kv = (base_current % 251).to(torch.uint8).view(
int(current_locs_t.numel()), 1, row_width
)
dense_ref = torch.zeros((dense_rows, 1, row_width), device=device, dtype=torch.uint8)
dense_tai = torch.zeros_like(dense_ref)
with patch.object(runtime, "_tai_materialize_runtime_enabled", return_value=False):
ref_kv, ref_locs, ref_mask = runtime.fill_current_kv_page_slots_and_remap_locs(
dense_kv_cache=dense_ref,
materialized_dense_locs=materialized_locs,
current_kv_cache=current_kv,
logical_locs=logical_locs_t,
current_locs=current_locs_t,
page_inverse=page_inverse,
page_size=page_size,
mask_non_current_in_current_pages=True,
)
runtime._tai_current_slot_fill_sparse_pages_self_test.cache_clear()
with envs.SGLANG_CP_SHARED_KV_USE_TAI_MATERIALIZE.override(True), patch.object(
runtime,
"_tai_current_slot_fill_supports_sparse_pages",
return_value=True,
):
tai_kv, tai_locs, tai_mask = runtime.fill_current_kv_page_slots_and_remap_locs(
dense_kv_cache=dense_tai,
materialized_dense_locs=materialized_locs,
current_kv_cache=current_kv,
logical_locs=logical_locs_t,
current_locs=current_locs_t,
page_inverse=page_inverse,
page_size=page_size,
mask_non_current_in_current_pages=True,
)
self.assertTrue(torch.equal(tai_locs, ref_locs))
self.assertTrue(torch.equal(tai_mask, ref_mask))
self.assertTrue(torch.equal(tai_kv, ref_kv))
@unittest.skipIf(not torch.cuda.is_available(), "CUDA is required")
def test_fp8_paged_dequant_matches_reference_for_parallel20_slot_locs(self):
from sglang.srt.layers.attention.nsa import cp_shared_kv_runtime as runtime
from sglang.srt.layers.attention.nsa.dequant_k_cache import (
dequantize_k_cache_paged,
)
device = torch.device("cuda", torch.cuda.current_device())
page_size = 64
prefix_lens = [704, 704, 640, 704, 704]
extend_lens = [29, 9, 60, 9, 17]
pages_per_request = max(
(prefix + extend + page_size - 1) // page_size
for prefix, extend in zip(prefix_lens, extend_lens)
)
rows = []
logical_locs = []
next_page = 1
for prefix_len, extend_len in zip(prefix_lens, extend_lens):
seq_len = prefix_len + extend_len
req_pages = []
for _ in range((seq_len + page_size - 1) // page_size):
req_pages.append(next_page)
next_page += 1
req_pages.extend([0] * (pages_per_request - len(req_pages)))
rows.append(req_pages)
for token_idx in range(seq_len):
page = req_pages[token_idx // page_size]
logical_locs.append(page * page_size + token_idx % page_size)
remap_logical_pages = torch.tensor(rows, device=device, dtype=torch.long)
slot_logical_pages, _ = runtime.build_slot_page_remap(remap_logical_pages)
page_inverse = runtime.build_slot_page_inverse(
slot_logical_pages, logical_page_capacity=next_page * page_size
)
logical_locs_t = torch.tensor(logical_locs, device=device, dtype=torch.long)
mixed_locs = runtime.remap_logical_locs_to_slot_dense_locs(
logical_locs_t, page_inverse=page_inverse, page_size=page_size
)
self.assertFalse(bool(torch.any(mixed_locs < 0).item()))
dense_rows = (int(slot_logical_pages.numel()) + 1) * page_size
packed_u8 = torch.empty((dense_rows, 656), device=device, dtype=torch.uint8)
packed_fp8 = packed_u8.view(torch.float8_e4m3fn)
nope = packed_fp8[:, :512]
scales = packed_fp8[:, 512:528].view(torch.float32)
rope = packed_fp8[:, 528:].view(torch.bfloat16)
token_ids = torch.arange(dense_rows, device=device, dtype=torch.float32).view(-1, 1)
dim_ids = torch.arange(512, device=device, dtype=torch.float32).view(1, -1)
nope.copy_(((token_ids % 7) + (dim_ids % 13) / 16.0).to(torch.float8_e4m3fn))
scales.copy_(torch.ones_like(scales))
rope_token_ids = torch.arange(dense_rows, device=device, dtype=torch.float32).view(-1, 1)
rope_dim_ids = torch.arange(64, device=device, dtype=torch.float32).view(1, -1)
rope.copy_((rope_token_ids * 0.25 + rope_dim_ids / 32.0).to(torch.bfloat16))
out = dequantize_k_cache_paged(
packed_fp8.view(dense_rows, 1, 656),
mixed_locs.to(torch.int64),
).view(-1, 576)
selected = packed_fp8.view(dense_rows, 656)[mixed_locs.to(torch.long)]
selected_nope = selected[:, :512].to(torch.float32)
selected_scales = selected[:, 512:528].view(torch.float32)
selected_rope = selected[:, 528:].view(torch.bfloat16)
expected = torch.empty((int(mixed_locs.numel()), 576), device=device, dtype=torch.bfloat16)
for tile in range(4):
expected[:, tile * 128 : (tile + 1) * 128] = (
selected_nope[:, tile * 128 : (tile + 1) * 128]
* selected_scales[:, tile : tile + 1]
).to(torch.bfloat16)
expected[:, 512:] = selected_rope
self.assertTrue(torch.equal(out, expected))
@unittest.skipIf(not torch.cuda.is_available(), "CUDA is required")
def test_flashmla_sparse_valid_rows_match_compact_with_compute_padding(self):
try:
from sgl_kernel.flash_mla import flash_mla_sparse_fwd
except Exception as exc:
self.skipTest(f"flash_mla_sparse_fwd is unavailable: {exc}")
device = torch.device("cuda", torch.cuda.current_device())
torch.manual_seed(1234)
num_heads = 64
head_dim = 576
value_dim = 512
topk = 2048
valid_lens = [29, 9, 60, 9, 17]
compute_rows_per_req = 64
output_rows = len(valid_lens) * compute_rows_per_req
kv_lens = [704 + x for x in valid_lens]
kv_total = sum(kv_lens)
q_padded = torch.randn(
(output_rows, num_heads, head_dim),
device=device,
dtype=torch.bfloat16,
) / 10
kv = torch.randn((kv_total, 1, head_dim), device=device, dtype=torch.bfloat16) / 10
indices_padded = torch.full(
(output_rows, 1, topk), -1, device=device, dtype=torch.int32
)
valid_rows = []
kv_base = 0
out_base = 0
for valid_len, kv_len in zip(valid_lens, kv_lens):
for row in range(valid_len):
out_row = out_base + row
valid_rows.append(out_row)
indices_padded[out_row, 0, :kv_len] = torch.arange(
kv_base, kv_base + kv_len, device=device, dtype=torch.int32
)
kv_base += kv_len
out_base += compute_rows_per_req
valid_rows_t = torch.tensor(valid_rows, device=device, dtype=torch.long)
sm_scale = 1.0 / (head_dim**0.5)
padded_out, _, _ = flash_mla_sparse_fwd(
q_padded, kv, indices_padded, sm_scale=sm_scale, d_v=value_dim
)
compact_out, _, _ = flash_mla_sparse_fwd(
q_padded[valid_rows_t].contiguous(),
kv,
indices_padded[valid_rows_t].contiguous(),
sm_scale=sm_scale,
d_v=value_dim,
)
torch.testing.assert_close(
padded_out[valid_rows_t].float(),
compact_out.float(),
atol=8e-4,
rtol=2.01 / 128,
)
def test_materialize_prefix_and_reuse_current_kv_page_slots_without_prefetcher(
self,
):
@@ -2547,6 +2787,66 @@ class TestCpSharedKVRuntimeHelpers(unittest.TestCase):
self.assertEqual(remap.dense_num_pages, 7)
self.assertEqual(remap.dense_locs.tolist(), [[4, 8, -1], [16, 0, 20]])
def test_token_slot_remap_duplicate_pages_keep_request_slot_identity(self):
from sglang.srt.layers.attention.nsa.cp_shared_kv_runtime import (
build_shared_token_kv_slot_remap,
)
from sglang.srt.mem_cache.cp_shared_kv_layout import CpSharedKVLayout
page_size = 4
layout = CpSharedKVLayout(page_size=page_size, cp_size=1, cp_rank=0)
kv_cache = torch.zeros((64, 1, 1), dtype=torch.float32)
# Two requests share logical page 1 as prefix. The slot layout gives
# the two occurrences distinct dense slots: req0 slot 0 -> dense page 1,
# req1 slot 0 -> dense page 3. A request-packed dense buffer must keep
# those request slots addressable independently.
remap_logical_pages = torch.tensor(
[
[1, 2],
[1, 3],
],
dtype=torch.int64,
)
logical_locs = torch.tensor(
[
[1 * page_size, 2 * page_size],
[1 * page_size, 3 * page_size],
],
dtype=torch.int64,
)
remap = build_shared_token_kv_slot_remap(
kv_cache=kv_cache,
logical_locs=logical_locs,
remap_logical_pages=remap_logical_pages,
layout=layout,
page_size=page_size,
)
self.assertEqual(remap.slot_logical_pages.tolist(), [1, 2, 1, 3])
self.assertEqual(remap.dense_num_pages, 5)
self.assertEqual(
tuple(remap.slot_sorted_logical_pages_by_row.shape),
tuple(remap_logical_pages.shape),
)
self.assertEqual(
tuple(remap.slot_sorted_dense_pages_by_row.shape),
tuple(remap_logical_pages.shape),
)
self.assertLess(
remap.slot_sorted_logical_pages_by_row.numel(),
remap.logical_page_capacity * remap_logical_pages.shape[0],
"row-aware slot remap must stay compact and avoid dense "
"[batch, logical_page_capacity] inverse allocation",
)
self.assertEqual(
remap.dense_locs.tolist(),
[
[1 * page_size, 2 * page_size],
[3 * page_size, 4 * page_size],
],
)
def test_forward_batch_token_slot_remap_is_cached_across_layers(self):
from sglang.srt.layers.attention.nsa import cp_shared_kv_runtime as runtime
from sglang.srt.mem_cache.cp_shared_kv_layout import CpSharedKVLayout
@@ -4086,6 +4386,137 @@ class TestCpSharedKVRuntimeHelpers(unittest.TestCase):
"remote-rank valid current loc must remain visible after current slot all-reduce",
)
def test_cp8_batch_kv_partial_current_keeps_request_packed_layout(self):
from sglang.srt.layers.attention.nsa import cp_shared_kv_runtime as runtime
from sglang.srt.mem_cache.cp_shared_kv_layout import CpSharedKVLayout
page_size = 4
cp_size = 8
prefix_lens = [8, 8, 4, 8]
extend_lens = [2, 3, 1, 2]
# Current pages are deliberately rank0-owned while prefix pages are
# spread across other owners. This matches the tiny warm-cache shape
# where all valid current rows live on CP0 but the prefix is assembled
# from all CP ranks.
logical_pages = torch.tensor(
[
[1, 2, 9],
[3, 4, 17],
[5, 25, 0],
[6, 7, 33],
],
dtype=torch.int64,
)
logical_locs_by_req = []
expected_rows = []
current_locs = []
current_rows = []
for req_id, (prefix_len, extend_len) in enumerate(
zip(prefix_lens, extend_lens, strict=True)
):
locs = []
values = []
for token_idx in range(prefix_len + extend_len):
page_idx = token_idx // page_size
token_off = token_idx % page_size
logical_page = int(logical_pages[req_id, page_idx].item())
loc = logical_page * page_size + token_off
value = float(req_id * 1000 + token_idx)
locs.append(loc)
values.append([value])
if token_idx >= prefix_len:
current_locs.append(loc)
current_rows.append([value])
logical_locs_by_req.append(locs)
expected_rows.extend(values)
max_seq_len = max(len(x) for x in logical_locs_by_req)
logical_locs = torch.full(
(len(logical_locs_by_req), max_seq_len),
-1,
dtype=torch.int64,
)
for req_id, locs in enumerate(logical_locs_by_req):
logical_locs[req_id, : len(locs)] = torch.tensor(locs, dtype=torch.int64)
current_locs_tensor = torch.tensor(current_locs, dtype=torch.int64)
current_kv = torch.tensor(current_rows, dtype=torch.float32)
expected = torch.tensor(expected_rows, dtype=torch.float32)
prefix_slot_spans = runtime.build_batch_prefix_slot_spans(
logical_pages=logical_pages,
prefix_lens_cpu=prefix_lens,
page_size=page_size,
)
current_slot_spans = runtime.build_batch_current_slot_spans(
logical_pages=logical_pages,
prefix_lens_cpu=prefix_lens,
extend_lens_cpu=extend_lens,
page_size=page_size,
)
rank_outputs = []
mixed_locs_ref = None
physical_tokens = 64 * page_size
for rank in range(cp_size):
layout = CpSharedKVLayout(page_size=page_size, cp_size=cp_size, cp_rank=rank)
kv_cache = torch.zeros((physical_tokens, 1), dtype=torch.float32)
for req_id, locs in enumerate(logical_locs_by_req):
for token_idx, loc in enumerate(locs):
if token_idx >= prefix_lens[req_id]:
continue
loc_tensor = torch.tensor([loc], dtype=torch.int64)
if bool(layout.owned_by_this_rank(loc_tensor)[0].item()):
phys = int(layout.logical_locs_to_physical(loc_tensor)[0].item())
kv_cache[phys] = float(req_id * 1000 + token_idx)
rank_current_locs = current_locs_tensor[
layout.owned_by_this_rank(current_locs_tensor)
]
if rank == 0:
# All selected current pages are rank0-owned by construction.
rank_current_kv = current_kv
self.assertEqual(rank_current_locs.tolist(), current_locs)
else:
rank_current_kv = torch.empty((0, 1), dtype=torch.float32)
self.assertEqual(rank_current_locs.numel(), 0)
slot_remap = runtime.build_shared_token_kv_slot_remap(
kv_cache=kv_cache,
logical_locs=logical_locs,
remap_logical_pages=logical_pages,
layout=layout,
page_size=page_size,
)
with patch.object(
runtime, "_all_reduce_materialized_buffer_range", _identity_all_reduce
):
mixed_kv, mixed_locs = runtime.materialize_prefix_and_reuse_current_kv_page_slots(
kv_cache=kv_cache,
logical_locs=logical_locs,
current_kv_cache=rank_current_kv,
current_locs=rank_current_locs,
slot_remap=slot_remap,
layout=layout,
page_size=page_size,
prefix_pages=0,
prefix_slot_spans=prefix_slot_spans,
current_slot_spans=current_slot_spans,
layer_id=0,
)
if mixed_locs_ref is None:
mixed_locs_ref = mixed_locs
else:
self.assertTrue(torch.equal(mixed_locs_ref, mixed_locs))
rank_outputs.append(mixed_kv)
assert mixed_locs_ref is not None
merged_kv = torch.stack(rank_outputs, dim=0).sum(dim=0)
packed_locs = mixed_locs_ref[mixed_locs_ref >= 0].to(torch.long)
actual = merged_kv[packed_locs]
self.assertTrue(torch.equal(actual, expected))
@unittest.skipIf(not torch.cuda.is_available(), "CUDA required")
def test_tai_batched_index_mqa_prepare_matches_getk_gets_reference_gsm8k_bs5(self):
from types import SimpleNamespace