feat: add page-aligned index validation for CP HiCache direct transfers
Add validate_page_aligned_token_indices utility and apply it across HiCache write/load paths, NSA indexer transfers, and CUDA direct copy kernels to reject malformed (partial, misaligned, non-contiguous) page groups before they reach native transfer code. Also validate supported CP HiCache backend/layout combinations at server startup.
This commit is contained in:
@@ -686,6 +686,38 @@ inline void transfer_page_direct(
|
||||
/* non_blocking= */ true);
|
||||
}
|
||||
|
||||
inline void validate_page_first_direct_indices(
|
||||
const int64_t* indices_ptr, int64_t num_indices, int64_t page_size, const char* name) {
|
||||
if (num_indices == 0) {
|
||||
return;
|
||||
}
|
||||
for (int64_t i = 0; i < num_indices; i += page_size) {
|
||||
const int64_t start = indices_ptr[i];
|
||||
TORCH_CHECK(
|
||||
start % page_size == 0,
|
||||
name,
|
||||
" page groups must start at page boundaries: group=",
|
||||
i / page_size,
|
||||
" start=",
|
||||
start,
|
||||
" page_size=",
|
||||
page_size);
|
||||
for (int64_t offset = 1; offset < page_size; ++offset) {
|
||||
TORCH_CHECK(
|
||||
indices_ptr[i + offset] == start + offset,
|
||||
name,
|
||||
" page groups must be contiguous page spans: group=",
|
||||
i / page_size,
|
||||
" offset=",
|
||||
offset,
|
||||
" expected=",
|
||||
start + offset,
|
||||
" got=",
|
||||
indices_ptr[i + offset]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void transfer_kv_direct(
|
||||
const std::vector<at::Tensor>& src_layers,
|
||||
std::vector<at::Tensor> dst_layers,
|
||||
@@ -749,6 +781,8 @@ inline void transfer_kv_page_first_direct_impl(
|
||||
const int64_t num_pages = src_indices_cpu.size(0) / page_size;
|
||||
int64_t* src_indices_ptr = src_indices_cpu.data_ptr<int64_t>();
|
||||
int64_t* dst_indices_ptr = dst_indices_cpu.data_ptr<int64_t>();
|
||||
validate_page_first_direct_indices(src_indices_ptr, src_indices_cpu.numel(), page_size, "src_indices");
|
||||
validate_page_first_direct_indices(dst_indices_ptr, dst_indices_cpu.numel(), page_size, "dst_indices");
|
||||
|
||||
auto fallback_to_page_copy = [&]() {
|
||||
if constexpr (IsLf2Pf) {
|
||||
|
||||
@@ -33,6 +33,15 @@ def ref_copy_with_indices_pf_direct(
|
||||
][layer_id].to(dst_pool.device)
|
||||
|
||||
|
||||
def make_page_indices(page_ids, page_size):
|
||||
return torch.cat(
|
||||
[
|
||||
torch.arange(p * page_size, (p + 1) * page_size, dtype=torch.int64)
|
||||
for p in page_ids
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def ref_copy_with_indices_page_head(
|
||||
src_pool,
|
||||
dst_pool,
|
||||
@@ -522,6 +531,54 @@ def test_transfer_kv_pf_direct(
|
||||
torch.set_default_dtype(original_dtype)
|
||||
|
||||
|
||||
@pytest.mark.skipif(is_hip(), reason="HIP uses the fallback path for this direct validation")
|
||||
def test_transfer_kv_all_layer_direct_lf_pf_rejects_misaligned_page_start():
|
||||
page_size = 4
|
||||
item_size = 8
|
||||
num_layers = 1
|
||||
src_pool = torch.randn(num_layers, 32, item_size, device="cuda")
|
||||
dst_pool = torch.zeros(8, num_layers, page_size, item_size).pin_memory()
|
||||
src_indices = torch.tensor([1, 2, 3, 4], dtype=torch.int64)
|
||||
dst_indices = make_page_indices([0], page_size)
|
||||
|
||||
with pytest.raises(RuntimeError, match="page boundaries"):
|
||||
transfer_kv_all_layer_direct_lf_pf(
|
||||
[src_pool[0]], [dst_pool], src_indices, dst_indices, page_size
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(is_hip(), reason="HIP uses the fallback path for this direct validation")
|
||||
def test_transfer_kv_all_layer_direct_lf_pf_rejects_non_contiguous_page_group():
|
||||
page_size = 4
|
||||
item_size = 8
|
||||
num_layers = 1
|
||||
src_pool = torch.randn(num_layers, 32, item_size, device="cuda")
|
||||
dst_pool = torch.zeros(8, num_layers, page_size, item_size).pin_memory()
|
||||
src_indices = torch.tensor([4, 5, 7, 6], dtype=torch.int64)
|
||||
dst_indices = make_page_indices([0], page_size)
|
||||
|
||||
with pytest.raises(RuntimeError, match="contiguous page spans"):
|
||||
transfer_kv_all_layer_direct_lf_pf(
|
||||
[src_pool[0]], [dst_pool], src_indices, dst_indices, page_size
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(is_hip(), reason="HIP uses the fallback path for this direct validation")
|
||||
def test_transfer_kv_per_layer_direct_pf_lf_rejects_misaligned_host_page_start():
|
||||
page_size = 4
|
||||
item_size = 8
|
||||
layer_id = 0
|
||||
src_pool = torch.randn(8, 1, page_size, item_size).pin_memory()
|
||||
dst_pool = torch.zeros(1, 32, item_size, device="cuda")
|
||||
src_indices = torch.tensor([1, 2, 3, 4], dtype=torch.int64)
|
||||
dst_indices = make_page_indices([0], page_size).to("cuda")
|
||||
|
||||
with pytest.raises(RuntimeError, match="page boundaries"):
|
||||
transfer_kv_per_layer_direct_pf_lf(
|
||||
[src_pool], [dst_pool[0]], src_indices, dst_indices, layer_id, page_size
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(is_hip(), reason="HIP is not supported for this test")
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
|
||||
@pytest.mark.parametrize("num_items_to_transfer", [256, 1024])
|
||||
|
||||
Reference in New Issue
Block a user