Fix CPP Radix Cache and add test to CI (#11645)

This commit is contained in:
cctry
2025-11-11 11:51:37 -08:00
committed by GitHub
parent e8114102fa
commit e0e6a6efb2
2 changed files with 5 additions and 2 deletions

View File

@@ -167,7 +167,7 @@ class RadixCacheCpp(BasePrefixCache):
token_ids = (req.origin_input_ids + req.output_ids)[:kv_committed_len]
kv_indices = self.req_to_token_pool.req_to_token[
req.req_pool_idx, :kv_committed_len
]
].to(dtype=torch.int64, copy=True)
# NOTE: our C++ implementation don't need `token_ids` and `kv_indices` to be page-aligned
# it will automatically align them, but length of them should be equal
@@ -204,7 +204,9 @@ class RadixCacheCpp(BasePrefixCache):
assert req.req_pool_idx is not None
token_ids = req.fill_ids
prefill_len = len(token_ids) # prefill only (maybe chunked)
kv_indices = self.req_to_token_pool.req_to_token[req.req_pool_idx, :prefill_len]
kv_indices = self.req_to_token_pool.req_to_token[
req.req_pool_idx, :prefill_len
].to(dtype=torch.int64, copy=True)
# NOTE: our C++ implementation don't need `token_ids` and `kv_indices` to be page-aligned
# it will automatically align them, but length of them should be equal

View File

@@ -217,6 +217,7 @@ suites = {
TestFile("openai_server/features/test_lora_openai_compatible.py", 150),
TestFile("batch_invariant/test_batch_invariant_ops.py", 10),
TestFile("test_deepseek_v3_deterministic.py", 240),
TestFile("test_cpp_radix_cache.py", 60),
],
"nightly-4-gpu-b200": [
TestFile("test_fp4_moe.py", 300),