chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 11 Packages Projects Releases Wiki Activity

Fix CPP Radix Cache and add test to CI (#11645)

Browse Source
This commit is contained in:
cctry
2025-11-11 11:51:37 -08:00
committed by GitHub
parent e8114102fa
commit e0e6a6efb2
2 changed files with 5 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
python/sglang/srt/mem_cache/radix_cache_cpp.py
View File

@@ -167,7 +167,7 @@ class RadixCacheCpp(BasePrefixCache):
token_ids = (req.origin_input_ids + req.output_ids)[:kv_committed_len]
kv_indices = self.req_to_token_pool.req_to_token[
req.req_pool_idx, :kv_committed_len
]
].to(dtype=torch.int64, copy=True)
# NOTE: our C++ implementation don't need `token_ids` and `kv_indices` to be page-aligned
# it will automatically align them, but length of them should be equal
@@ -204,7 +204,9 @@ class RadixCacheCpp(BasePrefixCache):
assert req.req_pool_idx is not None
token_ids = req.fill_ids
prefill_len = len(token_ids) # prefill only (maybe chunked)
kv_indices = self.req_to_token_pool.req_to_token[req.req_pool_idx, :prefill_len]
kv_indices = self.req_to_token_pool.req_to_token[
req.req_pool_idx, :prefill_len
].to(dtype=torch.int64, copy=True)
# NOTE: our C++ implementation don't need `token_ids` and `kv_indices` to be page-aligned
# it will automatically align them, but length of them should be equal