chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

Preserve pending CP HiCache backups through final insert

Browse Source 
Deferred non-chunked inserts can happen while a CP HiCache per-layer backup is still in flight. Rolling back the unattached prepared backup at unfinished-request time drops the only reservation that final insertion can attach, so cache_finished_req has to start a post-forward backup and catch up all layers synchronously.\n\nKeep the prepared backup for non-chunked deferred inserts and continue rolling it back for chunked middle inserts, where the suffix may be extended by later chunks and the old backup would cover the wrong range. Document the failure mode so future changes do not rediscover the same fallback path.\n\nConstraint: CP HiCache radix split cannot mutate in-flight backup nodes.\nConstraint: Chunked prefill middle inserts still need rollback because their backup range is not final.\nRejected: Always rollback unattached backups | causes post-forward catch_up_all_layers for non-chunked deferred inserts.\nRejected: Always preserve unattached backups | can attach stale backup ranges for chunked middle inserts.\nConfidence: high\nScope-risk: narrow\nDirective: Do not clear req.cp_hicache_prepared_backup on non-chunked deferred insert without proving final insert no longer needs it.\nTested: remote cjy-glm5-new py_compile radix_cache.py\nTested: remote cjy-glm5-new pytest test_cp_hicache_metadata.py::{nonchunked preserve,chunked rollback} => 2 passed\nNot-tested: live ETE replay after restarting prefill with this exact commit
This commit is contained in:
laoyao0822
2026-06-13 03:44:23 +08:00
parent ceb5345410
commit 1d168d061f
3 changed files with 102 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/registered/unit/mem_cache/test_cp_hicache_metadata.py
+49 -1
View File
@@ -2019,7 +2019,9 @@ class TestHiRadixCacheCPBackup(CustomTestCase):
self.assertEqual(req.prefix_indices.tolist(), list(range(6)))
self.assertEqual(cache.token_to_kv_pool_allocator.freed, [])
def test_cache_unfinished_req_deferred_insert_rolls_back_unattached_backup(self):
def test_cache_unfinished_req_deferred_insert_preserves_unattached_backup_for_final_insert(
self,
):
cache = HiRadixCache.__new__(HiRadixCache)
cache.disable = False
cache._uses_cp_hicache = True
@@ -2059,6 +2061,52 @@ class TestHiRadixCacheCPBackup(CustomTestCase):
cache.cache_unfinished_req(req, chunked=False)
self.assertEqual(rollbacks, [])
self.assertIs(req.cp_hicache_prepared_backup, prepared)
self.assertEqual(req.prefix_indices.tolist(), list(range(6)))
def test_cache_unfinished_chunked_deferred_insert_rolls_back_unattached_backup(
self,
):
cache = HiRadixCache.__new__(HiRadixCache)
cache.disable = False
cache._uses_cp_hicache = True
cache.is_eagle = False
cache.page_size = 4
deferred_node = TreeNode()
rollbacks = []
class Pool:
req_to_token = torch.arange(8, dtype=torch.int64).view(1, 8)
cache.req_to_token_pool = Pool()
cache.token_to_kv_pool_allocator = RecordingTokenAllocator()
cache.insert = lambda params: types.SimpleNamespace(
prefix_len=1,
pending_backup_deferred_node=deferred_node,
)
cache.match_prefix = lambda params: (_ for _ in ()).throw(
AssertionError("deferred insert must not rematch")
)
cache._free_kv_indices_range = lambda *args, **kwargs: (_ for _ in ()).throw(
AssertionError("deferred unfinished insert must keep request KV")
)
cache._rollback_prepared_cp_backup = lambda prepared, reason: rollbacks.append(
(prepared, reason)
)
prepared = types.SimpleNamespace(attached=False)
req = types.SimpleNamespace(
fill_ids=list(range(6)),
req_pool_idx=0,
extra_key=None,
cache_protected_len=1,
last_node=TreeNode(),
prefix_indices=torch.empty((0,), dtype=torch.int64),
cp_hicache_prepared_backup=prepared,
)
cache.cache_unfinished_req(req, chunked=True)
self.assertEqual(
rollbacks, [(prepared, "unfinished_pending_backup_split_deferred")]
)