SWA Prefix Cache (#7367)

Co-authored-by: Ying Sheng <sqy1415@gmail.com>
This commit is contained in:
Hanming Lu
2025-07-13 12:31:07 -07:00
committed by GitHub
co-authored by Ying Sheng
parent 0c55cbcfc5
commit 9379da77de
16 changed files with 1742 additions and 158 deletions
+5 -2
View File
@@ -61,7 +61,7 @@ class ChunkCache(BasePrefixCache):
def inc_lock_ref(self, node: Any):
return 0
def dec_lock_ref(self, node: Any):
def dec_lock_ref(self, node: Any, swa_uuid_for_lock: Optional[str] = None):
return 0
def pretty_print(self):
@@ -80,7 +80,7 @@ class SWAChunkCache(ChunkCache):
super().__init__(req_to_token_pool, token_to_kv_pool_allocator, page_size)
assert isinstance(token_to_kv_pool_allocator, SWATokenToKVPoolAllocator)
def evict(
def evict_swa(
self,
req: Req,
prelen: int,
@@ -95,3 +95,6 @@ class SWAChunkCache(ChunkCache):
]
self.token_to_kv_pool_allocator.free_swa(free_slots)
req.evicted_seqlen_local = new_evicted_seqlen_local
def evict(self, num_tokens: int):
pass