[RadixTree][3/N Refactor]:Support unified insert/evict params (#17401)

This commit is contained in:
zhangheng
2026-01-22 17:36:31 +08:00
committed by GitHub
parent 2262c5c9b5
commit f33022d039
13 changed files with 462 additions and 138 deletions
+12 -4
View File
@@ -9,6 +9,10 @@ import torch
from sglang.srt.mem_cache.base_prefix_cache import (
BasePrefixCache,
EvictParams,
EvictResult,
InsertParams,
InsertResult,
MatchPrefixParams,
MatchResult,
)
@@ -54,6 +58,10 @@ class ChunkCache(BasePrefixCache):
last_host_node=None,
)
def insert(self, params: InsertParams) -> InsertResult:
# ChunkCache does not support prefix caching, so insert is a no-op
return InsertResult(prefix_len=0)
def cache_finished_req(self, req: Req, is_insert: bool = True):
kv_committed_len = req.pop_committed_kv_cache()
# For decode server: if req.output_ids is empty, we want to free all req.origin_input_ids
@@ -70,8 +78,8 @@ class ChunkCache(BasePrefixCache):
# `req.prefix_indices` will be used in `PrefillAdder::add_chunked_req` later
req.prefix_indices = kv_indices.to(dtype=torch.int64, copy=True)
def evict(self, num_tokens: int):
pass
def evict(self, params: EvictParams) -> EvictResult:
return EvictResult()
def inc_lock_ref(self, node: Any):
return 0
@@ -103,5 +111,5 @@ class SWAChunkCache(ChunkCache):
), "sliding_window_size must be set for SWAChunkCache"
return True
def evict(self, num_tokens: int):
pass
def evict(self, params: EvictParams) -> EvictResult:
return EvictResult()