[RadixTree][3/N Refactor]:Support unified insert/evict params (#17401)
This commit is contained in:
@@ -9,6 +9,10 @@ import torch
|
||||
|
||||
from sglang.srt.mem_cache.base_prefix_cache import (
|
||||
BasePrefixCache,
|
||||
EvictParams,
|
||||
EvictResult,
|
||||
InsertParams,
|
||||
InsertResult,
|
||||
MatchPrefixParams,
|
||||
MatchResult,
|
||||
)
|
||||
@@ -54,6 +58,10 @@ class ChunkCache(BasePrefixCache):
|
||||
last_host_node=None,
|
||||
)
|
||||
|
||||
def insert(self, params: InsertParams) -> InsertResult:
|
||||
# ChunkCache does not support prefix caching, so insert is a no-op
|
||||
return InsertResult(prefix_len=0)
|
||||
|
||||
def cache_finished_req(self, req: Req, is_insert: bool = True):
|
||||
kv_committed_len = req.pop_committed_kv_cache()
|
||||
# For decode server: if req.output_ids is empty, we want to free all req.origin_input_ids
|
||||
@@ -70,8 +78,8 @@ class ChunkCache(BasePrefixCache):
|
||||
# `req.prefix_indices` will be used in `PrefillAdder::add_chunked_req` later
|
||||
req.prefix_indices = kv_indices.to(dtype=torch.int64, copy=True)
|
||||
|
||||
def evict(self, num_tokens: int):
|
||||
pass
|
||||
def evict(self, params: EvictParams) -> EvictResult:
|
||||
return EvictResult()
|
||||
|
||||
def inc_lock_ref(self, node: Any):
|
||||
return 0
|
||||
@@ -103,5 +111,5 @@ class SWAChunkCache(ChunkCache):
|
||||
), "sliding_window_size must be set for SWAChunkCache"
|
||||
return True
|
||||
|
||||
def evict(self, num_tokens: int):
|
||||
pass
|
||||
def evict(self, params: EvictParams) -> EvictResult:
|
||||
return EvictResult()
|
||||
|
||||
Reference in New Issue
Block a user