b41afa3754
Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com> Co-authored-by: cctry <shiyang@x.ai>
47 lines
1.3 KiB
Python
47 lines
1.3 KiB
Python
from __future__ import annotations
|
|
|
|
from abc import ABC, abstractmethod
|
|
from typing import TYPE_CHECKING, Tuple, Union
|
|
|
|
if TYPE_CHECKING:
|
|
from sglang.srt.mem_cache.radix_cache import TreeNode
|
|
|
|
|
|
class EvictionStrategy(ABC):
|
|
@abstractmethod
|
|
def get_priority(self, node: "TreeNode") -> Union[float, Tuple]:
|
|
pass
|
|
|
|
|
|
class LRUStrategy(EvictionStrategy):
|
|
def get_priority(self, node: "TreeNode") -> float:
|
|
return node.last_access_time
|
|
|
|
|
|
class LFUStrategy(EvictionStrategy):
|
|
def get_priority(self, node: "TreeNode") -> Tuple[int, float]:
|
|
return (node.hit_count, node.last_access_time)
|
|
|
|
|
|
class FIFOStrategy(EvictionStrategy):
|
|
def get_priority(self, node: "TreeNode") -> float:
|
|
return node.creation_time
|
|
|
|
|
|
class MRUStrategy(EvictionStrategy):
|
|
def get_priority(self, node: "TreeNode") -> float:
|
|
return -node.last_access_time
|
|
|
|
|
|
class FILOStrategy(EvictionStrategy):
|
|
def get_priority(self, node: "TreeNode") -> float:
|
|
return -node.creation_time
|
|
|
|
|
|
class PriorityStrategy(EvictionStrategy):
|
|
"""Priority-aware eviction: lower priority values evicted first, then LRU within same priority."""
|
|
|
|
def get_priority(self, node: "TreeNode") -> Tuple[int, float]:
|
|
# Return (priority, last_access_time) so lower priority nodes are evicted first
|
|
return (node.priority, node.last_access_time)
|