SWA Prefix Cache (#7367)

Co-authored-by: Ying Sheng <sqy1415@gmail.com>
This commit is contained in:
Hanming Lu
2025-07-13 12:31:07 -07:00
committed by GitHub
parent 0c55cbcfc5
commit 9379da77de
16 changed files with 1742 additions and 158 deletions

View File

@@ -1,5 +1,5 @@
from abc import ABC, abstractmethod
from typing import TYPE_CHECKING, Any, List, NamedTuple, Tuple
from typing import TYPE_CHECKING, Any, List, NamedTuple, Optional, Tuple
import torch
@@ -56,15 +56,27 @@ class BasePrefixCache(ABC):
pass
@abstractmethod
def dec_lock_ref(self, node: Any):
def dec_lock_ref(self, node: Any, swa_uuid_for_lock: Optional[str] = None):
pass
def evictable_size(self):
return 0
def full_evictable_size(self):
return 0
def swa_evictable_size(self):
return 0
def protected_size(self):
return 0
def full_protected_size(self):
return 0
def swa_protected_size(self):
return 0
def total_size(self):
raise NotImplementedError()