[Core] in batch prefix caching by delay scheduling (#2442)

This commit is contained in:
SangBin Cho
2024-12-11 12:51:50 -08:00
committed by GitHub
parent 864bf2ba00
commit 9208618b3e
8 changed files with 87 additions and 16 deletions
+2 -2
View File
@@ -2,7 +2,7 @@ from __future__ import annotations
"""Cache for chunked prefill, used when RadixCache is disabled."""
from typing import TYPE_CHECKING, Callable, List, Optional
from typing import TYPE_CHECKING, Callable, List, Optional, Tuple
from sglang.srt.mem_cache.base_prefix_cache import BasePrefixCache
from sglang.srt.mem_cache.memory_pool import BaseTokenToKVPool, ReqToTokenPool
@@ -30,7 +30,7 @@ class ChunkCache(BasePrefixCache):
def reset(self):
self.entries = {}
def match_prefix(self, rid: int, key: List[int]):
def match_prefix(self, rid: int, key: List[int]) -> Tuple[List[int], int]:
if rid not in self.entries:
return [], None