[HiCache] fix compatibility bugs with eagle and HiCacheStorage (#19570)
Co-authored-by: maoyuhan <susanmao1997@outlook.com>
This commit is contained in:
@@ -39,6 +39,7 @@ from sglang.srt.mem_cache.radix_cache import (
|
||||
compute_node_hash_values,
|
||||
split_node_hash_value,
|
||||
)
|
||||
from sglang.srt.mem_cache.utils import convert_to_bigram_key
|
||||
from sglang.srt.observability.metrics_collector import StorageMetricsCollector
|
||||
from sglang.srt.utils import bind_to_closest_numa_node_cuda
|
||||
|
||||
@@ -1298,6 +1299,11 @@ class HiRadixCache(RadixCache):
|
||||
last_hash: Optional[str] = None,
|
||||
prefix_keys: Optional[List[str]] = None,
|
||||
):
|
||||
new_input_tokens = (
|
||||
convert_to_bigram_key(new_input_tokens)
|
||||
if self.is_eagle
|
||||
else new_input_tokens
|
||||
)
|
||||
# align the number of fetching tokens to the page size
|
||||
prefetch_length = len(new_input_tokens) - (
|
||||
len(new_input_tokens) % self.page_size
|
||||
|
||||
Reference in New Issue
Block a user