[RadixTree][1/N Refactor]: Support unified match_prefix params (#17142)
Co-authored-by: yizhang2077 <1109276519@qq.com> Co-authored-by: pansicheng <sicheng.pan.chn@gmail.com>
This commit is contained in:
co-authored by
yizhang2077
pansicheng
parent
ce8a6ac690
commit
20b0523eca
@@ -35,7 +35,7 @@ import torch
|
||||
from sglang.srt.dllm.config import DllmConfig
|
||||
from sglang.srt.layers.attention.nsa.utils import is_nsa_prefill_cp_in_seq_split
|
||||
from sglang.srt.managers.schedule_batch import DllmStagingReqs, Req, ScheduleBatch
|
||||
from sglang.srt.mem_cache.base_prefix_cache import BasePrefixCache
|
||||
from sglang.srt.mem_cache.base_prefix_cache import BasePrefixCache, MatchPrefixParams
|
||||
from sglang.srt.mem_cache.radix_cache import RadixCache, RadixKey, TreeNode
|
||||
from sglang.srt.mem_cache.swa_memory_pool import SWATokenToKVPoolAllocator
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
@@ -190,7 +190,9 @@ class SchedulePolicy:
|
||||
extra_key = r.extra_key
|
||||
# NOTE: the prefix_indices must always be aligned with last_node
|
||||
match_result = self.tree_cache.match_prefix(
|
||||
rid=r.rid, key=RadixKey(token_ids=prefix_ids, extra_key=extra_key)
|
||||
MatchPrefixParams(
|
||||
key=RadixKey(token_ids=prefix_ids, extra_key=extra_key)
|
||||
)
|
||||
)
|
||||
(
|
||||
r.prefix_indices,
|
||||
@@ -213,8 +215,9 @@ class SchedulePolicy:
|
||||
# It is kind of common when the engine is long running (e.g., imagine the prefix "the").
|
||||
if len(r.prefix_indices) <= IN_BATCH_PREFIX_CACHING_CHECK_THRESHOLD:
|
||||
match_result = self.waiting_queue_radix_tree.match_prefix(
|
||||
rid=r.rid,
|
||||
key=RadixKey(token_ids=prefix_ids, extra_key=extra_key),
|
||||
MatchPrefixParams(
|
||||
key=RadixKey(token_ids=prefix_ids, extra_key=extra_key)
|
||||
)
|
||||
)
|
||||
in_batch_matching_prefixes = match_result.device_indices
|
||||
if (
|
||||
|
||||
Reference in New Issue
Block a user