Refactor prefix cache type checking (#17028)

This commit is contained in:
Ke Bao
2026-01-15 11:28:13 +08:00
committed by GitHub
parent 6b065298b5
commit 7f8a58fffb
12 changed files with 71 additions and 36 deletions

View File

@@ -351,10 +351,24 @@ def prepare_synthetic_inputs_for_latency_test(
return reqs
class TreeCacheNamespace(SimpleNamespace):
def supports_swa(self) -> bool:
return False
def supports_mamba(self) -> bool:
return False
def is_chunk_cache(self) -> bool:
return False
def is_tree_cache(self) -> bool:
return not self.is_chunk_cache()
@torch.no_grad
def extend(reqs, model_runner):
# Create dummy tree_cache for benchmarks (no prefix caching, just allocation)
dummy_tree_cache = SimpleNamespace(
dummy_tree_cache = TreeCacheNamespace(
page_size=model_runner.server_args.page_size,
device=model_runner.device,
token_to_kv_pool_allocator=model_runner.token_to_kv_pool_allocator,