[Bugfix] fix hicache mooncake backend extra config loading (#16808)

Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Co-authored-by: jinke15 <jinke15@jd.com>
This commit is contained in:
jinke
2026-03-18 15:07:39 +08:00
committed by GitHub
parent d20e9a20fa
commit ae15fca192

View File

@@ -1,5 +1,6 @@
from __future__ import annotations
import json
import logging
import threading
import time
@@ -77,6 +78,17 @@ class DecodeKVCacheOffloadManager:
self.tp_group = tp_group
self.tp_world_size = torch.distributed.get_world_size(group=self.tp_group)
hicache_storage_backend_extra_config = {}
if server_args.hicache_storage_backend_extra_config:
try:
hicache_storage_backend_extra_config = json.loads(
server_args.hicache_storage_backend_extra_config
)
except json.JSONDecodeError as e:
raise ValueError(
f"Invalid hicache storage backend extra config JSON: {e}"
)
self.cache_controller = HiCacheController(
token_to_kv_pool_allocator=self.token_to_kv_pool_allocator,
mem_pool_host=self.decode_host_mem_pool,
@@ -86,7 +98,7 @@ class DecodeKVCacheOffloadManager:
load_cache_event=threading.Event(),
storage_backend=server_args.hicache_storage_backend,
model_name=server_args.served_model_name,
storage_backend_extra_config=server_args.hicache_storage_backend_extra_config,
storage_backend_extra_config=hicache_storage_backend_extra_config,
)
self.ongoing_offload = {}