chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 11 Packages Projects Releases Wiki Activity

Attention backend selection bug fix for hicache (#16779)

Browse Source 
Co-authored-by: hxie <haiyong.xie@ustc.edu>
This commit is contained in:
Zhiqiang Xie
2026-01-10 00:22:41 -08:00
committed by GitHub
parent ad20127359
commit 675acecec6
1 changed files with 22 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
python/sglang/srt/server_args.py
View File

@@ -690,9 +690,6 @@ class ServerArgs:
# Apply model-specific adjustments.
self._handle_model_specific_adjustments()
# Handle Hicache settings.
self._handle_hicache()
# Set kernel backends.
self._handle_sampling_backend()
self._handle_attention_backend_compatibility()
@@ -701,6 +698,9 @@ class ServerArgs:
self._handle_amd_specifics()
self._handle_grammar_backend()
# Handle Hicache settings.
self._handle_hicache()
# Handle data parallelism.
self._handle_data_parallelism()
@@ -1993,23 +1993,14 @@ class ServerArgs:
)
def _handle_hicache(self):
if self.hicache_storage_backend == "mooncake":
if self.hicache_mem_layout == "layer_first":
if self.hicache_io_backend == "direct":
self.hicache_mem_layout = "page_first_direct"
elif self.hicache_io_backend == "kernel":
self.hicache_mem_layout = "page_first"
logger.warning(
f"Mooncake storage backend does not support layer_first layout, "
f"switching to {self.hicache_mem_layout} layout for {self.hicache_io_backend} io backend"
)
if self.hicache_mem_layout == "page_first_direct":
if self.hicache_io_backend not in ["direct", "kernel_ascend"]:
self.hicache_io_backend = "direct"
logger.warning(
"Page first direct layout only support direct io backend"
)
if (
self.hicache_mem_layout == "page_first_direct"
and self.hicache_io_backend is "kernel"
):
self.hicache_io_backend = "direct"
logger.warning(
"Kernel io backend does not support page first direct layout"
)
if (
self.enable_hierarchical_cache
@@ -2042,6 +2033,17 @@ class ServerArgs:
"Setting hicache_io_backend to vanilla I/O, which may lead to suboptimal performance with small page sizes."
)
if self.hicache_storage_backend == "mooncake":
if self.hicache_mem_layout == "layer_first":
if self.hicache_io_backend == "direct":
self.hicache_mem_layout = "page_first_direct"
elif self.hicache_io_backend == "kernel":
self.hicache_mem_layout = "page_first"
logger.warning(
f"Mooncake storage backend does not support layer_first layout, "
f"switching to {self.hicache_mem_layout} layout for {self.hicache_io_backend} io backend"
)
def _handle_speculative_decoding(self):
if (
self.speculative_draft_model_path is not None