diff --git a/python/sglang/srt/server_args.py b/python/sglang/srt/server_args.py index 3aefcae35..8eeec5ea3 100644 --- a/python/sglang/srt/server_args.py +++ b/python/sglang/srt/server_args.py @@ -690,9 +690,6 @@ class ServerArgs: # Apply model-specific adjustments. self._handle_model_specific_adjustments() - # Handle Hicache settings. - self._handle_hicache() - # Set kernel backends. self._handle_sampling_backend() self._handle_attention_backend_compatibility() @@ -701,6 +698,9 @@ class ServerArgs: self._handle_amd_specifics() self._handle_grammar_backend() + # Handle Hicache settings. + self._handle_hicache() + # Handle data parallelism. self._handle_data_parallelism() @@ -1993,23 +1993,14 @@ class ServerArgs: ) def _handle_hicache(self): - if self.hicache_storage_backend == "mooncake": - if self.hicache_mem_layout == "layer_first": - if self.hicache_io_backend == "direct": - self.hicache_mem_layout = "page_first_direct" - elif self.hicache_io_backend == "kernel": - self.hicache_mem_layout = "page_first" - logger.warning( - f"Mooncake storage backend does not support layer_first layout, " - f"switching to {self.hicache_mem_layout} layout for {self.hicache_io_backend} io backend" - ) - - if self.hicache_mem_layout == "page_first_direct": - if self.hicache_io_backend not in ["direct", "kernel_ascend"]: - self.hicache_io_backend = "direct" - logger.warning( - "Page first direct layout only support direct io backend" - ) + if ( + self.hicache_mem_layout == "page_first_direct" + and self.hicache_io_backend is "kernel" + ): + self.hicache_io_backend = "direct" + logger.warning( + "Kernel io backend does not support page first direct layout" + ) if ( self.enable_hierarchical_cache @@ -2042,6 +2033,17 @@ class ServerArgs: "Setting hicache_io_backend to vanilla I/O, which may lead to suboptimal performance with small page sizes." ) + if self.hicache_storage_backend == "mooncake": + if self.hicache_mem_layout == "layer_first": + if self.hicache_io_backend == "direct": + self.hicache_mem_layout = "page_first_direct" + elif self.hicache_io_backend == "kernel": + self.hicache_mem_layout = "page_first" + logger.warning( + f"Mooncake storage backend does not support layer_first layout, " + f"switching to {self.hicache_mem_layout} layout for {self.hicache_io_backend} io backend" + ) + def _handle_speculative_decoding(self): if ( self.speculative_draft_model_path is not None