chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 11 Packages Projects Releases Wiki Activity

configuration file support and nixl integration augmentation for hicache-storage-backend-extra-config (#16602)

Browse Source
This commit is contained in:
hxie
2026-01-22 14:31:48 -08:00
committed by GitHub
parent a296c99ff4
commit 13f88045b3
8 changed files with 549 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
docs/advanced_features/hicache_design.md
View File

@@ -152,4 +152,6 @@ Specifically, **LMCache**, an efficient KV cache layer for enterprise-scale LLM
- **`--enable-lmcache`**: Using LMCache as an alternative hierarchical cache solution.
- **`--hicache-storage-backend-extra-config HICACHE_STORAGE_BACKEND_EXTRA_CONFIG`**: JSON string containing extra configuration for the storage backend, e.g., `--hicache-storage-backend-extra-config '{"prefetch_threshold":512, "prefetch_timeout_base": 0.5, "prefetch_timeout_per_ki_token": 0.25}' `
- **`--hicache-storage-backend-extra-config HICACHE_STORAGE_BACKEND_EXTRA_CONFIG`**: the extra config can be either
- a JSON string containing extra configuration for the storage backend, e.g., `--hicache-storage-backend-extra-config '{"prefetch_threshold":512, "prefetch_timeout_base": 0.5, "prefetch_timeout_per_ki_token": 0.25}' `, or
- a TOML or JSON or YAML file specifying the extra configuration for the storage backend (to differentiate from the JSON string input, prepend a `@` in front of the file name), e.g., `--hicache-storage-backend-extra-config "@config.toml"` where `config.toml` is the config file containing the complex configurations. This can be useful when the configuration consists of many or complex key-value pairs (for instance, it is preferred to use a config file for NIXL backend as its configurations can be complex).

2
docs/advanced_features/server_arguments.md
View File

@@ -353,7 +353,7 @@ Please consult the documentation below and [server_args.py](https://github.com/s
| `--hicache-mem-layout` | The layout of host memory pool for hierarchical cache. | `layer_first` | `layer_first`, `page_first`, `page_first_direct`, `page_first_kv_split`, `page_head` |
| `--hicache-storage-backend` | The storage backend for hierarchical KV cache. Built-in backends: file, mooncake, hf3fs, nixl, aibrix. For dynamic backend, use --hicache-storage-backend-extra-config to specify: backend_name (custom name), module_path (Python module path), class_name (backend class name). | `None` | `file`, `mooncake`, `hf3fs`, `nixl`, `aibrix`, `dynamic`, `eic` |
| `--hicache-storage-prefetch-policy` | Control when prefetching from the storage backend should stop. | `best_effort` | `best_effort`, `wait_complete`, `timeout` |
| `--hicache-storage-backend-extra-config` | A dictionary in JSON string format containing extra configuration for the storage backend. | `None` | Type: str |
| `--hicache-storage-backend-extra-config` | A dictionary in JSON string format, or a string starting with a `@` followed by a config file in JSON/YAML/TOML format, containing extra configuration for the storage backend. | `None` | Type: str |
## Hierarchical sparse attention
| Argument | Description | Defaults | Options |

26
python/sglang/srt/mem_cache/hiradix_cache.py
View File

@@ -3,6 +3,7 @@ from __future__ import annotations
import heapq
import json
import logging
import os
import threading
import time
from typing import TYPE_CHECKING, List, Optional
@@ -158,11 +159,32 @@ class HiRadixCache(RadixCache):
Returns:
tuple: (extra_config_dict, prefetch_threshold, prefetch_timeout_base, prefetch_timeout_per_ki_token, hicache_storage_pass_prefix_keys)
"""
# Parse extra config JSON if provided
# Parse extra config if provided. Extra config can be a JSON string or a json/toml/yaml file path prefixed with "@".
extra_config = {}
if storage_backend_extra_config:
try:
extra_config = json.loads(storage_backend_extra_config)
if storage_backend_extra_config.startswith("@"):
# Read config from a json/toml/yaml file
path = storage_backend_extra_config[1:]
ext = os.path.splitext(path)[1].lower()
with open(path, "rb" if ext == ".toml" else "r") as f:
if ext == ".json":
extra_config = json.load(f)
elif ext == ".toml":
import tomllib
extra_config = tomllib.load(f)
elif ext in (".yaml", ".yml"):
import yaml
extra_config = yaml.safe_load(f)
else:
raise ValueError(
f"Unsupported config file {path} (config format: {ext})"
)
else:
# read config from JSON string
extra_config = json.loads(storage_backend_extra_config)
except Exception as e:
logger.error(f"Invalid backend extra config JSON: {e}")
raise e

310
python/sglang/srt/mem_cache/storage/nixl/README.md
View File

@@ -33,23 +33,116 @@ The main storage connector that provides:
### NixlUtils
Consolidated utility classes:
- **NixlBackendSelection** - Handles backend selection and creation
- **NixlBackendConfig** - Handles backend configuration
- **NixlRegistration** - Manages memory registration for tensors, files and objects
- **NixlFileManager** - Handles file system operations and NIXL tuple creation
## Using NIXL for HiCache backend
When running the SGLang server, indicate `nixl` for `hicache-storage-backend` parameter, for instance:
## Using NIXL as the HiCache Storage Backend
### 1. How Backend Plugin Selection Works
The NIXL backend can support **multiple storage plugins** (e.g., POSIX, GDS, GDS_MT, 3FS, object store, etc).
* Each plugin has its own configuration section in the TOML file.
* A plugin is considered **usable** if:
* Its required library is available on the system (POSIX, GDS, GDS_MT are natively supported by NIXL).
* Its configuration is valid.
* It is marked as `active = true` in the configuration file (if applicable).
* Some plugins (e.g., 3FS, GDS) require additional system libraries or hardware support.
* NIXL selects the backend based on **internal priority and availability**, if neither a config file nor an in-command-line config string is provided.
If a plugin is configured but its dependencies are missing, it will be skipped.
### 2. Setting the Storage Directory (Optional)
For POSIX / GDS / GDS_MT file-based backends, the default storage location is `/tmp/hicache_storage`. However, you can customize where cached data is stored:
```bash
python3 -m sglang.launch_server --model-path <model> --host <ip> --port <port> --page-size 64 --enable-hierarchical-cache --hicache-ratio 2 --hicache-size 64 --hicache-write-policy write_through --hicache-storage-backend nixl
export SGLANG_HICACHE_NIXL_BACKEND_STORAGE_DIR=/path/to/storage/dir
```
To customize the base directory for files, you can set the following environment variable:
### 3. How to Provide Configuration for Backends
There are three ways to specify configurations for the backends: default config, file based config, and command-line (JSON string based) config.
#### 1. Using Default Configuration
To enable HiCache with the NIXL backend, start the SGLang server with:
```bash
export SGLANG_HICACHE_NIXL_BACKEND_STORAGE_DIR=/path/to/desired/dir
python3 -m sglang.launch_server \
--model-path <model> \
--host <ip> \
--port <port> \
--page-size 64 \
--enable-hierarchical-cache \
--hicache-ratio 2 \
--hicache-size 64 \
--hicache-write-policy write_through \
--hicache-storage-backend nixl
```
Selection of any storage backend like 3FS requires availability of that library on the system, and the backend is selected based on the priority mentioned above.
By default, NIXL will use its internal backend selection logic to choose an available storage plugin (and use default configs for the selected storage plugin).
#### 2. Using a Configuration File (Recommended)
For non-trivial setups with complex configurations, it is recommended to use a **TOML configuration file** to define which backend plugin to use and its configurations, via `--hicache-storage-backend-extra-config`:
Below is an example command (note: detailed configs are defined in the config file):
```bash
python3 -m sglang.launch_server \
--model-path <model> \
--host <ip> \
--port <port> \
--page-size 64 \
--enable-hierarchical-cache \
--hicache-ratio 2 \
--hicache-size 64 \
--hicache-write-policy write_through \
--hicache-storage-backend nixl \
--hicache-storage-backend-extra-config "@config.nixl.toml"
```
> **Important**
>
> * The `@` prefix tells SGLang to load the configuration from a file.
> * The file can be in **TOML format** (other formats, JSON / YAML, are also supported).
> * This is the preferred way to configure NIXL storage backends.
The structure of the config file is described in further details in [Configuration File Spec](#Configuration-File-Specification).
#### 3. Using Command-line JSON String
For debugging or quick testing, you may pass a **JSON-style string** directly via `--hicache-storage-backend-extra-config`.
This requires explicitly specifying the plugin type via an environment variable, and this method can be applicable to **only a few** plugins (e.g., POSIX, GDS, GDS_MT)
The below example shows how to use command-line string to use the POSIX plugin where URING is enabled for async POSIX storage.
```bash
export SGLANG_HICACHE_NIXL_BACKEND_PLUGIN_TYPE=POSIX
python3 -m sglang.launch_server \
--model-path <model> \
--host <ip> \
--port <port> \
--page-size 64 \
--enable-hierarchical-cache \
--hicache-ratio 2 \
--hicache-size 64 \
--hicache-write-policy write_through \
--hicache-storage-backend nixl \
--hicache-storage-backend-extra-config "{'use_uring': 'true'}"
```
⚠️ **Note**:
This method is convenient for testing / experimenting. For production or multi-plugin setups, it is always recommended to use the config file based approach.
## Running Unit Tests
@@ -167,6 +260,211 @@ The NIXL backend selection follows this priority order:
- Key-value based storage
The system automatically selects the best available backend, with POSIX as the default fallback.
## Configuration File Specification
This section defines the structure, supported sections, configuration keys, data types, defaults, and semantics for the NIXL HiCache backend configuration file (`config.nixl.toml`).
The configuration file is written in **TOML** and consists of multiple **plugin-specific sections** under the `plugin.*` namespace. Each section configures one storage backend plugin. Only one plugin should be enabled via setting `active = true` in the corresponding plugin-specific section.
An example of the configuration is provided in [`nixl.config.toml.sample`](./nixl.config.toml.sample).
### 1. General Structure
```toml
[plugin.<backend_name>]
<key> = <value>
```
* `<backend_name>` identifies the storage backend plugin.
* Each plugin is configured independently.
* Plugins are selected at runtime based on:
* Availability of required libraries/hardware
* Plugin configuration validity
* Internal backend priority rules
* Unless otherwise stated, all configuration keys are **optional** and have sensible defaults.
### 2. POSIX File System Backend (`plugin.posix`)
#### Section
```toml
[plugin.posix]
```
#### Description
Configures the POSIX file-system-based backend.
This backend supports multiple asynchronous I/O mechanisms and automatically selects the most performant option supported by the system.
**Backend priority (highest to lowest):**
1. Linux AIO
2. `io_uring`
3. POSIX AIO
#### Configuration Keys
| Key | Type | Default | Description |
| --------------- | ------- | --------- | -------------------------------------------------------------------------------------------------------- |
| `use_uring` | string | `"false"` | Enables Linux `io_uring` for asynchronous I/O when set to `"true"`. Recommended on modern Linux kernels. |
| `use_posix_aio` | string | `"false"` | Enables POSIX AIO as an alternative async I/O mechanism. |
| `use_aio` | string | `"false"` | Enables generic Linux AIO. |
| `active` | boolean | N/A | Controls whether this plugin is eligible for backend selection. |
**Notes**
* Boolean-like options use **string values** (`"true"` / `"false"`) for compatibility.
* **Only one backend** (i.e., only one of `use_uring`, `use_aio`, `use_posix_aio`) should be included in the config.
### 3. NVIDIA GPUDirect Storage Backend (`plugin.gds`)
#### Section
```toml
[plugin.gds]
```
#### Description
Configures NVIDIA GPUDirect Storage (GDS) backend.
This backend enables direct data transfers between storage and GPU memory.
**Requirements**
* NVIDIA GPU with GDS support
* Compatible NVIDIA driver and CUDA runtime
* Supported filesystem
#### Configuration Keys
| Key | Type | Default | Description |
| ------------------ | ------- | ------------------ | ------------------------------------------------------ |
| `batch_pool_size` | integer | `128` | Number of I/O requests maintained in the request pool. |
| `batch_limit` | integer | `128` | Maximum number of requests issued in a single batch. |
| `max_request_size` | integer | `16777216` (16 MB) | Maximum size (in bytes) of a single I/O request. |
| `active` | boolean | N/A | Controls whether this plugin is eligible for backend selection.|
### 4. Multi-Threaded GDS Backend (`plugin.gds_mt`)
#### Section
```toml
[plugin.gds_mt]
```
#### Description
Configures the multi-threaded variant of the NVIDIA GDS backend, allowing parallel request processing using multiple CPU threads.
#### Configuration Keys
| Key | Type | Default | Description |
| -------------- | ------- | ------- | ----------------------------------------------------- |
| `thread_count` | integer | `4` | Number of worker threads used to submit GDS requests. |
| `active` | boolean | N/A | Controls whether this plugin is eligible for backend selection. |
### 5. 3FS Backend (`plugin.3fs`)
#### Section
```toml
[plugin.3fs]
```
#### Description
Configures the 3FS (third-party filesystem) backend.
**Requirements**
* 3FS client library installed
* Filesystem mounted and accessible on the host
#### Configuration Keys
| Key | Type | Default | Description |
| ------------- | ------- | -------- | ---------------------------------- |
| `mount_point` | string | *none* | Mount point of the 3FS filesystem. |
| `mem_config` | string | `"dram"` | Memory configuration mode. |
| `iopool_size` | integer | `64` | Size of the I/O pool. |
| `active` | boolean | N/A | Controls whether this plugin is eligible for backend selection. |
##### `mem_config` Valid Values
| Value | Description |
| --------- | --------------------------------------------------- |
| `dram` | Use DRAM for buffering |
| `dram_zc` | Use DRAM with zero-copy support |
| `auto` | Automatically select based on platform capabilities |
##### `iopool_size` Constraints
* Valid range: **[2⁶, 2²⁰]**
* Values outside this range may cause initialization failure.
### 6. Object Storage Backend (`plugin.obj`)
#### Section
```toml
[plugin.obj]
```
#### Description
Configures an object storage backend compatible with S3 APIs (e.g., AWS S3, MinIO, Ceph).
#### Configuration Keys
| Key | Type | Default | Description |
| ------------------------ | ------- | ------------ | ---------------------------------------------- |
| `num_threads` | integer | `4` | Number of client worker threads. |
| `endpoint_override` | string | `""` | Custom endpoint URL (for non-AWS S3 services). |
| `scheme` | string | `"http"` | Connection scheme (`http` or `https`). |
| `region` | string | `""` | Cloud region (if applicable). |
| `req_checksum` | string | `"required"` | Request checksum behavior. |
| `ca_bundle` | string | `""` | Path to a custom CA bundle. |
| `access_key` | string | `""` | Access key credential. |
| `secrete_key` | string | `""` | Secret key credential. |
| `session_token` | string | `""` | Session token (optional). |
| `use_virtual_addressing` | string | `"true"` | Enables virtual-hosted-style addressing. |
| `bucket` | string | `""` | Default bucket name. |
| `active` | boolean | N/A | Controls whether this plugin is eligible for backend selection. |
##### `req_checksum` Valid Values
| Value | Description |
| ----------- | ---------------------------------------------- |
| `required` | Always include a checksum |
| `supported` | Include checksum when supported by the backend |
### 7. Notes and Best Practices
* All plugin sections are optional.
* Multiple plugins may be configured in a single file. However, it is recommended that **only one plugin** is configured `active = true`.
* Plugins whose dependencies are unavailable will be skipped.
* Use a TOML configuration file instead of inline JSON for:
* Multi-plugin setups
* Production deployments
* Clear validation and maintainability
## Note
This is v0 of the NIXL connector. Future versions will focus on further performance optimizations such as memory pre-registration (pre-allocating and registering memory buffers to reduce registration overhead during transfers) and block merging (combining related blocks as offsets within the same file to reduce file operations and improve throughput). These optimizations require changes at a higher layer, as the current HiCache API doesn't expose information like block relationships or hash patterns that would enable these optimizations.

17
python/sglang/srt/mem_cache/storage/nixl/hicache_nixl.py
View File

@@ -8,7 +8,12 @@ import torch
from sglang.srt.mem_cache.hicache_storage import HiCacheStorage, HiCacheStorageConfig
from .nixl_utils import NixlBackendSelection, NixlFileManager, NixlRegistration
from .nixl_utils import (
NixlBackendConfig,
NixlBackendSelection,
NixlFileManager,
NixlRegistration,
)
try:
from nixl._api import nixl_agent, nixl_agent_config
@@ -29,9 +34,15 @@ class HiCacheNixl(HiCacheStorage):
self,
storage_config: HiCacheStorageConfig,
file_path: str = "/tmp/hicache_storage",
plugin: str = "auto",
):
"""Initialize NIXL storage connector."""
# create nixlconfig from the --hicache-storage-backend-extra-config
nixlconfig = NixlBackendConfig(storage_config.extra_config)
# select the NIXL backend plugin from extra_config or environment variable
plugin = nixlconfig.get_specified_plugin()
# Might be better to be unified across HiCache backends and moved to HiCacheController
file_path = os.getenv("SGLANG_HICACHE_NIXL_BACKEND_STORAGE_DIR", file_path)
self.file_manager = (
@@ -57,7 +68,7 @@ class HiCacheNixl(HiCacheStorage):
self.agent_name = f"hicache_nixl_{str(uuid.uuid4())}"
self.agent = nixl_agent(self.agent_name, agent_config)
self.backend_selector = NixlBackendSelection(plugin)
self.backend_selector = NixlBackendSelection(plugin, nixlconfig)
if not self.backend_selector.create_backend(self.agent):
raise RuntimeError("Failed to create NIXL backend")

117
python/sglang/srt/mem_cache/storage/nixl/nixl.config.toml.sample Normal file
View File

@@ -0,0 +1,117 @@
################################################################################
# IMPORTANT
# 1. to enable a plugin, add "active = true" in the corresponding section
# 2. the configs inside plugin.posix (i.e., use_aio, use_uring, use_posix_aio)
# are mutually exclusive
################################################################################
########################################
# POSIX FILE SYSTEM BACKEND
########################################
[plugin.posix]
# Configuration for the POSIX file-based backend.
#
# The supported backends include:
# 1. AIO (`use_aio = "true"`)
# 2. io_uring (`use_uring = "true"`)
# 3. POSIX AIO (`use_posix_aio = "true"`)
#
# If not specified, NIXL will automatically detect and use available backends based on the following default priority: AIO > io_uring > POSIX AIO
# Enable Linux io_uring for async I/O (recommended if supported)
use_uring = "true"
# Enable POSIX AIO (alternative async I/O mechanism)
# use_posix_aio = "true"
# Enable generic AIO
# use_aio = "true"
# Whether this plugin is eligible for selection
active = true
########################################
# NVIDIA GDS (GPUDirect Storage) BACKEND
########################################
[plugin.gds]
# Configuration for NVIDIA GPUDirect Storage
# Requires compatible GPU, driver, and filesystem support
# Number of requests per batch, default: 128
batch_pool_size = 128
# Maximum number of requests issued at once, default: 128
batch_limit = 128
# Maximum size of a single request (bytes), default: 16MB
max_request_size = 16777216 # 16 MB
########################################
# MULTI-THREADED GDS BACKEND
########################################
[plugin.gds_mt]
# Multi-threaded variant of the GDS backend
# Number of worker threads, default: 4
thread_count = 4
########################################
# 3FS (THIRD-PARTY FILE SYSTEM) BACKEND
########################################
[plugin.3fs]
# Configuration for 3FS backend
# Requires the 3FS library to be installed and mounted
# Mount point of the 3FS filesystem
mount_point = "/mnt/3fs"
# Memory configuration mode:
# dram - use DRAM
# dram_zc - DRAM with zero-copy
# auto - let backend decide
mem_config = "dram"
# Size of the I/O pool
# Valid range: [2^6, 2^20]
iopool_size = 64
########################################
# OBJECT STORAGE BACKEND (S3 / COMPATIBLE)
########################################
[plugin.obj]
# Object storage backend (e.g., S3-compatible services)
# Number of client worker threads, default: 4
num_threads = 4
# Override endpoint (useful for non-AWS S3 services)
endpoint_override = ""
# Connection scheme: http or https, default: http
scheme = "http"
# Cloud region (if applicable)
region = ""
# Request checksum behavior:
# required, supported
req_checksum = "required"
# Custom CA bundle path (if needed)
ca_bundle = ""
# Credentials
access_key = ""
secrete_key = ""
session_token = ""
# Use virtual-hosted-style addressing (true/false), default: true
use_virtual_addressing = "true"
# Default bucket name
bucket = ""

91
python/sglang/srt/mem_cache/storage/nixl/nixl_utils.py
View File

@@ -7,6 +7,67 @@ import torch
logger = logging.getLogger(__name__)
class NixlBackendConfig:
"""Handles NIXL backend configurations"""
def __init__(self, config: Optional[dict[str, str]] = None):
"""Initialize backend configuration.
Args:
config: configurations in a dictionary. This config comes from --hicache-storage-backend-extra-config
config can be in two forms:
1. fully qualified form (for all plugins, some of them are enabled, others not):
{'plugin': { 'posix': {...}, 'gds': {...}, ...}}
2. flat form (for a specific selected plugin), assuming all params apply to a selected plugin
{'param1': 'value1', 'param2': 'value2', ...}
"""
self.config = config or {}
def get_specified_plugin(self) -> str:
"""decide which plugin to use: either config or SGLANG_HICACHE_NIXL_BACKEND_PLUGIN specifies the plugin, if not, use "auto" """
if "plugin" in self.config:
# fully qualified form: {'plugin': { 'posix': {...}, 'gds': {...}, ...}}
# choose the FIRST active plugin
for key, item in self.config["plugin"].items():
if item.get("active", False) in [True, "true", "True"]:
plugin = key.upper()
break
else:
# config is empty, or in flat form {'param1': 'value1', 'param2': 'value2', ...}
plugin = os.getenv("SGLANG_HICACHE_NIXL_BACKEND_PLUGIN", "auto")
return plugin
def get_backend_initparams(self, backend_name) -> dict:
"""Get initialization parameters from config of NIXL backend for backend creation.
Args:
backend_name: a specific backend's name (already converted "auto" into a specific backend name)
"""
initparams = {}
# config can be in two forms:
if "plugin" in self.config:
# fully qualified form: {'plugin': { 'posix': {...}, 'gds': {...}, ...}}
if backend_name.lower() in self.config["plugin"]:
config_data = self.config["plugin"][backend_name.lower()]
else:
logger.debug(
f"No specific config found for plugin {backend_name} in extra_config. Use default init params."
)
config_data = {}
else:
# flat form {'param1': 'value1', 'param2': 'value2', ...}
config_data = self.config
for key, value in config_data.items():
initparams[key] = value
return initparams
class NixlBackendSelection:
"""Handles NIXL backend selection and creation."""
@@ -15,7 +76,9 @@ class NixlBackendSelection:
# Priority order for File-based plugins in case of auto selection (add more as needed)
OBJ_PLUGINS = ["OBJ"] # Based on Amazon S3 SDK
def __init__(self, plugin: str = "auto"):
def __init__(
self, plugin: str = "auto", nixlconfig: Optional[NixlBackendConfig] = None
):
"""Initialize backend selection.
Args:
plugin: Plugin to use (default "auto" selects best available).
@@ -25,6 +88,7 @@ class NixlBackendSelection:
self.plugin = plugin
self.backend_name = None
self.mem_type = None
self.nixlconfig = nixlconfig
def set_bucket(self, bucket_name: str) -> None:
"""Set AWS bucket name in environment variable."""
@@ -60,17 +124,30 @@ class NixlBackendSelection:
)
return False
# obtain initparams for the backend from the NIXL config
initparams = (
self.nixlconfig.get_backend_initparams(self.backend_name)
if self.nixlconfig
else {}
)
# Create backend and set memory type
if self.backend_name in self.OBJ_PLUGINS:
if self.backend_name in self.OBJ_PLUGINS and "bucket" not in initparams:
bucket = os.environ.get("AWS_DEFAULT_BUCKET")
if not bucket:
logger.error(
"AWS_DEFAULT_BUCKET environment variable must be set for object storage"
)
return False
agent.create_backend(self.backend_name, {"bucket": bucket})
else:
agent.create_backend(self.backend_name)
initparams["bucket"] = bucket
# create backend using initialization parameters
agent.create_backend(self.backend_name, initparams)
logger.info(
f"NixlBackendSelection.create_backend: backend_name {self.backend_name} initparams {initparams} customParams {agent.get_backend_params(self.backend_name)} supported plugins {plugin_list}"
)
self.mem_type = "OBJ" if self.backend_name in self.OBJ_PLUGINS else "FILE"
logger.debug(
@@ -79,7 +156,9 @@ class NixlBackendSelection:
return True
except Exception as e:
logger.error(f"Failed to create NIXL backend: {e}")
logger.error(
f"Failed to create NIXL backend: {e}, backend_name {self.backend_name}, supported plugins {plugin_list} initparams {initparams}"
)
return False

2
python/sglang/srt/server_args.py
View File

@@ -4022,7 +4022,7 @@ class ServerArgs:
"--hicache-storage-backend-extra-config",
type=str,
default=ServerArgs.hicache_storage_backend_extra_config,
help="A dictionary in JSON string format containing extra configuration for the storage backend.",
help="A dictionary in JSON string format, or a string starting with a leading '@' and a config file in JSON/YAML/TOML format, containing extra configuration for the storage backend.",
)
# Hierarchical sparse attention