chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 15 Packages Projects Releases Wiki Activity

[NPU] Update NPU doc for model and features supported (#17385)

Browse Source
This commit is contained in:
Hexq0210
2026-01-20 12:28:06 +08:00
committed by GitHub
parent c560e14216
commit 1b192cf198
3 changed files with 107 additions and 115 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/platforms/ascend_npu_best_practice.md
+24 -30
View File
@@ -13,7 +13,7 @@ you encounter issues or have any questions, please [open an issue](https://githu
| Deepseek-R1 | Atlas 800I A3 | 32 | PD Separation | 3.9K-1K | W8A8 | [Optimal Configuration](#deepseek-r1-low-latency-20ms-2) |
| Deepseek-R1 | Atlas 800I A3 | 32 | PD Separation | 3.5K-1.5K | W8A8 | [Optimal Configuration](#deepseek-r1-low-latency-20ms-3) |
| Deepseek-R1 | Atlas 800I A3 | 32 | PD Separation | 3.5K-1K | W8A8 | [Optimal Configuration](#deepseek-r1-low-latency-20ms-4) |
| Deepseek-V3.2 | Atlas 800I A3 | 32 | PD Separation | 64K-1K | W8A8 | [Optimal Configuration](#deepseek-v32-low-latency-30ms) |
| Deepseek-V3.2 | Atlas 800I A3 | 32 | PD Separation | 64K-3K | W8A8 | [Optimal Configuration](#deepseek-v32-low-latency-30ms) |
### High Throughput
@@ -135,7 +135,7 @@ do
export SGLANG_ENABLE_SPEC_V2=1
export HCCL_BUFFSIZE=650
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=78
export TASK_QUEUE_ENABLE=0
export TASK_QUEUE_ENABLE=1
export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1
export HCCL_SOCKET_IFNAME=xxx
export GLOO_SOCKET_IFNAME=xxx
@@ -170,7 +170,7 @@ python -m sglang_router.launch_router \
#### Benchmark
```shell
python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6688 --max-concurrency 832 --random-input-len 3500 --random-output-len 1500 --num-prompts 3328 --random-range-ratio 1
python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6688 --max-concurrency 768 --random-input-len 3500 --random-output-len 1500 --num-prompts 3072 --random-range-ratio 1 --request-rate 16
```
### DeepSeek R1 Low Latency 20ms 1
@@ -247,13 +247,13 @@ do
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export HCCL_BUFFSIZE=650
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=4
export TASK_QUEUE_ENABLE=0
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=12
export TASK_QUEUE_ENABLE=1
export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1
export HCCL_SOCKET_IFNAME=xxx
export GLOO_SOCKET_IFNAME=xxx
python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode decode --host ${D_IP[$i]} \
--port 8001 --trust-remote-code --dist-init-addr DIP1:5000 --nnodes 2 --node-rank $i --tp-size 32 --dp-size 8 \
--port 8001 --trust-remote-code --dist-init-addr DIP1:5000 --nnodes 2 --node-rank $i --tp-size 32 --dp-size 16 \
--mem-fraction-static 0.75 --max-running-requests 32 --attention-backend ascend --device npu --quantization modelslim \
--moe-a2a-backend deepep --enable-dp-attention --deepep-mode low_latency --enable-dp-lm-head --moe-dense-tp 1 \
--cuda-graph-bs 2 4 6 --disaggregation-transfer-backend ascend --watchdog-timeout 9000 --context-length 8192 \
@@ -305,7 +305,7 @@ Please Turn to [DeepSeek R1 Low Latency 20ms](#deepSeek-r1-low-latency-20ms-1)
#### Benchmark
```bash
python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6688 --max-concurrency 32 --random-input-len 3900 --random-output-len 1000 --num-prompts 32 --random-range-ratio 1
python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6688 --max-concurrency 768 --random-input-len 3900 --random-output-len 1000 --num-prompts 768 --random-range-ratio 1 --request-rate 16
```
### DeepSeek R1 Low Latency 20ms 3
@@ -327,7 +327,7 @@ Please Turn to [DeepSeek R1 Low Latency 20ms](#deepSeek-r1-low-latency-20ms-1)
#### Benchmark
```bash
python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6688 --max-concurrency 32 --random-input-len 3500 --random-output-len 1500 --num-prompts 32 --random-range-ratio 1
python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6688 --max-concurrency 768 --random-input-len 3500 --random-output-len 1500 --num-prompts 768 --random-range-ratio 1 --request-rate 16
```
### DeepSeek R1 Low Latency 20ms 4
@@ -349,7 +349,7 @@ Please Turn to [DeepSeek R1 Low Latency 20ms](#deepSeek-r1-low-latency-20ms-1)
#### Benchmark
```bash
python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6688 --max-concurrency 32 --random-input-len 3500 --random-output-len 1000 --num-prompts 32 --random-range-ratio 1
python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6688 --max-concurrency 768 --random-input-len 3500 --random-output-len 1000 --num-prompts 768 --random-range-ratio 1 --request-rate 16
```
### DeepSeek R1 High Performance 50ms 2
@@ -513,7 +513,7 @@ do
export SGLANG_ENABLE_SPEC_V2=1
export HCCL_BUFFSIZE=720
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=96
export TASK_QUEUE_ENABLE=0
export TASK_QUEUE_ENABLE=1
export HCCL_SOCKET_IFNAME=xxx
export GLOO_SOCKET_IFNAME=xxx
python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode decode --host ${D_IP[$i]} \
@@ -706,7 +706,7 @@ do
export SGLANG_ENABLE_SPEC_V2=1
export HCCL_BUFFSIZE=720
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=96
export TASK_QUEUE_ENABLE=0
export TASK_QUEUE_ENABLE=1
export HCCL_SOCKET_IFNAME=xxx
export GLOO_SOCKET_IFNAME=xxx
python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode decode --host ${D_IP[$i]} \
@@ -750,7 +750,7 @@ Hardware: Atlas 800I A3 32Card
DeployMode: PD Separation
DataSets: 64K1K
DataSets: 64K3K
TPOT: 30ms
@@ -947,7 +947,7 @@ python -m sglang_router.launch_router \
#### Benchmark
```shell
python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6688 --max-concurrency 32 --random-input-len 64000 --random-output-len 1000 --num-prompts 64 --random-range-ratio 1 --request-rate 0.25
python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6688 --max-concurrency 32 --random-input-len 64000 --random-output-len 3000 --num-prompts 64 --random-range-ratio 1
```
### Qwen3 235B High Throughput 50ms 1
@@ -1019,7 +1019,6 @@ do
--host ${P_IP[$i]} --port 8000 --disaggregation-bootstrap-port 8995 --trust-remote-code \
--nnodes 1 --node-rank $i --tp-size 16 --dp-size 16 --mem-fraction-static 0.6 \
--disable-radix-cache \
--ep-dispatch-algorithm static --init-expert-location /path/to/expert_distribution_recorder.pt \
--attention-backend ascend --device npu --quantization modelslim --disaggregation-transfer-backend ascend \
--speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \
--speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \
@@ -1080,7 +1079,7 @@ python -m sglang_router.launch_router \
#### Benchmark
```shell
python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7239 --max-concurrency 768 --random-input-len 3500 --random-output-len 1500 --num-prompts 3072 --random-range-ratio 1
python -m sglang.bench_serving --dataset-name random --backend sglang-oai --host 127.0.0.1 --port 7239 --max-concurrency 860 --random-input-len 3500 --random-output-len 1500 --num-prompts 3440 --random-range-ratio 1
```
### Qwen3 235B High Throughput 50ms 2
@@ -1132,7 +1131,7 @@ export GLOO_SOCKET_IFNAME=lo
export HCCL_OP_EXPANSION_MODE="AIV"
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=2
python -m sglang.launch_server --model-path $MODEL_PATH \
--host 127.0.0.1 --port 7439 --trust-remote-code --nnodes 1 --node-rank 0 \
@@ -1149,7 +1148,7 @@ python -m sglang.launch_server --model-path $MODEL_PATH \
#### Benchmark
```shell
python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7439 --max-concurrency 288 --random-input-len 3500 --random-output-len 1500 --num-prompts 1088 --random-range-ratio 1
python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7439 --max-concurrency 272 --random-input-len 3500 --random-output-len 1500 --num-prompts 1088 --random-range-ratio 1
```
### Qwen3-235B Atlas 800I A3-8Card PD Mixed 2K-2K 100ms
@@ -1270,7 +1269,6 @@ export GLOO_SOCKET_IFNAME=xxx
export HCCL_OP_EXPANSION_MODE="AIV"
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
python -m sglang.launch_server --model-path $MODEL_PATH \
--host 127.0.0.1 --port 7439 --trust-remote-code --nnodes 1 --node-rank 0 \
@@ -1322,8 +1320,6 @@ source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=16
MODEL_PATH=xxx
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
@@ -1348,17 +1344,16 @@ do
echo "${MIX_IP[$i]}"
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
python -m sglang.launch_server --model-path ${MODEL_PATH} \
--host 127.0.0.1 --port 7439 --trust-remote-code \
--nnodes 2 --node-rank $i --tp-size 32 --dp-size 32 --mem-fraction-static 0.8 --max-running-requests 768 \
--attention-backend ascend --device npu --quantization modelslim --enable-dp-attention \
--moe-a2a-backend deepep --deepep-mode auto --cuda-graph-bs 6 8 10 12 18 24 \
--dist-init-addr ${MIX_IP[0]}:5000 --chunked-prefill-size 32768 --max-prefill-tokens 458880 \
--speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \
--dist-init-addr ${MIX_IP[0]}:5000 --chunked-prefill-size 131072 --max-prefill-tokens 458880 \
--speculative-algorithm EAGLE3 --speculative-draft-model-path xxx --speculative-draft-model-quantization= unquant \
--speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \
--watchdog-timeout 9000 --context-length 8192 \
--context-length 8192 --disable-radix-cache \
--enable-dp-lm-head --dtype bfloat16
NODE_RANK=$i
break
@@ -1493,10 +1488,10 @@ export SGLANG_ENABLE_SPEC_V2=1
python -m sglang.launch_server --model-path $MODEL_PATH \
--host 127.0.0.1 --port 7439 --trust-remote-code --nnodes 1 --node-rank 0 \
--attention-backend ascend --device npu --quantization modelslim \
--attention-backend ascend --device npu \
--max-running-requests 32 \
--disable-radix-cache \
--chunked-prefill-size 32768 --max-prefill-tokens 65536 --speculative-draft-model-quantization unquant \
--chunked-prefill-size 24576 --max-prefill-tokens 65536 \
--speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \
--speculative-num-steps 4 --speculative-eagle-topk 1 --speculative-num-draft-tokens 5 \
--tp-size 8 --mem-fraction-static 0.72 --cuda-graph-bs 8 16 24 32 --dtype bfloat16
@@ -1624,13 +1619,12 @@ export GLOO_SOCKET_IFNAME=lo
export HCCL_OP_EXPANSION_MODE="AIV"
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export DISABLE_EAGLE3_QUANT=1
python -m sglang.launch_server --model-path $MODEL_PATH \
--host 127.0.0.1 --port 7339 --trust-remote-code --nnodes 1 --node-rank 0 \
--attention-backend ascend --device npu \
--max-running-requests 1 \
--disable-radix-cache \
--disable-radix-cache --speculative-draft-model-quantization unquant \
--speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \
--speculative-num-steps 4 --speculative-eagle-topk 1 --speculative-num-draft-tokens 5 \
--chunked-prefill-size -1 --max-prefill-tokens 65536 \
@@ -1766,7 +1760,7 @@ python -m sglang.launch_server --model-path $MODEL_PATH \
--disable-radix-cache --speculative-draft-model-quantization unquant \
--speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \
--speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \
--chunked-prefill-size 65536 --max-prefill-tokens 49152 \
--chunked-prefill-size -1 --max-prefill-tokens 49152 \
--tp-size 4 --mem-fraction-static 0.7 --cuda-graph-bs 54 60 66 72 78 84 90 108 114 120 --dtype bfloat16
```
@@ -2105,7 +2099,7 @@ python -m sglang.launch_server --model-path $MODEL_PATH \
#### Benchmark
```shell
python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7439 --max-concurrency 80 --random-input-len 3500 --random-output-len 1500 --num-prompts 320 --random-range-ratio 1 --request-rate 8
python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7439 --max-concurrency 80 --random-input-len 3500 --random-output-len 1500 --num-prompts 320 --random-range-ratio 1
```
### Qwen3 Next High Throughput 50ms
docs/platforms/ascend_npu_support_features.md
+75 -77
View File
@@ -21,7 +21,7 @@ click [Server Arguments](https://docs.sglang.io/advanced_features/server_argumen
| `--context-length` | `None` | Type: int | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--is-embedding` | `False` | bool flag (set to enable) | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--enable-multimodal` | `None` | bool flag (set to enable) | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--revision` | `None` | Type: str | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--revision` | `None` | Type: str | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--model-impl` | `auto` | `auto`, `sglang`,<br/> `transformers` | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
## HTTP server
@@ -33,8 +33,8 @@ click [Server Arguments](https://docs.sglang.io/advanced_features/server_argumen
| `--skip-server-warmup` | `False` | bool flag <br/>(set to enable) | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--warmups` | `None` | Type: str | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--nccl-port` | `None` | Type: int | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--fastapi-root-path` | `None` | Type: str | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--grpc-mode` | `False` | bool flag <br/>(set to enable) | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--fastapi-root-path` | `None` | Type: str | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--grpc-mode` | `False` | bool flag <br/>(set to enable) | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--checkpoint-engine-` <br/> `wait-weights-` <br/> `before-ready` | `False` | bool flag <br/>(set to enable) | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
## Quantization and data type
@@ -45,7 +45,7 @@ click [Server Arguments](https://docs.sglang.io/advanced_features/server_argumen
| `--quantization` | `None` | `modelslim` | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--quantization-param-path` | `None` | Type: str | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--kv-cache-dtype` | `auto` | `auto` | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--enable-fp32-lm-head` | `False` | bool flag <br/> (set to enable) | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--enable-fp32-lm-head` | `False` | bool flag <br/> (set to enable) | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--modelopt-quant` | `None` | Type: str | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--modelopt-checkpoint-`<br/>`restore-path` | `None` | Type: str | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--modelopt-checkpoint-`<br/>`save-path` | `None` | Type: str | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
@@ -73,7 +73,7 @@ click [Server Arguments](https://docs.sglang.io/advanced_features/server_argumen
| `--hybrid-kvcache-ratio` | `None` | Optional[float] | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--swa-full-tokens-ratio` | `0.8` | Type: float | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--disable-hybrid-swa-memory` | `False` | bool flag<br/> (set to enable) | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--abort-on-priority-`<br/>`when-disabled` | `False` | bool flag<br/> (set to enable) | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--abort-on-priority-`<br/>`when-disabled` | `False` | bool flag<br/> (set to enable) | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--enable-dynamic-chunking` | `False` | bool flag<br/> (set to enable) | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
## Runtime options
@@ -88,15 +88,15 @@ click [Server Arguments](https://docs.sglang.io/advanced_features/server_argumen
| `--stream-interval` | `1` | Type: int | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--stream-output` | `False` | bool flag (set to enable) | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--random-seed` | `None` | Type: int | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--constrained-json-`<br/>`whitespace-pattern` | `None` | Type: str | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--constrained-json-`<br/>`disable-any-whitespace` | `False` | bool flag (set to enable) | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--constrained-json-`<br/>`whitespace-pattern` | `None` | Type: str | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--constrained-json-`<br/>`disable-any-whitespace` | `False` | bool flag (set to enable) | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--watchdog-timeout` | `300` | Type: float | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--soft-watchdog-timeout` | `300` | Type: float | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--dist-timeout` | `None` | Type: int | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--base-gpu-id` | `0` | Type: int | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--gpu-id-step` | `1` | Type: int | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--sleep-on-idle` | `False` | bool flag (set to enable) | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--custom-sigquit-handler` | `None` | Optional[Callable] | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--custom-sigquit-handler` | `None` | Optional[Callable] | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
## Logging
@@ -106,32 +106,32 @@ click [Server Arguments](https://docs.sglang.io/advanced_features/server_argumen
| `--log-level-http` | `None` | Type: str | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--log-requests` | `False` | bool flag<br/> (set to enable) | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--log-requests-level` | `2` | `0`, `1`, `2`, `3` | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--log-requests-format` | text | text, json | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--log-requests-format` | text | text, json | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--crash-dump-folder` | `None` | Type: str | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--crash-on-nan` | `False` | Type: str | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--enable-metrics` | `False` | bool flag<br/> (set to enable) | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--enable-metrics-for-`<br/>`all-schedulers` | `False` | bool flag<br/> (set to enable) | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--tokenizer-metrics-`<br/>`custom-labels-header` | `x-custom-labels` | Type: str | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--tokenizer-metrics-`<br/>`allowed-custom-labels` | `None` | List[str] | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--bucket-time-to-`<br/>`first-token` | `None` | List[float] | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--bucket-inter-token-`<br/>`latency` | `None` | List[float] | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--bucket-e2e-request-`<br/>`latency` | `None` | List[float] | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--collect-tokens-`<br/>`histogram` | `False` | bool flag<br/> (set to enable) | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--prompt-tokens-buckets` | `None` | List[str] | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--generation-tokens-buckets` | `None` | List[str] | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--gc-warning-threshold-secs` | `0.0` | Type: float | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--enable-metrics` | `False` | bool flag<br/> (set to enable) | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--enable-metrics-for-`<br/>`all-schedulers` | `False` | bool flag<br/> (set to enable) | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--tokenizer-metrics-`<br/>`custom-labels-header` | `x-custom-labels` | Type: str | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--tokenizer-metrics-`<br/>`allowed-custom-labels` | `None` | List[str] | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--bucket-time-to-`<br/>`first-token` | `None` | List[float] | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--bucket-inter-token-`<br/>`latency` | `None` | List[float] | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--bucket-e2e-request-`<br/>`latency` | `None` | List[float] | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--collect-tokens-`<br/>`histogram` | `False` | bool flag<br/> (set to enable) | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--prompt-tokens-buckets` | `None` | List[str] | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--generation-tokens-buckets` | `None` | List[str] | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--gc-warning-threshold-secs` | `0.0` | Type: float | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--decode-log-interval` | `40` | Type: int | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--enable-request-time-`<br/>`stats-logging` | `False` | bool flag<br/> (set to enable) | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--kv-events-config` | `None` | Type: str | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--enable-trace` | `False` | bool flag<br/> (set to enable) | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--oltp-traces-endpoint` | `localhost:4317` | Type: str | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--enable-trace` | `False` | bool flag<br/> (set to enable) | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--oltp-traces-endpoint` | `localhost:4317` | Type: str | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
## RequestMetricsExporter configuration
| Argument | Defaults | Options | A2 | A3 |
|---------------------------------------|----------|--------------------------------|:--------------------------------------:|:--------------------------------------:|
| `--export-metrics-to-`<br/>`file` | `False` | bool flag<br/> (set to enable) | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--export-metrics-to-`<br/>`file-dir` | `None` | Type: str | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| Argument | Defaults | Options | A2 | A3 |
|---------------------------------------|----------|--------------------------------|:----------------------------------------:|:----------------------------------------:|
| `--export-metrics-to-`<br/>`file` | `False` | bool flag<br/> (set to enable) | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--export-metrics-to-`<br/>`file-dir` | `None` | Type: str | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
## API related
@@ -146,16 +146,16 @@ click [Server Arguments](https://docs.sglang.io/advanced_features/server_argumen
| `--enable-cache-report` | `False` | bool flag<br/> (set to enable) | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--reasoning-parser` | `None` | `deepseek-r1` | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--tool-call-parser` | `None` | `llama`,`pythonic` | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--sampling-defaults` | `model` | `openai`, `model` | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--sampling-defaults` | `model` | `openai`, `model` | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--tool-server` | `None` | Type: str | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
## Data parallelism
| Argument | Defaults | Options | A2 | A3 |
|----------------------------------------|---------------|-------------------------------------------------------------|:----------------------------------------:|:----------------------------------------:|
| `--data-parallel-size`<br/>`--dp-size` | `1` | Type: int | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| Argument | Defaults | Options | A2 | A3 |
|----------------------------------------|---------------|-----------------------------------------------------------|:----------------------------------------:|:----------------------------------------:|
| `--data-parallel-size`<br/>`--dp-size` | `1` | Type: int | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--load-balance-method` | `round_robin` | `round_robin`,<br/> `total_requests`,<br/> `total_tokens` | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--prefill-round-robin-balance` | `False` | bool flag<br/> (set to enable) | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--prefill-round-robin-balance` | `False` | bool flag<br/> (set to enable) | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
## Multi-node distributed serving
@@ -208,13 +208,13 @@ click [Server Arguments](https://docs.sglang.io/advanced_features/server_argumen
| `--speculative-algorithm` | `None` | `EAGLE3`,<br/> `NEXTN` | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--speculative-draft-model-path`<br/>`--speculative-draft-model` | `None` | Type: str | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--speculative-draft-model-`<br/>`revision` | `None` | Type: str | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--speculative-draft-load-format` | `None` | `auto` | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--speculative-draft-load-format` | `None` | `auto` | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--speculative-num-steps` | `None` | Type: int | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--speculative-eagle-topk` | `None` | Type: int | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--speculative-num-draft-tokens` | `None` | Type: int | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--speculative-accept-`<br/>`threshold-single` | `1.0` | Type: float | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--speculative-accept-`<br/>`threshold-acc` | `1.0` | Type: float | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--speculative-token-map` | `None` | Type: str | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--speculative-token-map` | `None` | Type: str | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--speculative-attention-`<br/>`mode` | `prefill` | `prefill`,<br/> `decode` | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--speculative-moe-runner-`<br/>`backend` | `None` | `auto` | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--speculative-moe-a2a-`<br/>`backend` | `None` | `ascend_fuseep` | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
@@ -235,28 +235,28 @@ click [Server Arguments](https://docs.sglang.io/advanced_features/server_argumen
## Expert parallelism
| Argument | Defaults | Options | A2 | A3 |
|-------------------------------------------------------|-----------|---------------------------------------------|:----------------------------------------:|:----------------------------------------:|
| `--expert-parallel-size`<br/>`--ep-size`<br/>`--ep` | `1` | Type: int | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--moe-a2a-backend` | `none` | `none`,<br/> `deepep`,<br/> `ascend_fuseep` | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--moe-runner-backend` | `auto` | `auto`, `triton` | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--flashinfer-mxfp4-`<br/>`moe-precision` | `default` | `default`,<br/> `bf16` | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--enable-flashinfer-`<br/>`allreduce-fusion` | `False` | bool flag<br/> (set to enable) | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--deepep-mode` | `auto` | `normal`, <br/>`low_latency`,<br/> `auto` | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--deepep-config` | `None` | Type: str | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--ep-num-redundant-experts` | `0` | Type: int | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--ep-dispatch-algorithm` | `None` | Type: str | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--init-expert-location` | `trivial` | Type: str | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--enable-eplb` | `False` | bool flag<br/> (set to enable) | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--eplb-algorithm` | `auto` | Type: str | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--eplb-rebalance-layers-`<br/>`per-chunk` | `None` | Type: int | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--eplb-min-rebalancing-`<br/>`utilization-threshold` | `1.0` | Type: float | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--expert-distribution-`<br/>`recorder-mode` | `None` | Type: str | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--expert-distribution-`<br/>`recorder-buffer-size` | `None` | Type: int | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--enable-expert-distribution-`<br/>`metrics` | `False` | bool flag <br/>(set to enable) | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--moe-dense-tp-size` | `None` | Type: int | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--elastic-ep-backend` | `None` | `none`, `mooncake` | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--mooncake-ib-device` | `None` | Type: str | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| Argument | Defaults | Options | A2 | A3 |
|-------------------------------------------------------|-----------|---------------------------------------------|:-----------------------------------------:|:----------------------------------------:|
| `--expert-parallel-size`<br/>`--ep-size`<br/>`--ep` | `1` | Type: int | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--moe-a2a-backend` | `none` | `none`,<br/> `deepep`,<br/> `ascend_fuseep` | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--moe-runner-backend` | `auto` | `auto`, `triton` | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--flashinfer-mxfp4-`<br/>`moe-precision` | `default` | `default`,<br/> `bf16` | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--enable-flashinfer-`<br/>`allreduce-fusion` | `False` | bool flag<br/> (set to enable) | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--deepep-mode` | `auto` | `normal`, <br/>`low_latency`,<br/> `auto` | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--deepep-config` | `None` | Type: str | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--ep-num-redundant-experts` | `0` | Type: int | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--ep-dispatch-algorithm` | `None` | Type: str | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--init-expert-location` | `trivial` | Type: str | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--enable-eplb` | `False` | bool flag<br/> (set to enable) | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--eplb-algorithm` | `auto` | Type: str | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--eplb-rebalance-layers-`<br/>`per-chunk` | `None` | Type: int | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--eplb-min-rebalancing-`<br/>`utilization-threshold` | `1.0` | Type: float | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--expert-distribution-`<br/>`recorder-mode` | `None` | Type: str | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--expert-distribution-`<br/>`recorder-buffer-size` | `None` | Type: int | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--enable-expert-distribution-`<br/>`metrics` | `False` | bool flag <br/>(set to enable) | ***<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--moe-dense-tp-size` | `None` | Type: int | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--elastic-ep-backend` | `None` | `none`, `mooncake` | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--mooncake-ib-device` | `None` | Type: str | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
## Mamba Cache
@@ -342,7 +342,7 @@ click [Server Arguments](https://docs.sglang.io/advanced_features/server_argumen
| `--enable-symm-mem` | `False` | bool flag<br/> (set to enable) | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--disable-flashinfer-`<br/>`cutlass-moe-fp4-allgather` | `False` | bool flag<br/> (set to enable) | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--enable-tokenizer-`<br/>`batch-encode` | `False` | bool flag<br/> (set to enable) | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--disable-tokenizer-`<br/>`batch-encode` | `False` | bool flag<br/> (set to enable) | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--disable-tokenizer-`<br/>`batch-encode` | `False` | bool flag<br/> (set to enable) | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--disable-outlines-`<br/>`disk-cache` | `False` | bool flag<br/> (set to enable) | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--disable-custom-`<br/>`all-reduce` | `False` | bool flag<br/> (set to enable) | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--enable-mscclpp` | `False` | bool flag<br/> (set to enable) | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
@@ -355,7 +355,7 @@ click [Server Arguments](https://docs.sglang.io/advanced_features/server_argumen
| `--enable-single-`<br/>`batch-overlap` | `False` | bool flag<br/> (set to enable) | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--tbo-token-`<br/>`distribution-threshold` | `0.48` | Type: float | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--enable-torch-`<br/>`compile` | `False` | bool flag<br/> (set to enable) | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--enable-torch-`<br/>`compile-debug-mode` | `False` | bool flag<br/> (set to enable) | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--enable-torch-`<br/>`compile-debug-mode` | `False` | bool flag<br/> (set to enable) | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--enable-piecewise-`<br/>`cuda-graph` | `False` | bool flag<br/> (set to enable) | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--piecewise-cuda-`<br/>`graph-tokens` | `None` | Type: JSON<br/> list | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--piecewise-cuda-`<br/>`graph-compiler` | `eager` | ["eager", "inductor"] | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
@@ -380,13 +380,13 @@ click [Server Arguments](https://docs.sglang.io/advanced_features/server_argumen
| `--disable-fast-`<br/>`image-processor` | `False` | bool flag<br/> (set to enable) | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--keep-mm-feature-`<br/>`on-device` | `False` | bool flag<br/> (set to enable) | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--enable-return-`<br/>`hidden-states` | `False` | bool flag<br/> (set to enable) | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--enable-return-`<br/>`routed-experts` | `False` | bool flag<br/> (set to enable) | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--enable-return-`<br/>`routed-experts` | `False` | bool flag<br/> (set to enable) | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--scheduler-recv-`<br/>`interval` | `1` | Type: int | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--numa-node` | `None` | List[int] | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--rl-on-policy-target` | `None` | `fsdp` | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--enable-layerwise-`<br/>`nvtx-marker` | `False` | bool flag<br/> (set to enable) | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--enable-attn-tp-`<br/>`input-scattered` | `False` | bool flag<br/> (set to enable) | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--enable-nsa-prefill-`<br/>`context-parallel` | `False` | bool flag<br/> (set to enable) | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--enable-nsa-prefill-`<br/>`context-parallel` | `False` | bool flag<br/> (set to enable) | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--enable-fused-qk-`<br/>`norm-rope` | `False` | bool flag<br/> (set to enable) | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
## Dynamic batch tokenizer
@@ -401,10 +401,9 @@ click [Server Arguments](https://docs.sglang.io/advanced_features/server_argumen
| Argument | Defaults | Options | A2 | A3 |
|--------------------------------------------|----------|-----------|:----------------------------------------:|:----------------------------------------:|
| `--debug-tensor-dump-`<br/>`output-folder` | `None` | Type: str | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--debug-tensor-dump-`<br/>`layers` | `None` | List[int] | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--debug-tensor-dump-`<br/>`output-folder` | `None` | Type: str | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--debug-tensor-dump-`<br/>`layers` | `None` | List[int] | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--debug-tensor-dump-`<br/>`input-file` | `None` | Type: str | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--debug-tensor-dump-`<br/>`inject` | `False` | Type: str | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
## PD disaggregation
@@ -415,21 +414,20 @@ click [Server Arguments](https://docs.sglang.io/advanced_features/server_argumen
| `--disaggregation-bootstrap-port` | `8998` | Type: int | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--disaggregation-decode-tp` | `None` | Type: int | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--disaggregation-decode-dp` | `None` | Type: int | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--disaggregation-prefill-pp` | `1` | Type: int | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--disaggregation-ib-device` | `None` | Type: str | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--disaggregation-decode-`<br/>`enable-offload-kvcache` | `False` | bool flag<br/> (set to enable) | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--disaggregation-decode-`<br/>`enable-fake-auto` | `False` | bool flag<br/> (set to enable) | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--disaggregation-decode-`<br/>`enable-offload-kvcache` | `False` | bool flag<br/> (set to enable) | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--disaggregation-decode-`<br/>`enable-fake-auto` | `False` | bool flag<br/> (set to enable) | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--num-reserved-decode-tokens` | `512` | Type: int | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--disaggregation-decode-`<br/>`polling-interval` | `1` | Type: int | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
## Encode prefill disaggregation
| Argument | Defaults | Options | A2 | A3 |
|------------------------------|--------------------|----------------------------------------------------------------|:--------------------------------------:|:--------------------------------------:|
| `--encoder-only` | `False` | bool flag<br/> (set to enable) | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--language-only` | `False` | bool flag<br/> (set to enable) | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--encoder-transfer-backend` | `zmq_to_scheduler` | `zmq_to_scheduler`, <br/> `zmq_to_tokenizer`,<br/> `mooncake` | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| `--encoder-urls` | `[]` | List[str] | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| Argument | Defaults | Options | A2 | A3 |
|------------------------------|--------------------|----------------------------------------------------------------|:----------------------------------------:|:----------------------------------------:|
| `--encoder-only` | `False` | bool flag<br/> (set to enable) | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--language-only` | `False` | bool flag<br/> (set to enable) | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--encoder-transfer-backend` | `zmq_to_scheduler` | `zmq_to_scheduler`, <br/> `zmq_to_tokenizer`,<br/> `mooncake` | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| `--encoder-urls` | `[]` | List[str] | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
## Custom weight loader
@@ -478,12 +476,12 @@ click [Server Arguments](https://docs.sglang.io/advanced_features/server_argumen
## For registering hooks
| Argument | Defaults | Options | A2 | A3 |
|-------------------|----------|-----------------|----------------------------------------|----------------------------------------|
| `--forward-hooks` | `None` | Type: JSON list | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| Argument | Defaults | Options | A2 | A3 |
|-------------------|----------|-----------------|------------------------------------------|------------------------------------------|
| `--forward-hooks` | `None` | Type: JSON list | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
## Configuration file support
| Argument | Defaults | Options | A2 | A3 |
|------------|----------|-----------|----------------------------------------|----------------------------------------|
| `--config` | `None` | Type: str | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| Argument | Defaults | Options | A2 | A3 |
|------------|----------|-----------|------------------------------------------|------------------------------------------|
| `--config` | `None` | Type: str | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
docs/platforms/ascend_npu_support_models.md
+8 -8
View File
@@ -46,12 +46,12 @@ You are welcome to enable various models based on your business requirements.
| baichuan-inc/Baichuan2-13B-Chat | Baichuan 2 (7B, 13B) | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| baidu/ERNIE-4.5-21B-A3B-PT | ERNIE-4.5 (4.5, 4.5MoE series) | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| openbmb/MiniCPM3-4B | MiniCPM (v3, 4B) | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| openai/gpt-oss-120b | GPTOSS | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| openai/gpt-oss-120b | GPTOSS | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
## Multimodal Language Models
| Models | Model Family (Variants) | A2 Supported | A3 Supported |
|-----------------------------------------------|---------------------------|------------------------------------------|:----------------------------------------:|
| Models | Model Family (Variants) | A2 Supported | A3 Supported |
|-----------------------------------------------|---------------------------|:----------------------------------------:|:----------------------------------------:|
| Qwen/Qwen2.5-VL-3B-Instruct | Qwen-VL | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| Qwen/Qwen2.5-VL-72B-Instruct | Qwen-VL | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| Qwen/Qwen3-VL-30B-A3B-Instruct | Qwen-VL | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
@@ -70,18 +70,18 @@ You are welcome to enable various models based on your business requirements.
| lmms-lab/llava-onevision-qwen2-7b-ov | LLaVA-OneVision | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| Kimi/Kimi-VL-A3B-Instruct | Kimi-VL (A3B) | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| ZhipuAI/GLM-4.5V | GLM-4.5V (106B) | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| meta-llama/Llama-3.2-11B-Vision-Instruct | Llama 3.2 Vision (11B) | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| meta-llama/Llama-3.2-11B-Vision-Instruct | Llama 3.2 Vision (11B) | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
## Embedding Models
| Models | Model Family | A2 Supported | A3 Supported |
|-------------------------------------------|--------------------------|------------------------------------------|:----------------------------------------:|
| Models | Model Family | A2 Supported | A3 Supported |
|-------------------------------------------|--------------------------|:----------------------------------------:|:----------------------------------------:|
| intfloat/e5-mistral-7b-instruct | E5 (Llama/Mistral based) | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| iic/gte_Qwen2-1.5B-instruct | GTE-Qwen2 | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| Qwen/Qwen3-Embedding-8B | Qwen3-Embedding | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| Alibaba-NLP/gme-Qwen2-VL-2B-Instruct | GME (Multimodal) | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| AI-ModelScope/clip-vit-large-patch14-336 | CLIP | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| BAAI/bge-large-en-v1.5 | BGE | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| BAAI/bge-large-en-v1.5 | BGE | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
## Reward Models
@@ -91,7 +91,7 @@ You are welcome to enable various models based on your business requirements.
| Shanghai_AI_Laboratory/internlm2-7b-reward | InternLM 2 Reward | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| Qwen/Qwen2.5-Math-RM-72B | Qwen2.5 Reward - Math | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| jason9693/Qwen2.5-1.5B-apeach | Qwen2.5 Reward - Sequence | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
| Skywork/Skywork-Reward-Gemma-2-27B-v0.2 | Gemma 2-27B Reward | **<span style="color: red;">×</span>** | **<span style="color: red;">×</span>** |
| Skywork/Skywork-Reward-Gemma-2-27B-v0.2 | Gemma 2-27B Reward | **<span style="color: green;">√</span>** | **<span style="color: green;">√</span>** |
## Rerank Models