diff --git a/docs/platforms/ascend_npu_best_practice.md b/docs/platforms/ascend_npu_best_practice.md index fe08a74a6..639b343f1 100644 --- a/docs/platforms/ascend_npu_best_practice.md +++ b/docs/platforms/ascend_npu_best_practice.md @@ -13,7 +13,7 @@ you encounter issues or have any questions, please [open an issue](https://githu | Deepseek-R1 | Atlas 800I A3 | 32 | PD Separation | 3.9K-1K | W8A8 | [Optimal Configuration](#deepseek-r1-low-latency-20ms-2) | | Deepseek-R1 | Atlas 800I A3 | 32 | PD Separation | 3.5K-1.5K | W8A8 | [Optimal Configuration](#deepseek-r1-low-latency-20ms-3) | | Deepseek-R1 | Atlas 800I A3 | 32 | PD Separation | 3.5K-1K | W8A8 | [Optimal Configuration](#deepseek-r1-low-latency-20ms-4) | -| Deepseek-V3.2 | Atlas 800I A3 | 32 | PD Separation | 64K-1K | W8A8 | [Optimal Configuration](#deepseek-v32-low-latency-30ms) | +| Deepseek-V3.2 | Atlas 800I A3 | 32 | PD Separation | 64K-3K | W8A8 | [Optimal Configuration](#deepseek-v32-low-latency-30ms) | ### High Throughput @@ -135,7 +135,7 @@ do export SGLANG_ENABLE_SPEC_V2=1 export HCCL_BUFFSIZE=650 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=78 - export TASK_QUEUE_ENABLE=0 + export TASK_QUEUE_ENABLE=1 export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1 export HCCL_SOCKET_IFNAME=xxx export GLOO_SOCKET_IFNAME=xxx @@ -170,7 +170,7 @@ python -m sglang_router.launch_router \ #### Benchmark ```shell -python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6688 --max-concurrency 832 --random-input-len 3500 --random-output-len 1500 --num-prompts 3328 --random-range-ratio 1 +python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6688 --max-concurrency 768 --random-input-len 3500 --random-output-len 1500 --num-prompts 3072 --random-range-ratio 1 --request-rate 16 ``` ### DeepSeek R1 Low Latency 20ms 1 @@ -247,13 +247,13 @@ do export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_SPEC_V2=1 export HCCL_BUFFSIZE=650 - export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=4 - export TASK_QUEUE_ENABLE=0 + export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=12 + export TASK_QUEUE_ENABLE=1 export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1 export HCCL_SOCKET_IFNAME=xxx export GLOO_SOCKET_IFNAME=xxx python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode decode --host ${D_IP[$i]} \ - --port 8001 --trust-remote-code --dist-init-addr DIP1:5000 --nnodes 2 --node-rank $i --tp-size 32 --dp-size 8 \ + --port 8001 --trust-remote-code --dist-init-addr DIP1:5000 --nnodes 2 --node-rank $i --tp-size 32 --dp-size 16 \ --mem-fraction-static 0.75 --max-running-requests 32 --attention-backend ascend --device npu --quantization modelslim \ --moe-a2a-backend deepep --enable-dp-attention --deepep-mode low_latency --enable-dp-lm-head --moe-dense-tp 1 \ --cuda-graph-bs 2 4 6 --disaggregation-transfer-backend ascend --watchdog-timeout 9000 --context-length 8192 \ @@ -305,7 +305,7 @@ Please Turn to [DeepSeek R1 Low Latency 20ms](#deepSeek-r1-low-latency-20ms-1) #### Benchmark ```bash -python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6688 --max-concurrency 32 --random-input-len 3900 --random-output-len 1000 --num-prompts 32 --random-range-ratio 1 +python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6688 --max-concurrency 768 --random-input-len 3900 --random-output-len 1000 --num-prompts 768 --random-range-ratio 1 --request-rate 16 ``` ### DeepSeek R1 Low Latency 20ms 3 @@ -327,7 +327,7 @@ Please Turn to [DeepSeek R1 Low Latency 20ms](#deepSeek-r1-low-latency-20ms-1) #### Benchmark ```bash -python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6688 --max-concurrency 32 --random-input-len 3500 --random-output-len 1500 --num-prompts 32 --random-range-ratio 1 +python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6688 --max-concurrency 768 --random-input-len 3500 --random-output-len 1500 --num-prompts 768 --random-range-ratio 1 --request-rate 16 ``` ### DeepSeek R1 Low Latency 20ms 4 @@ -349,7 +349,7 @@ Please Turn to [DeepSeek R1 Low Latency 20ms](#deepSeek-r1-low-latency-20ms-1) #### Benchmark ```bash -python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6688 --max-concurrency 32 --random-input-len 3500 --random-output-len 1000 --num-prompts 32 --random-range-ratio 1 +python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6688 --max-concurrency 768 --random-input-len 3500 --random-output-len 1000 --num-prompts 768 --random-range-ratio 1 --request-rate 16 ``` ### DeepSeek R1 High Performance 50ms 2 @@ -513,7 +513,7 @@ do export SGLANG_ENABLE_SPEC_V2=1 export HCCL_BUFFSIZE=720 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=96 - export TASK_QUEUE_ENABLE=0 + export TASK_QUEUE_ENABLE=1 export HCCL_SOCKET_IFNAME=xxx export GLOO_SOCKET_IFNAME=xxx python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode decode --host ${D_IP[$i]} \ @@ -706,7 +706,7 @@ do export SGLANG_ENABLE_SPEC_V2=1 export HCCL_BUFFSIZE=720 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=96 - export TASK_QUEUE_ENABLE=0 + export TASK_QUEUE_ENABLE=1 export HCCL_SOCKET_IFNAME=xxx export GLOO_SOCKET_IFNAME=xxx python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode decode --host ${D_IP[$i]} \ @@ -750,7 +750,7 @@ Hardware: Atlas 800I A3 32Card DeployMode: PD Separation -DataSets: 64K1K +DataSets: 64K3K TPOT: 30ms @@ -947,7 +947,7 @@ python -m sglang_router.launch_router \ #### Benchmark ```shell -python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6688 --max-concurrency 32 --random-input-len 64000 --random-output-len 1000 --num-prompts 64 --random-range-ratio 1 --request-rate 0.25 +python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6688 --max-concurrency 32 --random-input-len 64000 --random-output-len 3000 --num-prompts 64 --random-range-ratio 1 ``` ### Qwen3 235B High Throughput 50ms 1 @@ -1019,7 +1019,6 @@ do --host ${P_IP[$i]} --port 8000 --disaggregation-bootstrap-port 8995 --trust-remote-code \ --nnodes 1 --node-rank $i --tp-size 16 --dp-size 16 --mem-fraction-static 0.6 \ --disable-radix-cache \ - --ep-dispatch-algorithm static --init-expert-location /path/to/expert_distribution_recorder.pt \ --attention-backend ascend --device npu --quantization modelslim --disaggregation-transfer-backend ascend \ --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \ --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ @@ -1080,7 +1079,7 @@ python -m sglang_router.launch_router \ #### Benchmark ```shell -python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7239 --max-concurrency 768 --random-input-len 3500 --random-output-len 1500 --num-prompts 3072 --random-range-ratio 1 +python -m sglang.bench_serving --dataset-name random --backend sglang-oai --host 127.0.0.1 --port 7239 --max-concurrency 860 --random-input-len 3500 --random-output-len 1500 --num-prompts 3440 --random-range-ratio 1 ``` ### Qwen3 235B High Throughput 50ms 2 @@ -1132,7 +1131,7 @@ export GLOO_SOCKET_IFNAME=lo export HCCL_OP_EXPANSION_MODE="AIV" export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_SPEC_V2=1 -export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 +export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=2 python -m sglang.launch_server --model-path $MODEL_PATH \ --host 127.0.0.1 --port 7439 --trust-remote-code --nnodes 1 --node-rank 0 \ @@ -1149,7 +1148,7 @@ python -m sglang.launch_server --model-path $MODEL_PATH \ #### Benchmark ```shell -python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7439 --max-concurrency 288 --random-input-len 3500 --random-output-len 1500 --num-prompts 1088 --random-range-ratio 1 +python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7439 --max-concurrency 272 --random-input-len 3500 --random-output-len 1500 --num-prompts 1088 --random-range-ratio 1 ``` ### Qwen3-235B Atlas 800I A3-8Card PD Mixed 2K-2K 100ms @@ -1270,7 +1269,6 @@ export GLOO_SOCKET_IFNAME=xxx export HCCL_OP_EXPANSION_MODE="AIV" export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_SPEC_V2=1 -export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 python -m sglang.launch_server --model-path $MODEL_PATH \ --host 127.0.0.1 --port 7439 --trust-remote-code --nnodes 1 --node-rank 0 \ @@ -1322,8 +1320,6 @@ source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=16 - MODEL_PATH=xxx export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 @@ -1348,17 +1344,16 @@ do echo "${MIX_IP[$i]}" export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_SPEC_V2=1 - export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 python -m sglang.launch_server --model-path ${MODEL_PATH} \ --host 127.0.0.1 --port 7439 --trust-remote-code \ --nnodes 2 --node-rank $i --tp-size 32 --dp-size 32 --mem-fraction-static 0.8 --max-running-requests 768 \ --attention-backend ascend --device npu --quantization modelslim --enable-dp-attention \ --moe-a2a-backend deepep --deepep-mode auto --cuda-graph-bs 6 8 10 12 18 24 \ - --dist-init-addr ${MIX_IP[0]}:5000 --chunked-prefill-size 32768 --max-prefill-tokens 458880 \ - --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \ + --dist-init-addr ${MIX_IP[0]}:5000 --chunked-prefill-size 131072 --max-prefill-tokens 458880 \ + --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx --speculative-draft-model-quantization= unquant \ --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ - --watchdog-timeout 9000 --context-length 8192 \ + --context-length 8192 --disable-radix-cache \ --enable-dp-lm-head --dtype bfloat16 NODE_RANK=$i break @@ -1493,10 +1488,10 @@ export SGLANG_ENABLE_SPEC_V2=1 python -m sglang.launch_server --model-path $MODEL_PATH \ --host 127.0.0.1 --port 7439 --trust-remote-code --nnodes 1 --node-rank 0 \ - --attention-backend ascend --device npu --quantization modelslim \ + --attention-backend ascend --device npu \ --max-running-requests 32 \ --disable-radix-cache \ - --chunked-prefill-size 32768 --max-prefill-tokens 65536 --speculative-draft-model-quantization unquant \ + --chunked-prefill-size 24576 --max-prefill-tokens 65536 \ --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \ --speculative-num-steps 4 --speculative-eagle-topk 1 --speculative-num-draft-tokens 5 \ --tp-size 8 --mem-fraction-static 0.72 --cuda-graph-bs 8 16 24 32 --dtype bfloat16 @@ -1624,13 +1619,12 @@ export GLOO_SOCKET_IFNAME=lo export HCCL_OP_EXPANSION_MODE="AIV" export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_SPEC_V2=1 -export DISABLE_EAGLE3_QUANT=1 python -m sglang.launch_server --model-path $MODEL_PATH \ --host 127.0.0.1 --port 7339 --trust-remote-code --nnodes 1 --node-rank 0 \ --attention-backend ascend --device npu \ --max-running-requests 1 \ - --disable-radix-cache \ + --disable-radix-cache --speculative-draft-model-quantization unquant \ --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \ --speculative-num-steps 4 --speculative-eagle-topk 1 --speculative-num-draft-tokens 5 \ --chunked-prefill-size -1 --max-prefill-tokens 65536 \ @@ -1766,7 +1760,7 @@ python -m sglang.launch_server --model-path $MODEL_PATH \ --disable-radix-cache --speculative-draft-model-quantization unquant \ --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \ --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ - --chunked-prefill-size 65536 --max-prefill-tokens 49152 \ + --chunked-prefill-size -1 --max-prefill-tokens 49152 \ --tp-size 4 --mem-fraction-static 0.7 --cuda-graph-bs 54 60 66 72 78 84 90 108 114 120 --dtype bfloat16 ``` @@ -2105,7 +2099,7 @@ python -m sglang.launch_server --model-path $MODEL_PATH \ #### Benchmark ```shell -python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7439 --max-concurrency 80 --random-input-len 3500 --random-output-len 1500 --num-prompts 320 --random-range-ratio 1 --request-rate 8 +python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7439 --max-concurrency 80 --random-input-len 3500 --random-output-len 1500 --num-prompts 320 --random-range-ratio 1 ``` ### Qwen3 Next High Throughput 50ms diff --git a/docs/platforms/ascend_npu_support_features.md b/docs/platforms/ascend_npu_support_features.md index 581346e6e..fb2cdbfe5 100644 --- a/docs/platforms/ascend_npu_support_features.md +++ b/docs/platforms/ascend_npu_support_features.md @@ -21,7 +21,7 @@ click [Server Arguments](https://docs.sglang.io/advanced_features/server_argumen | `--context-length` | `None` | Type: int | **√** | **√** | | `--is-embedding` | `False` | bool flag (set to enable) | **√** | **√** | | `--enable-multimodal` | `None` | bool flag (set to enable) | **√** | **√** | -| `--revision` | `None` | Type: str | **×** | **×** | +| `--revision` | `None` | Type: str | **√** | **√** | | `--model-impl` | `auto` | `auto`, `sglang`,
`transformers` | **√** | **√** | ## HTTP server @@ -33,8 +33,8 @@ click [Server Arguments](https://docs.sglang.io/advanced_features/server_argumen | `--skip-server-warmup` | `False` | bool flag
(set to enable) | **√** | **√** | | `--warmups` | `None` | Type: str | **√** | **√** | | `--nccl-port` | `None` | Type: int | **√** | **√** | -| `--fastapi-root-path` | `None` | Type: str | **×** | **×** | -| `--grpc-mode` | `False` | bool flag
(set to enable) | **×** | **×** | +| `--fastapi-root-path` | `None` | Type: str | **√** | **√** | +| `--grpc-mode` | `False` | bool flag
(set to enable) | **√** | **√** | | `--checkpoint-engine-`
`wait-weights-`
`before-ready` | `False` | bool flag
(set to enable) | **×** | **×** | ## Quantization and data type @@ -45,7 +45,7 @@ click [Server Arguments](https://docs.sglang.io/advanced_features/server_argumen | `--quantization` | `None` | `modelslim` | **√** | **√** | | `--quantization-param-path` | `None` | Type: str | **×** | **×** | | `--kv-cache-dtype` | `auto` | `auto` | **√** | **√** | -| `--enable-fp32-lm-head` | `False` | bool flag
(set to enable) | **×** | **×** | +| `--enable-fp32-lm-head` | `False` | bool flag
(set to enable) | **√** | **√** | | `--modelopt-quant` | `None` | Type: str | **×** | **×** | | `--modelopt-checkpoint-`
`restore-path` | `None` | Type: str | **×** | **×** | | `--modelopt-checkpoint-`
`save-path` | `None` | Type: str | **×** | **×** | @@ -73,7 +73,7 @@ click [Server Arguments](https://docs.sglang.io/advanced_features/server_argumen | `--hybrid-kvcache-ratio` | `None` | Optional[float] | **×** | **×** | | `--swa-full-tokens-ratio` | `0.8` | Type: float | **×** | **×** | | `--disable-hybrid-swa-memory` | `False` | bool flag
(set to enable) | **×** | **×** | -| `--abort-on-priority-`
`when-disabled` | `False` | bool flag
(set to enable) | **×** | **×** | +| `--abort-on-priority-`
`when-disabled` | `False` | bool flag
(set to enable) | **√** | **√** | | `--enable-dynamic-chunking` | `False` | bool flag
(set to enable) | **×** | **×** | ## Runtime options @@ -88,15 +88,15 @@ click [Server Arguments](https://docs.sglang.io/advanced_features/server_argumen | `--stream-interval` | `1` | Type: int | **√** | **√** | | `--stream-output` | `False` | bool flag (set to enable) | **√** | **√** | | `--random-seed` | `None` | Type: int | **√** | **√** | -| `--constrained-json-`
`whitespace-pattern` | `None` | Type: str | **×** | **×** | -| `--constrained-json-`
`disable-any-whitespace` | `False` | bool flag (set to enable) | **×** | **×** | +| `--constrained-json-`
`whitespace-pattern` | `None` | Type: str | **√** | **√** | +| `--constrained-json-`
`disable-any-whitespace` | `False` | bool flag (set to enable) | **√** | **√** | | `--watchdog-timeout` | `300` | Type: float | **√** | **√** | | `--soft-watchdog-timeout` | `300` | Type: float | **√** | **√** | | `--dist-timeout` | `None` | Type: int | **√** | **√** | | `--base-gpu-id` | `0` | Type: int | **√** | **√** | | `--gpu-id-step` | `1` | Type: int | **√** | **√** | | `--sleep-on-idle` | `False` | bool flag (set to enable) | **√** | **√** | -| `--custom-sigquit-handler` | `None` | Optional[Callable] | **×** | **×** | +| `--custom-sigquit-handler` | `None` | Optional[Callable] | **√** | **√** | ## Logging @@ -106,32 +106,32 @@ click [Server Arguments](https://docs.sglang.io/advanced_features/server_argumen | `--log-level-http` | `None` | Type: str | **√** | **√** | | `--log-requests` | `False` | bool flag
(set to enable) | **√** | **√** | | `--log-requests-level` | `2` | `0`, `1`, `2`, `3` | **√** | **√** | -| `--log-requests-format` | text | text, json | **×** | **×** | +| `--log-requests-format` | text | text, json | **√** | **√** | | `--crash-dump-folder` | `None` | Type: str | **×** | **×** | | `--crash-on-nan` | `False` | Type: str | **×** | **×** | -| `--enable-metrics` | `False` | bool flag
(set to enable) | **×** | **×** | -| `--enable-metrics-for-`
`all-schedulers` | `False` | bool flag
(set to enable) | **×** | **×** | -| `--tokenizer-metrics-`
`custom-labels-header` | `x-custom-labels` | Type: str | **×** | **×** | -| `--tokenizer-metrics-`
`allowed-custom-labels` | `None` | List[str] | **×** | **×** | -| `--bucket-time-to-`
`first-token` | `None` | List[float] | **×** | **×** | -| `--bucket-inter-token-`
`latency` | `None` | List[float] | **×** | **×** | -| `--bucket-e2e-request-`
`latency` | `None` | List[float] | **×** | **×** | -| `--collect-tokens-`
`histogram` | `False` | bool flag
(set to enable) | **×** | **×** | -| `--prompt-tokens-buckets` | `None` | List[str] | **×** | **×** | -| `--generation-tokens-buckets` | `None` | List[str] | **×** | **×** | -| `--gc-warning-threshold-secs` | `0.0` | Type: float | **×** | **×** | +| `--enable-metrics` | `False` | bool flag
(set to enable) | **√** | **√** | +| `--enable-metrics-for-`
`all-schedulers` | `False` | bool flag
(set to enable) | **√** | **√** | +| `--tokenizer-metrics-`
`custom-labels-header` | `x-custom-labels` | Type: str | **√** | **√** | +| `--tokenizer-metrics-`
`allowed-custom-labels` | `None` | List[str] | **√** | **√** | +| `--bucket-time-to-`
`first-token` | `None` | List[float] | **√** | **√** | +| `--bucket-inter-token-`
`latency` | `None` | List[float] | **√** | **√** | +| `--bucket-e2e-request-`
`latency` | `None` | List[float] | **√** | **√** | +| `--collect-tokens-`
`histogram` | `False` | bool flag
(set to enable) | **√** | **√** | +| `--prompt-tokens-buckets` | `None` | List[str] | **√** | **√** | +| `--generation-tokens-buckets` | `None` | List[str] | **√** | **√** | +| `--gc-warning-threshold-secs` | `0.0` | Type: float | **√** | **√** | | `--decode-log-interval` | `40` | Type: int | **√** | **√** | | `--enable-request-time-`
`stats-logging` | `False` | bool flag
(set to enable) | **√** | **√** | | `--kv-events-config` | `None` | Type: str | **×** | **×** | -| `--enable-trace` | `False` | bool flag
(set to enable) | **×** | **×** | -| `--oltp-traces-endpoint` | `localhost:4317` | Type: str | **×** | **×** | +| `--enable-trace` | `False` | bool flag
(set to enable) | **√** | **√** | +| `--oltp-traces-endpoint` | `localhost:4317` | Type: str | **√** | **√** | ## RequestMetricsExporter configuration -| Argument | Defaults | Options | A2 | A3 | -|---------------------------------------|----------|--------------------------------|:--------------------------------------:|:--------------------------------------:| -| `--export-metrics-to-`
`file` | `False` | bool flag
(set to enable) | **×** | **×** | -| `--export-metrics-to-`
`file-dir` | `None` | Type: str | **×** | **×** | +| Argument | Defaults | Options | A2 | A3 | +|---------------------------------------|----------|--------------------------------|:----------------------------------------:|:----------------------------------------:| +| `--export-metrics-to-`
`file` | `False` | bool flag
(set to enable) | **√** | **√** | +| `--export-metrics-to-`
`file-dir` | `None` | Type: str | **√** | **√** | ## API related @@ -146,16 +146,16 @@ click [Server Arguments](https://docs.sglang.io/advanced_features/server_argumen | `--enable-cache-report` | `False` | bool flag
(set to enable) | **√** | **√** | | `--reasoning-parser` | `None` | `deepseek-r1` | **√** | **√** | | `--tool-call-parser` | `None` | `llama`,`pythonic` | **√** | **√** | -| `--sampling-defaults` | `model` | `openai`, `model` | **×** | **×** | +| `--sampling-defaults` | `model` | `openai`, `model` | **√** | **√** | | `--tool-server` | `None` | Type: str | **×** | **×** | ## Data parallelism -| Argument | Defaults | Options | A2 | A3 | -|----------------------------------------|---------------|-------------------------------------------------------------|:----------------------------------------:|:----------------------------------------:| -| `--data-parallel-size`
`--dp-size` | `1` | Type: int | **√** | **√** | +| Argument | Defaults | Options | A2 | A3 | +|----------------------------------------|---------------|-----------------------------------------------------------|:----------------------------------------:|:----------------------------------------:| +| `--data-parallel-size`
`--dp-size` | `1` | Type: int | **√** | **√** | | `--load-balance-method` | `round_robin` | `round_robin`,
`total_requests`,
`total_tokens` | **√** | **√** | -| `--prefill-round-robin-balance` | `False` | bool flag
(set to enable) | **√** | **√** | +| `--prefill-round-robin-balance` | `False` | bool flag
(set to enable) | **√** | **√** | ## Multi-node distributed serving @@ -208,13 +208,13 @@ click [Server Arguments](https://docs.sglang.io/advanced_features/server_argumen | `--speculative-algorithm` | `None` | `EAGLE3`,
`NEXTN` | **√** | **√** | | `--speculative-draft-model-path`
`--speculative-draft-model` | `None` | Type: str | **√** | **√** | | `--speculative-draft-model-`
`revision` | `None` | Type: str | **√** | **√** | -| `--speculative-draft-load-format` | `None` | `auto` | **×** | **×** | +| `--speculative-draft-load-format` | `None` | `auto` | **√** | **√** | | `--speculative-num-steps` | `None` | Type: int | **√** | **√** | | `--speculative-eagle-topk` | `None` | Type: int | **√** | **√** | | `--speculative-num-draft-tokens` | `None` | Type: int | **√** | **√** | | `--speculative-accept-`
`threshold-single` | `1.0` | Type: float | **×** | **×** | | `--speculative-accept-`
`threshold-acc` | `1.0` | Type: float | **×** | **×** | -| `--speculative-token-map` | `None` | Type: str | **×** | **×** | +| `--speculative-token-map` | `None` | Type: str | **√** | **√** | | `--speculative-attention-`
`mode` | `prefill` | `prefill`,
`decode` | **√** | **√** | | `--speculative-moe-runner-`
`backend` | `None` | `auto` | **√** | **√** | | `--speculative-moe-a2a-`
`backend` | `None` | `ascend_fuseep` | **√** | **√** | @@ -235,28 +235,28 @@ click [Server Arguments](https://docs.sglang.io/advanced_features/server_argumen ## Expert parallelism -| Argument | Defaults | Options | A2 | A3 | -|-------------------------------------------------------|-----------|---------------------------------------------|:----------------------------------------:|:----------------------------------------:| -| `--expert-parallel-size`
`--ep-size`
`--ep` | `1` | Type: int | **√** | **√** | -| `--moe-a2a-backend` | `none` | `none`,
`deepep`,
`ascend_fuseep` | **√** | **√** | -| `--moe-runner-backend` | `auto` | `auto`, `triton` | **√** | **√** | -| `--flashinfer-mxfp4-`
`moe-precision` | `default` | `default`,
`bf16` | **×** | **×** | -| `--enable-flashinfer-`
`allreduce-fusion` | `False` | bool flag
(set to enable) | **×** | **×** | -| `--deepep-mode` | `auto` | `normal`,
`low_latency`,
`auto` | **√** | **√** | -| `--deepep-config` | `None` | Type: str | **×** | **×** | -| `--ep-num-redundant-experts` | `0` | Type: int | **×** | **×** | -| `--ep-dispatch-algorithm` | `None` | Type: str | **×** | **×** | -| `--init-expert-location` | `trivial` | Type: str | **×** | **×** | -| `--enable-eplb` | `False` | bool flag
(set to enable) | **×** | **×** | -| `--eplb-algorithm` | `auto` | Type: str | **×** | **×** | -| `--eplb-rebalance-layers-`
`per-chunk` | `None` | Type: int | **×** | **×** | -| `--eplb-min-rebalancing-`
`utilization-threshold` | `1.0` | Type: float | **×** | **×** | -| `--expert-distribution-`
`recorder-mode` | `None` | Type: str | **×** | **×** | -| `--expert-distribution-`
`recorder-buffer-size` | `None` | Type: int | **×** | **×** | -| `--enable-expert-distribution-`
`metrics` | `False` | bool flag
(set to enable) | **×** | **×** | -| `--moe-dense-tp-size` | `None` | Type: int | **√** | **√** | -| `--elastic-ep-backend` | `None` | `none`, `mooncake` | **×** | **×** | -| `--mooncake-ib-device` | `None` | Type: str | **×** | **×** | +| Argument | Defaults | Options | A2 | A3 | +|-------------------------------------------------------|-----------|---------------------------------------------|:-----------------------------------------:|:----------------------------------------:| +| `--expert-parallel-size`
`--ep-size`
`--ep` | `1` | Type: int | **√** | **√** | +| `--moe-a2a-backend` | `none` | `none`,
`deepep`,
`ascend_fuseep` | **√** | **√** | +| `--moe-runner-backend` | `auto` | `auto`, `triton` | **√** | **√** | +| `--flashinfer-mxfp4-`
`moe-precision` | `default` | `default`,
`bf16` | **×** | **×** | +| `--enable-flashinfer-`
`allreduce-fusion` | `False` | bool flag
(set to enable) | **×** | **×** | +| `--deepep-mode` | `auto` | `normal`,
`low_latency`,
`auto` | **√** | **√** | +| `--deepep-config` | `None` | Type: str | **×** | **×** | +| `--ep-num-redundant-experts` | `0` | Type: int | **√** | **√** | +| `--ep-dispatch-algorithm` | `None` | Type: str | **√** | **√** | +| `--init-expert-location` | `trivial` | Type: str | **√** | **√** | +| `--enable-eplb` | `False` | bool flag
(set to enable) | **√** | **√** | +| `--eplb-algorithm` | `auto` | Type: str | **√** | **√** | +| `--eplb-rebalance-layers-`
`per-chunk` | `None` | Type: int | **√** | **√** | +| `--eplb-min-rebalancing-`
`utilization-threshold` | `1.0` | Type: float | **√** | **√** | +| `--expert-distribution-`
`recorder-mode` | `None` | Type: str | **√** | **√** | +| `--expert-distribution-`
`recorder-buffer-size` | `None` | Type: int | **√** | **√** | +| `--enable-expert-distribution-`
`metrics` | `False` | bool flag
(set to enable) | ***√** | **√** | +| `--moe-dense-tp-size` | `None` | Type: int | **√** | **√** | +| `--elastic-ep-backend` | `None` | `none`, `mooncake` | **×** | **×** | +| `--mooncake-ib-device` | `None` | Type: str | **×** | **×** | ## Mamba Cache @@ -342,7 +342,7 @@ click [Server Arguments](https://docs.sglang.io/advanced_features/server_argumen | `--enable-symm-mem` | `False` | bool flag
(set to enable) | **×** | **×** | | `--disable-flashinfer-`
`cutlass-moe-fp4-allgather` | `False` | bool flag
(set to enable) | **×** | **×** | | `--enable-tokenizer-`
`batch-encode` | `False` | bool flag
(set to enable) | **√** | **√** | -| `--disable-tokenizer-`
`batch-encode` | `False` | bool flag
(set to enable) | **×** | **×** | +| `--disable-tokenizer-`
`batch-encode` | `False` | bool flag
(set to enable) | **√** | **√** | | `--disable-outlines-`
`disk-cache` | `False` | bool flag
(set to enable) | **√** | **√** | | `--disable-custom-`
`all-reduce` | `False` | bool flag
(set to enable) | **√** | **√** | | `--enable-mscclpp` | `False` | bool flag
(set to enable) | **×** | **×** | @@ -355,7 +355,7 @@ click [Server Arguments](https://docs.sglang.io/advanced_features/server_argumen | `--enable-single-`
`batch-overlap` | `False` | bool flag
(set to enable) | **×** | **×** | | `--tbo-token-`
`distribution-threshold` | `0.48` | Type: float | **×** | **×** | | `--enable-torch-`
`compile` | `False` | bool flag
(set to enable) | **√** | **√** | -| `--enable-torch-`
`compile-debug-mode` | `False` | bool flag
(set to enable) | **×** | **×** | +| `--enable-torch-`
`compile-debug-mode` | `False` | bool flag
(set to enable) | **√** | **√** | | `--enable-piecewise-`
`cuda-graph` | `False` | bool flag
(set to enable) | **×** | **×** | | `--piecewise-cuda-`
`graph-tokens` | `None` | Type: JSON
list | **×** | **×** | | `--piecewise-cuda-`
`graph-compiler` | `eager` | ["eager", "inductor"] | **×** | **×** | @@ -380,13 +380,13 @@ click [Server Arguments](https://docs.sglang.io/advanced_features/server_argumen | `--disable-fast-`
`image-processor` | `False` | bool flag
(set to enable) | **×** | **×** | | `--keep-mm-feature-`
`on-device` | `False` | bool flag
(set to enable) | **×** | **×** | | `--enable-return-`
`hidden-states` | `False` | bool flag
(set to enable) | **√** | **√** | -| `--enable-return-`
`routed-experts` | `False` | bool flag
(set to enable) | **×** | **×** | +| `--enable-return-`
`routed-experts` | `False` | bool flag
(set to enable) | **√** | **√** | | `--scheduler-recv-`
`interval` | `1` | Type: int | **×** | **×** | | `--numa-node` | `None` | List[int] | **×** | **×** | | `--rl-on-policy-target` | `None` | `fsdp` | **×** | **×** | | `--enable-layerwise-`
`nvtx-marker` | `False` | bool flag
(set to enable) | **×** | **×** | | `--enable-attn-tp-`
`input-scattered` | `False` | bool flag
(set to enable) | **×** | **×** | -| `--enable-nsa-prefill-`
`context-parallel` | `False` | bool flag
(set to enable) | **×** | **×** | +| `--enable-nsa-prefill-`
`context-parallel` | `False` | bool flag
(set to enable) | **√** | **√** | | `--enable-fused-qk-`
`norm-rope` | `False` | bool flag
(set to enable) | **×** | **×** | ## Dynamic batch tokenizer @@ -401,10 +401,9 @@ click [Server Arguments](https://docs.sglang.io/advanced_features/server_argumen | Argument | Defaults | Options | A2 | A3 | |--------------------------------------------|----------|-----------|:----------------------------------------:|:----------------------------------------:| -| `--debug-tensor-dump-`
`output-folder` | `None` | Type: str | **×** | **×** | -| `--debug-tensor-dump-`
`layers` | `None` | List[int] | **×** | **×** | +| `--debug-tensor-dump-`
`output-folder` | `None` | Type: str | **√** | **√** | +| `--debug-tensor-dump-`
`layers` | `None` | List[int] | **√** | **√** | | `--debug-tensor-dump-`
`input-file` | `None` | Type: str | **√** | **√** | -| `--debug-tensor-dump-`
`inject` | `False` | Type: str | **×** | **×** | ## PD disaggregation @@ -415,21 +414,20 @@ click [Server Arguments](https://docs.sglang.io/advanced_features/server_argumen | `--disaggregation-bootstrap-port` | `8998` | Type: int | **√** | **√** | | `--disaggregation-decode-tp` | `None` | Type: int | **√** | **√** | | `--disaggregation-decode-dp` | `None` | Type: int | **√** | **√** | -| `--disaggregation-prefill-pp` | `1` | Type: int | **×** | **×** | | `--disaggregation-ib-device` | `None` | Type: str | **×** | **×** | -| `--disaggregation-decode-`
`enable-offload-kvcache` | `False` | bool flag
(set to enable) | **×** | **×** | -| `--disaggregation-decode-`
`enable-fake-auto` | `False` | bool flag
(set to enable) | **×** | **×** | +| `--disaggregation-decode-`
`enable-offload-kvcache` | `False` | bool flag
(set to enable) | **√** | **√** | +| `--disaggregation-decode-`
`enable-fake-auto` | `False` | bool flag
(set to enable) | **√** | **√** | | `--num-reserved-decode-tokens` | `512` | Type: int | **√** | **√** | | `--disaggregation-decode-`
`polling-interval` | `1` | Type: int | **√** | **√** | ## Encode prefill disaggregation -| Argument | Defaults | Options | A2 | A3 | -|------------------------------|--------------------|----------------------------------------------------------------|:--------------------------------------:|:--------------------------------------:| -| `--encoder-only` | `False` | bool flag
(set to enable) | **×** | **×** | -| `--language-only` | `False` | bool flag
(set to enable) | **×** | **×** | -| `--encoder-transfer-backend` | `zmq_to_scheduler` | `zmq_to_scheduler`,
`zmq_to_tokenizer`,
`mooncake` | **×** | **×** | -| `--encoder-urls` | `[]` | List[str] | **×** | **×** | +| Argument | Defaults | Options | A2 | A3 | +|------------------------------|--------------------|----------------------------------------------------------------|:----------------------------------------:|:----------------------------------------:| +| `--encoder-only` | `False` | bool flag
(set to enable) | **√** | **√** | +| `--language-only` | `False` | bool flag
(set to enable) | **√** | **√** | +| `--encoder-transfer-backend` | `zmq_to_scheduler` | `zmq_to_scheduler`,
`zmq_to_tokenizer`,
`mooncake` | **√** | **√** | +| `--encoder-urls` | `[]` | List[str] | **√** | **√** | ## Custom weight loader @@ -478,12 +476,12 @@ click [Server Arguments](https://docs.sglang.io/advanced_features/server_argumen ## For registering hooks -| Argument | Defaults | Options | A2 | A3 | -|-------------------|----------|-----------------|----------------------------------------|----------------------------------------| -| `--forward-hooks` | `None` | Type: JSON list | **×** | **×** | +| Argument | Defaults | Options | A2 | A3 | +|-------------------|----------|-----------------|------------------------------------------|------------------------------------------| +| `--forward-hooks` | `None` | Type: JSON list | **√** | **√** | ## Configuration file support -| Argument | Defaults | Options | A2 | A3 | -|------------|----------|-----------|----------------------------------------|----------------------------------------| -| `--config` | `None` | Type: str | **×** | **×** | +| Argument | Defaults | Options | A2 | A3 | +|------------|----------|-----------|------------------------------------------|------------------------------------------| +| `--config` | `None` | Type: str | **√** | **√** | diff --git a/docs/platforms/ascend_npu_support_models.md b/docs/platforms/ascend_npu_support_models.md index 701aa1ac7..1ecc3c48f 100644 --- a/docs/platforms/ascend_npu_support_models.md +++ b/docs/platforms/ascend_npu_support_models.md @@ -46,12 +46,12 @@ You are welcome to enable various models based on your business requirements. | baichuan-inc/Baichuan2-13B-Chat | Baichuan 2 (7B, 13B) | **√** | **√** | | baidu/ERNIE-4.5-21B-A3B-PT | ERNIE-4.5 (4.5, 4.5MoE series) | **√** | **√** | | openbmb/MiniCPM3-4B | MiniCPM (v3, 4B) | **√** | **√** | -| openai/gpt-oss-120b | GPTOSS | **×** | **×** | +| openai/gpt-oss-120b | GPTOSS | **√** | **√** | ## Multimodal Language Models -| Models | Model Family (Variants) | A2 Supported | A3 Supported | -|-----------------------------------------------|---------------------------|------------------------------------------|:----------------------------------------:| +| Models | Model Family (Variants) | A2 Supported | A3 Supported | +|-----------------------------------------------|---------------------------|:----------------------------------------:|:----------------------------------------:| | Qwen/Qwen2.5-VL-3B-Instruct | Qwen-VL | **√** | **√** | | Qwen/Qwen2.5-VL-72B-Instruct | Qwen-VL | **√** | **√** | | Qwen/Qwen3-VL-30B-A3B-Instruct | Qwen-VL | **√** | **√** | @@ -70,18 +70,18 @@ You are welcome to enable various models based on your business requirements. | lmms-lab/llava-onevision-qwen2-7b-ov | LLaVA-OneVision | **√** | **√** | | Kimi/Kimi-VL-A3B-Instruct | Kimi-VL (A3B) | **√** | **√** | | ZhipuAI/GLM-4.5V | GLM-4.5V (106B) | **√** | **√** | -| meta-llama/Llama-3.2-11B-Vision-Instruct | Llama 3.2 Vision (11B) | **×** | **×** | +| meta-llama/Llama-3.2-11B-Vision-Instruct | Llama 3.2 Vision (11B) | **√** | **√** | ## Embedding Models -| Models | Model Family | A2 Supported | A3 Supported | -|-------------------------------------------|--------------------------|------------------------------------------|:----------------------------------------:| +| Models | Model Family | A2 Supported | A3 Supported | +|-------------------------------------------|--------------------------|:----------------------------------------:|:----------------------------------------:| | intfloat/e5-mistral-7b-instruct | E5 (Llama/Mistral based) | **√** | **√** | | iic/gte_Qwen2-1.5B-instruct | GTE-Qwen2 | **√** | **√** | | Qwen/Qwen3-Embedding-8B | Qwen3-Embedding | **√** | **√** | | Alibaba-NLP/gme-Qwen2-VL-2B-Instruct | GME (Multimodal) | **√** | **√** | | AI-ModelScope/clip-vit-large-patch14-336 | CLIP | **√** | **√** | -| BAAI/bge-large-en-v1.5 | BGE | **×** | **×** | +| BAAI/bge-large-en-v1.5 | BGE | **√** | **√** | ## Reward Models @@ -91,7 +91,7 @@ You are welcome to enable various models based on your business requirements. | Shanghai_AI_Laboratory/internlm2-7b-reward | InternLM 2 Reward | **√** | **√** | | Qwen/Qwen2.5-Math-RM-72B | Qwen2.5 Reward - Math | **√** | **√** | | jason9693/Qwen2.5-1.5B-apeach | Qwen2.5 Reward - Sequence | **√** | **√** | -| Skywork/Skywork-Reward-Gemma-2-27B-v0.2 | Gemma 2-27B Reward | **×** | **×** | +| Skywork/Skywork-Reward-Gemma-2-27B-v0.2 | Gemma 2-27B Reward | **√** | **√** | ## Rerank Models