chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 11 Packages Projects Releases Wiki Activity

[Ascend ] Add qwen3.5 122B/35B/27B deployment examples on doc (#19339)

Browse Source 
Co-authored-by: sglang-npu-bot <sglangnpu@163.com>
This commit is contained in:
Hexq0210
2026-02-25 21:09:22 +08:00
committed by GitHub
parent 59b9d1e86d
commit 56891e46bc
2 changed files with 139 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
docs/platforms/ascend_npu_glm5_examples.md
View File

@@ -1,4 +1,4 @@
# GLM-5
# GLM-5 examples
## Introduction

149
docs/platforms/ascend_npu_qwen3_5_examples.md
View File

@@ -1,4 +1,4 @@
# Qwen3.5
# Qwen3.5 examples
## Environment Preparation
@@ -8,9 +8,9 @@ The dependencies required for the NPU runtime environment have been integrated i
```{code-block} bash
#Atlas 800 A3
docker pull swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:cann8.5.0-a3-qwen3.5
docker pull quay.io/ascend/sglang:v0.5.9-cann8.5.0-a3
#Atlas 800 A2
docker pull swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:cann8.5.0-910b-qwen3.5
docker pull quay.io/ascend/sglang:v0.5.9-cann8.5.0-910b
#start container
docker run -itd --shm-size=16g --privileged=true --name ${NAME} \
@@ -39,17 +39,17 @@ docker run -itd --shm-size=16g --privileged=true --name ${NAME} \
--device=/dev/davinci_manager:/dev/davinci_manager \
--device=/dev/hisi_hdc:/dev/hisi_hdc \
--entrypoint=bash \
swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:${TAG}
quay.io/ascend/sglang:${tag}
```
## Deployment
### Single-node Deployment
- Quantized model `qwen35_w8a8` can be deployed on 1 Atlas 800 A3 (64G × 16) .
Run the following script to execute online inference.
#### Qwen3.5 397B
```shell
# high performance cpu
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
@@ -69,10 +69,6 @@ source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export STREAMS_PER_DEVICE=32
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_NPU_USE_MULTI_STREAM=1
export HCCL_BUFFSIZE=1000
export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=lo
@@ -83,7 +79,8 @@ python3 -m sglang.launch_server \
--attention-backend ascend \
--device npu \
--tp-size 16 --nnodes 1 --node-rank 0 \
--chunked-prefill-size 16384 --max-prefill-tokens 280000 \
--chunked-prefill-size 4096 --max-prefill-tokens 280000 \
--disable-radix-cache \
--trust-remote-code \
--host 127.0.0.1 \
--mem-fraction-static 0.7 \
@@ -95,6 +92,136 @@ python3 -m sglang.launch_server \
--dtype bfloat16
```
#### Qwen3.5 122B
```shell
# high performance cpu
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
sysctl -w vm.swappiness=0
sysctl -w kernel.numa_balancing=0
sysctl -w kernel.sched_migration_cost_ns=50000
# bind cpu
export SGLANG_SET_CPU_AFFINITY=1
unset https_proxy
unset http_proxy
unset HTTPS_PROXY
unset HTTP_PROXY
unset ASCEND_LAUNCH_BLOCKING
# cann
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export STREAMS_PER_DEVICE=32
export HCCL_BUFFSIZE=1000
export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=lo
export GLOO_SOCKET_IFNAME=lo
python3 -m sglang.launch_server \
--model-path $MODEL_PATH \
--attention-backend ascend \
--device npu \
--tp-size 8 --nnodes 1 --node-rank 0 \
--chunked-prefill-size 4096 --max-prefill-tokens 280000 \
--disable-radix-cache \
--trust-remote-code \
--host 127.0.0.1 \
--mem-fraction-static 0.7 \
--port 8000 \
--cuda-graph-bs 16 \
--quantization modelslim \
--enable-multimodal \
--mm-attention-backend ascend_attn \
--dtype bfloat16
```
#### Qwen3.5 35B
```shell
# high performance cpu
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
sysctl -w vm.swappiness=0
sysctl -w kernel.numa_balancing=0
sysctl -w kernel.sched_migration_cost_ns=50000
# bind cpu
export SGLANG_SET_CPU_AFFINITY=1
unset https_proxy
unset http_proxy
unset HTTPS_PROXY
unset HTTP_PROXY
unset ASCEND_LAUNCH_BLOCKING
# cann
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export STREAMS_PER_DEVICE=32
export HCCL_BUFFSIZE=1000
export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=lo
export GLOO_SOCKET_IFNAME=lo
python3 -m sglang.launch_server \
--model-path $MODEL_PATH \
--attention-backend ascend \
--device npu \
--tp-size 2 --nnodes 1 --node-rank 0 \
--chunked-prefill-size 4096 --max-prefill-tokens 280000 \
--disable-radix-cache \
--trust-remote-code \
--host 127.0.0.1 \
--mem-fraction-static 0.7 \
--port 8000 \
--cuda-graph-bs 16 \
--quantization modelslim \
--enable-multimodal \
--mm-attention-backend ascend_attn \
--dtype bfloat16
```
#### Qwen3.5 27B
```shell
# high performance cpu
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
sysctl -w vm.swappiness=0
sysctl -w kernel.numa_balancing=0
sysctl -w kernel.sched_migration_cost_ns=50000
# bind cpu
export SGLANG_SET_CPU_AFFINITY=1
unset https_proxy
unset http_proxy
unset HTTPS_PROXY
unset HTTP_PROXY
unset ASCEND_LAUNCH_BLOCKING
# cann
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export STREAMS_PER_DEVICE=32
export HCCL_BUFFSIZE=1000
export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=lo
export GLOO_SOCKET_IFNAME=lo
python3 -m sglang.launch_server \
--model-path $MODEL_PATH \
--attention-backend ascend \
--device npu \
--tp-size 2 \
--chunked-prefill-size -1 --max-prefill-tokens 120000 \
--disable-radix-cache \
--trust-remote-code \
--host 127.0.0.1 \
--mem-fraction-static 0.8 \
--port 8000 \
--cuda-graph-bs 32 \
--enable-multimodal \
--mm-attention-backend ascend_attn
```
### Prefill-Decode Disaggregation
Not test yet.