diff --git a/docs/platforms/ascend_npu_glm5_examples.md b/docs/platforms/ascend_npu_glm5_examples.md index 5f1d2cde2..0adfe1d94 100644 --- a/docs/platforms/ascend_npu_glm5_examples.md +++ b/docs/platforms/ascend_npu_glm5_examples.md @@ -1,4 +1,4 @@ -# GLM-5 +# GLM-5 examples ## Introduction diff --git a/docs/platforms/ascend_npu_qwen3_5_examples.md b/docs/platforms/ascend_npu_qwen3_5_examples.md index 8b71f03c3..124b571f6 100644 --- a/docs/platforms/ascend_npu_qwen3_5_examples.md +++ b/docs/platforms/ascend_npu_qwen3_5_examples.md @@ -1,4 +1,4 @@ -# Qwen3.5 +# Qwen3.5 examples ## Environment Preparation @@ -8,9 +8,9 @@ The dependencies required for the NPU runtime environment have been integrated i ```{code-block} bash #Atlas 800 A3 -docker pull swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:cann8.5.0-a3-qwen3.5 +docker pull quay.io/ascend/sglang:v0.5.9-cann8.5.0-a3 #Atlas 800 A2 -docker pull swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:cann8.5.0-910b-qwen3.5 +docker pull quay.io/ascend/sglang:v0.5.9-cann8.5.0-910b #start container docker run -itd --shm-size=16g --privileged=true --name ${NAME} \ @@ -39,17 +39,17 @@ docker run -itd --shm-size=16g --privileged=true --name ${NAME} \ --device=/dev/davinci_manager:/dev/davinci_manager \ --device=/dev/hisi_hdc:/dev/hisi_hdc \ --entrypoint=bash \ -swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:${TAG} +quay.io/ascend/sglang:${tag} ``` ## Deployment ### Single-node Deployment -- Quantized model `qwen35_w8a8` can be deployed on 1 Atlas 800 A3 (64G × 16) . - Run the following script to execute online inference. +#### Qwen3.5 397B + ```shell # high performance cpu echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor @@ -69,10 +69,6 @@ source /usr/local/Ascend/ascend-toolkit/set_env.sh source /usr/local/Ascend/nnal/atb/set_env.sh export STREAMS_PER_DEVICE=32 -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 -export SGLANG_ENABLE_SPEC_V2=1 -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_NPU_USE_MULTI_STREAM=1 export HCCL_BUFFSIZE=1000 export HCCL_OP_EXPANSION_MODE=AIV export HCCL_SOCKET_IFNAME=lo @@ -83,7 +79,8 @@ python3 -m sglang.launch_server \ --attention-backend ascend \ --device npu \ --tp-size 16 --nnodes 1 --node-rank 0 \ - --chunked-prefill-size 16384 --max-prefill-tokens 280000 \ + --chunked-prefill-size 4096 --max-prefill-tokens 280000 \ + --disable-radix-cache \ --trust-remote-code \ --host 127.0.0.1 \ --mem-fraction-static 0.7 \ @@ -95,6 +92,136 @@ python3 -m sglang.launch_server \ --dtype bfloat16 ``` +#### Qwen3.5 122B + +```shell +# high performance cpu +echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor +sysctl -w vm.swappiness=0 +sysctl -w kernel.numa_balancing=0 +sysctl -w kernel.sched_migration_cost_ns=50000 +# bind cpu +export SGLANG_SET_CPU_AFFINITY=1 + +unset https_proxy +unset http_proxy +unset HTTPS_PROXY +unset HTTP_PROXY +unset ASCEND_LAUNCH_BLOCKING +# cann +source /usr/local/Ascend/ascend-toolkit/set_env.sh +source /usr/local/Ascend/nnal/atb/set_env.sh + +export STREAMS_PER_DEVICE=32 +export HCCL_BUFFSIZE=1000 +export HCCL_OP_EXPANSION_MODE=AIV +export HCCL_SOCKET_IFNAME=lo +export GLOO_SOCKET_IFNAME=lo + +python3 -m sglang.launch_server \ + --model-path $MODEL_PATH \ + --attention-backend ascend \ + --device npu \ + --tp-size 8 --nnodes 1 --node-rank 0 \ + --chunked-prefill-size 4096 --max-prefill-tokens 280000 \ + --disable-radix-cache \ + --trust-remote-code \ + --host 127.0.0.1 \ + --mem-fraction-static 0.7 \ + --port 8000 \ + --cuda-graph-bs 16 \ + --quantization modelslim \ + --enable-multimodal \ + --mm-attention-backend ascend_attn \ + --dtype bfloat16 +``` + +#### Qwen3.5 35B + +```shell +# high performance cpu +echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor +sysctl -w vm.swappiness=0 +sysctl -w kernel.numa_balancing=0 +sysctl -w kernel.sched_migration_cost_ns=50000 +# bind cpu +export SGLANG_SET_CPU_AFFINITY=1 + +unset https_proxy +unset http_proxy +unset HTTPS_PROXY +unset HTTP_PROXY +unset ASCEND_LAUNCH_BLOCKING +# cann +source /usr/local/Ascend/ascend-toolkit/set_env.sh +source /usr/local/Ascend/nnal/atb/set_env.sh + +export STREAMS_PER_DEVICE=32 +export HCCL_BUFFSIZE=1000 +export HCCL_OP_EXPANSION_MODE=AIV +export HCCL_SOCKET_IFNAME=lo +export GLOO_SOCKET_IFNAME=lo + +python3 -m sglang.launch_server \ + --model-path $MODEL_PATH \ + --attention-backend ascend \ + --device npu \ + --tp-size 2 --nnodes 1 --node-rank 0 \ + --chunked-prefill-size 4096 --max-prefill-tokens 280000 \ + --disable-radix-cache \ + --trust-remote-code \ + --host 127.0.0.1 \ + --mem-fraction-static 0.7 \ + --port 8000 \ + --cuda-graph-bs 16 \ + --quantization modelslim \ + --enable-multimodal \ + --mm-attention-backend ascend_attn \ + --dtype bfloat16 +``` + +#### Qwen3.5 27B + +```shell +# high performance cpu +echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor +sysctl -w vm.swappiness=0 +sysctl -w kernel.numa_balancing=0 +sysctl -w kernel.sched_migration_cost_ns=50000 +# bind cpu +export SGLANG_SET_CPU_AFFINITY=1 + +unset https_proxy +unset http_proxy +unset HTTPS_PROXY +unset HTTP_PROXY +unset ASCEND_LAUNCH_BLOCKING +# cann +source /usr/local/Ascend/ascend-toolkit/set_env.sh +source /usr/local/Ascend/nnal/atb/set_env.sh + +export STREAMS_PER_DEVICE=32 +export HCCL_BUFFSIZE=1000 +export HCCL_OP_EXPANSION_MODE=AIV +export HCCL_SOCKET_IFNAME=lo +export GLOO_SOCKET_IFNAME=lo + +python3 -m sglang.launch_server \ + --model-path $MODEL_PATH \ + --attention-backend ascend \ + --device npu \ + --tp-size 2 \ + --chunked-prefill-size -1 --max-prefill-tokens 120000 \ + --disable-radix-cache \ + --trust-remote-code \ + --host 127.0.0.1 \ + --mem-fraction-static 0.8 \ + --port 8000 \ + --cuda-graph-bs 32 \ + --enable-multimodal \ + --mm-attention-backend ascend_attn +``` + ### Prefill-Decode Disaggregation Not test yet.