From 0a9099e137d43c3871688f72eb5bd9f7ee1ae05d Mon Sep 17 00:00:00 2001 From: amote-i <49533125+amote-i@users.noreply.github.com> Date: Wed, 21 Jan 2026 15:36:26 +0800 Subject: [PATCH] update ascend docs (#17457) --- docs/platforms/ascend_npu_deepseek_example.md | 184 ++++++++---------- docs/platforms/ascend_npu_support_models.md | 2 +- 2 files changed, 77 insertions(+), 109 deletions(-) diff --git a/docs/platforms/ascend_npu_deepseek_example.md b/docs/platforms/ascend_npu_deepseek_example.md index efa57ae3c..d0b207f18 100644 --- a/docs/platforms/ascend_npu_deepseek_example.md +++ b/docs/platforms/ascend_npu_deepseek_example.md @@ -180,134 +180,102 @@ python -m sglang_router.launch_router \ W8A8 Model weights could be found [here](https://modelers.cn/models/State_Cloud/Deepseek-R1-bf16-hfd-w8a8). -1. Prefill: +1. Prefill & Decode: ```shell +echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor +sysctl -w vm.swappiness=0 +sysctl -w kernel.numa_balancing=0 +sysctl -w kernel.sched_migration_cost_ns=50000 +export SGLANG_SET_CPU_AFFINITY=1 +unset ASCEND_LAUNCH_BLOCKING +source /usr/local/Ascend/ascend-toolkit/set_env.sh +source /usr/local/Ascend/nnal/atb/set_env.sh +export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH + export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export STREAMS_PER_DEVICE=32 -#memfabric config store -export ASCEND_MF_STORE_URL="tcp://:" +export ASCEND_MF_STORE_URL="tcp://your prefill ip1:24669" -#Deepep communication settings -export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 -export HCCL_BUFFSIZE=1536 +P_IP=('your prefill ip1' 'your prefill ip2') + +D_IP=('your decode ip1' 'your decode ip2') + +MODEL_PATH=xxx -#npu acceleration operator export SGLANG_NPU_USE_MLAPO=1 export SGLANG_USE_FIA_NZ=1 -export ENABLE_MOE_NZ=1 -export TASK_QUEUE_ENABLE=2 -#Please list all host ips of Prefill instance -P_HOST_IP=('xx.xx.xx.xx' 'xx.xx.xx.xx') - -for i in "${!P_HOST_IP[@]}"; +LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` +LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` +echo "${LOCAL_HOST1}" +echo "${LOCAL_HOST2}" +# prefill +for i in "${!P_IP[@]}"; do - python -m sglang.launch_server \ - --model-path ${MODEL_PATH} \ - --host ${P_HOST_IP[$i]} \ - --port 8000 \ - --disaggregation-mode prefill \ - --disaggregation-bootstrap-port $((8996+$i)) \ - --disaggregation-transfer-backend ascend \ - --trust-remote-code \ - --nnodes 1 \ - --node-rank 0 \ - --tp-size 16 \ - --mem-fraction-static 0.81 \ - --attention-backend ascend \ - --device npu \ - --max-running-requests 8 \ - --context-length 8192 \ - --disable-radix-cache \ - --chunked-prefill-size -1 \ - --max-prefill-tokens 28680 \ - --moe-a2a-backend deepep \ - --deepep-mode normal \ - --speculative-algorithm NEXTN \ - --speculative-num-steps 1 \ - --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 2 \ - --dp-size 2 \ - --enable-dp-attention \ - --disable-shared-experts-fusion \ - --dtype bfloat16 + if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]]; + then + echo "${P_IP[$i]}" + export HCCL_BUFFSIZE=1536 + export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 + export TASK_QUEUE_ENABLE=2 + + export HCCL_SOCKET_IFNAME=lo + export GLOO_SOCKET_IFNAME=lo + python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode prefill --host ${P_IP[$i]} \ + --port 8000 --disaggregation-bootstrap-port $((8998+$i)) --trust-remote-code --nnodes 1 --node-rank 0 \ + --tp-size 16 --mem-fraction-static 0.81 --attention-backend ascend --device npu --quantization modelslim \ + --disaggregation-transfer-backend ascend --max-running-requests 8 --context-length 8192 --disable-radix-cache \ + --chunked-prefill-size -1 --max-prefill-tokens 28680 --moe-a2a-backend deepep --deepep-mode normal \ + --speculative-algorithm NEXTN --speculative-num-steps 1 --speculative-eagle-topk 1 --speculative-num-draft-tokens 2 \ + --dp-size 2 --enable-dp-attention --disable-shared-experts-fusion --dtype bfloat16 --enable-attn-tp-input-scattered + NODE_RANK=$i + break + fi +done + +# decode +for i in "${!D_IP[@]}"; +do + if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]]; + then + echo "${D_IP[$i]}" + export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 + export SGLANG_ENABLE_SPEC_V2=1 + export HCCL_BUFFSIZE=650 + export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=78 + export TASK_QUEUE_ENABLE=1 + export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1 + export HCCL_SOCKET_IFNAME=xxx + export GLOO_SOCKET_IFNAME=xxx + python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode decode --host ${D_IP[$i]} \ + --port 8001 --trust-remote-code --dist-init-addr ${D_IP[0]}:5000 --nnodes 2 --node-rank $i --tp-size 32 --dp-size 32 \ + --mem-fraction-static 0.815 --max-running-requests 832 --attention-backend ascend --device npu --quantization modelslim \ + --moe-a2a-backend deepep --enable-dp-attention --deepep-mode low_latency --enable-dp-lm-head --moe-dense-tp 1 \ + --cuda-graph-bs 12 14 16 18 20 22 24 26 --disaggregation-transfer-backend ascend --watchdog-timeout 9000 --context-length 8192 \ + --speculative-algorithm NEXTN --speculative-num-steps 2 --speculative-eagle-topk 1 --speculative-num-draft-tokens 3 \ + --tokenizer-worker-num 4 --prefill-round-robin-balance --disable-shared-experts-fusion --dtype bfloat16 \ + --load-balance-method decode_round_robin + NODE_RANK=$i + break + fi done ``` -2. Decode: - -```shell -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export STREAMS_PER_DEVICE=32 - -#memfabric config store -export ASCEND_MF_STORE_URL="tcp://:" - -#Deepep communication settings -export HCCL_BUFFSIZE=600 -export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=78 - -#spec overlap -export SGLANG_ENABLE_SPEC_V2=1 -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - -#npu acceleration operator -unset TASK_QUEUE_ENABLE -export SGLANG_NPU_USE_MLAPO=1 -export SGLANG_USE_FIA_NZ=1 -export ENABLE_MOE_NZ=1 - -#please list all host ips of Prefill instance -D_HOST_IP=('xx.xx.xx.xx' 'xx.xx.xx.xx') - -for i in "${!D_HOST_IP[@]}"; -do - python -m sglang.launch_server - --model-path ${MODEL_PATH} \ - --disaggregation-mode decode \ - --disaggregation-transfer-backend ascend \ - --host ${D_HOST_IP[$i]} \ - --port 8001 \ - --trust-remote-code \ - --dist-init-addr ${D_HOST_IP[0]}:5000 \ - --nnodes 2 \ - --node-rank $i \ - --tp-size 32 \ - --dp-size 32 \ - --mem-fraction-static 0.8 \ - --max-running-requests 832 \ - --attention-backend ascend \ - --device npu \ - --moe-a2a-backend deepep \ - --enable-dp-attention \ - --deepep-mode low_latency \ - --enable-dp-lm-head \ - --cuda-graph-bs 8 10 12 14 16 18 20 22 24 26 \ - --watchdog-timeout 9000 \ - --context-length 8192 \ - --speculative-algorithm NEXTN \ - --speculative-num-steps 2 \ - --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 3 \ - --tokenizer-worker-num 4 \ - --disable-shared-experts-fusion \ - --dtype bfloat16 -done -``` - -3. SGLang Model Gateway (former Router): +2. SGLang Model Gateway (former Router): ```shell +export SGLANG_DP_ROUND_ROBIN=1 python -m sglang_router.launch_router \ --pd-disaggregation \ --policy cache_aware \ - --prefill http://:8000 8996 \ - --prefill http://:8000 8997 \ - --decode http://:8001 \ + --prefill http://P_IP:8000 8998 \ + --prefill http://P_IP:8000 8999 \ + --decode http://D_IP:8001 \ --host 127.0.0.1 \ - --port 6688 + --port 6688 \ + --mini-lb ``` #### test gsm8k diff --git a/docs/platforms/ascend_npu_support_models.md b/docs/platforms/ascend_npu_support_models.md index 2c5c4c9a9..16594ecff 100644 --- a/docs/platforms/ascend_npu_support_models.md +++ b/docs/platforms/ascend_npu_support_models.md @@ -47,7 +47,7 @@ You are welcome to enable various models based on your business requirements. | baichuan-inc/Baichuan2-13B-Chat | Baichuan 2 (7B, 13B) | **√** | **√** | | baidu/ERNIE-4.5-21B-A3B-PT | ERNIE-4.5 (4.5, 4.5MoE series) | **√** | **√** | | openbmb/MiniCPM3-4B | MiniCPM (v3, 4B) | **√** | **√** | -| Kimi/Kimi-K2-Think | Kimi | **√** | **√** | +| Kimi/Kimi-K2-Thinking | Kimi | **√** | **√** | | openai/gpt-oss-120b | GPTOSS | **×** | **×** | ## Multimodal Language Models