[NPU] NZ for non-quantized MOE, Qwen3 MOE double memory consumption fix (#15904)
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
parent
1953efb60e
commit
b77b0ffd60
@@ -22,7 +22,6 @@ export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
#npu acceleration operator
|
||||
export SGLANG_NPU_USE_MLAPO=1
|
||||
export SGLANG_USE_FIA_NZ=1
|
||||
export ENABLE_MOE_NZ=1
|
||||
|
||||
python3 -m sglang.launch_server \
|
||||
--model-path ${MODEL_PATH} \
|
||||
@@ -71,7 +70,6 @@ export HCCL_BUFFSIZE=1536
|
||||
#npu acceleration operator
|
||||
export SGLANG_NPU_USE_MLAPO=1
|
||||
export SGLANG_USE_FIA_NZ=1
|
||||
export ENABLE_MOE_NZ=1
|
||||
export TASK_QUEUE_ENABLE=2
|
||||
|
||||
python -m sglang.launch_server \
|
||||
|
||||
@@ -62,7 +62,6 @@ export HCCL_BUFFSIZE=1536
|
||||
export HCCL_OP_EXPANSION_MODE=AIV
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=32
|
||||
export SGLANG_DEEPEP_BF16_DISPATCH=1
|
||||
export ENABLE_ASCEND_MOE_NZ=1
|
||||
|
||||
python -m sglang.launch_server \
|
||||
--device npu \
|
||||
@@ -84,7 +83,6 @@ export STREAMS_PER_DEVICE=32
|
||||
export HCCL_BUFFSIZE=1536
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=32
|
||||
export SGLANG_DEEPEP_BF16_DISPATCH=1
|
||||
export ENABLE_ASCEND_MOE_NZ=1
|
||||
|
||||
python -m sglang.launch_server \
|
||||
--model-path Qwen/Qwen3-235B-A22B-Instruct-2507 \
|
||||
|
||||
Reference in New Issue
Block a user