chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 11 Packages Projects Releases Wiki Activity

Add NPU basic function testcases (#19382)

Browse Source 
Co-authored-by: cy <chenyang08056032@163.com>
Co-authored-by: Cherry_ming <136634645@qq.com>
This commit is contained in:
Sugar920
2026-03-16 15:09:56 +08:00
committed by GitHub
parent e96a3752a0
commit 895e56097c
87 changed files with 4587 additions and 333 deletions
Download Patch File Download Diff File Expand all files Collapse all files

108
test/registered/ascend/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_auto_deepseek_v3_2_w8a8.py Normal file
View File

@@ -0,0 +1,108 @@
import os
import unittest
from types import SimpleNamespace
from sglang.srt.utils import kill_process_tree
from sglang.test.ascend.test_ascend_utils import DEEPSEEK_V3_2_W8A8_WEIGHTS_PATH
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.few_shot_gsm8k import run_eval as run_gsm8k
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_URL_FOR_TEST,
CustomTestCase,
popen_launch_server,
)
register_npu_ci(est_time=400, suite="nightly-16-npu-a3", nightly=True)
class TestDeepEpDeepseekV32(CustomTestCase):
"""Testcase: Verify that for the DeepSeek V3.2 model in the single-machine colocation scenario,
its inference accuracy on the MMLU and GSM8K dataset meets the preset standard when the parameter --deepep-mode auto is configured.
[Test Category] Expert Parallelism
[Test Target] --moe-a2a-backend deepep;--deepep-mode
"""
@classmethod
def setUpClass(cls):
cls.model = DEEPSEEK_V3_2_W8A8_WEIGHTS_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=6000,
other_args=[
"--trust-remote-code",
"--tp-size",
"16",
"--quantization",
"modelslim",
"--moe-a2a-backend",
"deepep",
"--deepep-mode",
"auto",
"--mem-fraction-static",
0.82,
"--disable-cuda-graph",
"--disable-radix-cache",
"--context-length",
40960,
"--max-prefill-tokens",
40960,
"--max-total-tokens",
40960,
],
env={
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"STREAMS_PER_DEVICE": "32",
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "16",
"HCCL_BUFFSIZE": "1600",
"HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_NPU_USE_MLAPO": "0",
"SGLANG_NPU_USE_MULTI_STREAM": "1",
"TASK_QUEUE_ENABLE": "0",
**os.environ,
},
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_mmlu(self):
expect_score = 0.85
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="mmlu",
num_examples=128,
num_threads=32,
)
print("Starting mmlu test...")
metrics = run_eval(args)
self.assertGreater(metrics["score"], expect_score)
def test_gsm8k(self):
expect_accuracy = 0.95
args = SimpleNamespace(
num_shots=8,
data_path=None,
timeout=60000,
num_questions=200,
max_new_tokens=512,
parallel=128,
host="http://127.0.0.1",
port=int(self.base_url.split(":")[-1]),
)
print("Starting gsm8k test...")
metrics = run_gsm8k(args)
self.assertGreaterEqual(
metrics["accuracy"],
expect_accuracy,
f'Accuracy of {self.model} is {str(metrics["accuracy"])}, is lower than {expect_accuracy}',
)
if __name__ == "__main__":
unittest.main()

128
test/registered/ascend/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_auto_qwen3_480b.py Normal file
View File

@@ -0,0 +1,128 @@
import os
import unittest
from types import SimpleNamespace
from sglang.srt.utils import kill_process_tree
from sglang.test.ascend.test_ascend_utils import (
QWEN3_CODER_480B_A35B_INSTRUCT_W8A8_QUAROT_WEIGHTS_PATH,
)
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.few_shot_gsm8k import run_eval as run_gsm8k
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
popen_launch_server,
)
register_npu_ci(est_time=200, suite="nightly-16-npu-a3", nightly=True)
class TestDeepEpQwen(CustomTestCase):
"""
Testcase:Test the Qwen3-Coder-480B-A35B-Instruct-w8a8-QuaRot model with DeepEP's auto mode enabled,
and verify that there is no drop in accuracy compared to when DeepEP is not enabled.
[Test Category] Expert Parallelism
[Test Target] --moe-a2a-backend, --deepep-mode
"""
@classmethod
def setUpClass(cls):
cls.model = QWEN3_CODER_480B_A35B_INSTRUCT_W8A8_QUAROT_WEIGHTS_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
"--trust-remote-code",
"--nnodes",
"1",
"--node-rank",
"0",
"--attention-backend",
"ascend",
"--device",
"npu",
"--quantization",
"modelslim",
"--max-running-requests",
96,
"--context-length",
8192,
"--dtype",
"bfloat16",
"--chunked-prefill-size",
28672,
"--max-prefill-tokens",
458880,
"--disable-radix-cache",
"--moe-a2a-backend",
"deepep",
"--deepep-mode",
"auto",
"--tp-size",
16,
"--dp-size",
4,
"--enable-dp-attention",
"--enable-dp-lm-head",
"--mem-fraction-static",
0.7,
"--cuda-graph-bs",
16,
20,
24,
],
env={
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600",
"HCCL_BUFFSIZE": "2100",
"HCCL_OP_EXPANSION_MODE": "AIV",
**os.environ,
},
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_mmlu(self):
expect_score = 0.61
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="mmlu",
num_examples=8,
num_threads=32,
)
metrics = run_eval(args)
self.assertGreater(metrics["score"], expect_score)
def test_gsm8k(self):
expect_accuracy = 0.91
host = "http://127.0.0.1"
port = int(self.base_url.split(":")[-1])
args = SimpleNamespace(
num_shots=8,
data_path=None,
num_questions=200,
max_new_tokens=512,
parallel=128,
host=host,
port=port,
)
metrics = run_gsm8k(args)
self.assertGreaterEqual(
metrics["accuracy"],
expect_accuracy,
f'Accuracy of {self.model} is {str(metrics["accuracy"])}, is lower than {expect_accuracy}',
)
if __name__ == "__main__":
unittest.main()

118
test/registered/ascend/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_auto_qwen3_next.py Normal file
View File

@@ -0,0 +1,118 @@
import os
import unittest
from types import SimpleNamespace
from sglang.srt.utils import kill_process_tree
from sglang.test.ascend.test_ascend_utils import (
QWEN3_NEXT_80B_A3B_INSTRUCT_WEIGHTS_PATH,
)
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.few_shot_gsm8k import run_eval as run_gsm8k
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
popen_launch_server,
)
register_npu_ci(
est_time=200,
suite="nightly-8-npu-a3",
nightly=True,
disabled="https://github.com/Ascend/sglang/issues/58",
)
class TestQwen3Next(CustomTestCase):
"""
Testcase:Test the Qwen3-Next-80B-A3B-Instruct-W8A8 model with DeepEP's auto mode enabled, and verify that there is
no drop in accuracy compared to when DeepEP is not enabled.
[Test Category] Parameter
[Test Target] --moe-a2a-backend deepep, --deepep-mode auto
"""
@classmethod
def setUpClass(cls):
cls.model = QWEN3_NEXT_80B_A3B_INSTRUCT_WEIGHTS_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
"--trust-remote-code",
"--attention-backend",
"ascend",
"--device",
"npu",
"--tp-size",
8,
"--mem-fraction-static",
0.8,
"--max-running-requests",
80,
"--watchdog-timeout",
9000,
"--disable-radix-cache",
"--disable-cuda-graph",
"--max-prefill-tokens",
28672,
"--max-total-tokens",
450560,
"--moe-a2a-backend",
"deepep",
"--deepep-mode",
"auto",
"--chunked-prefill-size",
-1,
],
env={
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"STREAMS_PER_DEVICE": "32",
"HCCL_OP_EXPANSION_MODE": "AIV",
"HCCL_ALGO": "level0:NA;level1:ring",
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "20",
"HCCL_BUFFSIZE": "2000",
**os.environ,
},
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_mmlu(self):
expect_score = 0.56
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="mmlu",
num_examples=8,
num_threads=32,
)
metrics = run_eval(args)
self.assertGreater(metrics["score"], expect_score)
def test_gsm8k(self):
expect_accuracy = 0.9
args = SimpleNamespace(
num_shots=5,
data_path=None,
num_questions=200,
max_new_tokens=512,
parallel=128,
host="http://127.0.0.1",
port=int(self.base_url.split(":")[-1]),
)
metrics = run_gsm8k(args)
self.assertGreaterEqual(
metrics["accuracy"],
expect_accuracy,
f'Accuracy of {self.model} is {str(metrics["accuracy"])}, is lower than {expect_accuracy}',
)
if __name__ == "__main__":
unittest.main()

107
test/registered/ascend/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_low_latency_deepseek_v3_2_w8a8.py Normal file
View File

@@ -0,0 +1,107 @@
import os
import unittest
from types import SimpleNamespace
from sglang.srt.utils import kill_process_tree
from sglang.test.ascend.test_ascend_utils import DEEPSEEK_V3_2_W8A8_WEIGHTS_PATH
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.few_shot_gsm8k import run_eval as run_gsm8k
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_URL_FOR_TEST,
CustomTestCase,
popen_launch_server,
)
register_npu_ci(est_time=200, suite="nightly-16-npu-a3", nightly=False)
class TestDeepEpDeepseekV32(CustomTestCase):
"""Testcase: Verify that for the DeepSeek V3.2 model in the single-machine colocation scenario,
its inference accuracy on the MMLU and GSM8K dataset meets the preset standard when the parameter --deepep-mode low_latency is configured.
[Test Category] Expert Parallelism
[Test Target] --moe-a2a-backend deepep;--deepep-mode
[Test Suggestions] Mixing deployment + low_latency mode is not recommended.
"""
@classmethod
def setUpClass(cls):
cls.model = DEEPSEEK_V3_2_W8A8_WEIGHTS_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=6000,
other_args=[
"--trust-remote-code",
"--tp-size",
"16",
"--quantization",
"modelslim",
"--moe-a2a-backend",
"deepep",
"--deepep-mode",
"low_latency",
"--mem-fraction-static",
0.82,
"--disable-cuda-graph",
"--disable-radix-cache",
"--context-length",
40960,
"--max-prefill-tokens",
128,
"--max-total-tokens",
40960,
],
env={
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"STREAMS_PER_DEVICE": "32",
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "128",
"HCCL_BUFFSIZE": "2048",
"HCCL_OP_EXPANSION_MODE": "AIV",
"TASK_QUEUE_ENABLE": "0",
**os.environ,
},
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_mmlu(self):
expect_score = 0.85
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="mmlu",
num_examples=128,
num_threads=32,
)
print("Starting mmlu test...")
metrics = run_eval(args)
self.assertGreater(metrics["score"], expect_score)
def test_gsm8k(self):
expect_accuracy = 0.95
args = SimpleNamespace(
num_shots=8,
data_path=None,
timeout=60000,
num_questions=200,
max_new_tokens=512,
parallel=128,
host="http://127.0.0.1",
port=int(self.base_url.split(":")[-1]),
)
print("Starting gsm8k test...")
metrics = run_gsm8k(args)
self.assertGreaterEqual(
metrics["accuracy"],
expect_accuracy,
f'Accuracy of {self.model} is {str(metrics["accuracy"])}, is lower than {expect_accuracy}',
)
if __name__ == "__main__":
unittest.main()

125
test/registered/ascend/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_low_latency_qwen3_480b.py Normal file
View File

@@ -0,0 +1,125 @@
import os
import unittest
from types import SimpleNamespace
from sglang.srt.utils import kill_process_tree
from sglang.test.ascend.test_ascend_utils import (
QWEN3_CODER_480B_A35B_INSTRUCT_W8A8_QUAROT_WEIGHTS_PATH,
)
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.few_shot_gsm8k import run_eval as run_gsm8k
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
popen_launch_server,
)
register_npu_ci(est_time=200, suite="nightly-16-npu-a3", nightly=False)
class TestDeepEpQwen(CustomTestCase):
"""
Testcase:Test the Qwen3-Coder-480B-A35B-Instruct-w8a8-QuaRot model with DeepEP's low_latency mode enabled,
and verify that there is no drop in accuracy compared to when DeepEP is not enabled.
[Test Category] Expert Parallelism
[Test Target] --moe-a2a-backend, --deepep-mode
[Test Suggestions] Mixing deployment + low_latency mode is not recommended.
"""
@classmethod
def setUpClass(cls):
cls.model = QWEN3_CODER_480B_A35B_INSTRUCT_W8A8_QUAROT_WEIGHTS_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
"--trust-remote-code",
"--nnodes",
"1",
"--node-rank",
"0",
"--attention-backend",
"ascend",
"--device",
"npu",
"--quantization",
"modelslim",
"--max-running-requests",
96,
"--context-length",
8192,
"--dtype",
"bfloat16",
"--chunked-prefill-size",
1024,
"--max-prefill-tokens",
458880,
"--disable-radix-cache",
"--moe-a2a-backend",
"deepep",
"--deepep-mode",
"low_latency",
"--tp-size",
16,
"--dp-size",
4,
"--enable-dp-attention",
"--enable-dp-lm-head",
"--mem-fraction-static",
0.7,
"--cuda-graph-bs",
16,
20,
24,
],
env={
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600",
"HCCL_BUFFSIZE": "2100",
"HCCL_OP_EXPANSION_MODE": "AIV",
**os.environ,
},
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_mmlu(self):
expect_score = 0.61
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="mmlu",
num_examples=8,
num_threads=32,
)
metrics = run_eval(args)
self.assertGreater(metrics["score"], expect_score)
def test_gsm8k(self):
expect_accuracy = 0.91
args = SimpleNamespace(
num_shots=8,
data_path=None,
num_questions=200,
max_new_tokens=512,
parallel=128,
host="http://127.0.0.1",
port=int(self.base_url.split(":")[-1]),
)
metrics = run_gsm8k(args)
self.assertGreaterEqual(
metrics["accuracy"],
expect_accuracy,
f'Accuracy of {self.model} is {str(metrics["accuracy"])}, is lower than {expect_accuracy}',
)
if __name__ == "__main__":
unittest.main()

118
test/registered/ascend/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_low_latency_qwen3_next.py Normal file
View File

@@ -0,0 +1,118 @@
import os
import unittest
from types import SimpleNamespace
from sglang.srt.utils import kill_process_tree
from sglang.test.ascend.test_ascend_utils import (
QWEN3_NEXT_80B_A3B_INSTRUCT_WEIGHTS_PATH,
)
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.few_shot_gsm8k import run_eval as run_gsm8k
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
popen_launch_server,
)
register_npu_ci(
est_time=200,
suite="nightly-8-npu-a3",
nightly=True,
disabled="https://github.com/Ascend/sglang/issues/58",
)
class TestQwen3Next(CustomTestCase):
"""
Testcase:Test the Qwen3-Next-80B-A3B-Instruct-W8A8 model with DeepEP's low_latency mode enabled, and verify that
there is no drop in accuracy compared to when DeepEP is not enabled.
[Test Category] Parameter
[Test Target] --moe-a2a-backend deepep, --deepep-mode low_latency
"""
@classmethod
def setUpClass(cls):
cls.model = QWEN3_NEXT_80B_A3B_INSTRUCT_WEIGHTS_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
"--trust-remote-code",
"--attention-backend",
"ascend",
"--device",
"npu",
"--tp-size",
8,
"--mem-fraction-static",
0.8,
"--max-running-requests",
80,
"--watchdog-timeout",
9000,
"--disable-radix-cache",
"--disable-cuda-graph",
"--chunked-prefill-size",
1024,
"--max-prefill-tokens",
28672,
"--max-total-tokens",
450560,
"--moe-a2a-backend",
"deepep",
"--deepep-mode",
"low_latency",
],
env={
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"STREAMS_PER_DEVICE": "32",
"HCCL_OP_EXPANSION_MODE": "AIV",
"HCCL_ALGO": "level0:NA;level1:ring",
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "20",
"HCCL_BUFFSIZE": "2048",
**os.environ,
},
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_mmlu(self):
expect_score = 0.56
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="mmlu",
num_examples=8,
num_threads=32,
)
metrics = run_eval(args)
self.assertGreater(metrics["score"], expect_score)
def test_gsm8k(self):
expect_accuracy = 0.9
args = SimpleNamespace(
num_shots=5,
data_path=None,
num_questions=200,
max_new_tokens=512,
parallel=128,
host="http://127.0.0.1",
port=int(self.base_url.split(":")[-1]),
)
metrics = run_gsm8k(args)
self.assertGreaterEqual(
metrics["accuracy"],
expect_accuracy,
f'Accuracy of {self.model} is {str(metrics["accuracy"])}, is lower than {expect_accuracy}',
)
if __name__ == "__main__":
unittest.main()