chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 11 Packages Projects Releases Wiki Activity

Add NPU basic function testcases (#19382)

Browse Source 
Co-authored-by: cy <chenyang08056032@163.com>
Co-authored-by: Cherry_ming <136634645@qq.com>
This commit is contained in:
Sugar920
2026-03-16 15:09:56 +08:00
committed by GitHub
parent e96a3752a0
commit 895e56097c
87 changed files with 4587 additions and 333 deletions
Download Patch File Download Diff File Expand all files Collapse all files

127
test/registered/ascend/basic_function/HiCache/test_npu_hierarchical_cache.py Normal file
View File

@@ -0,0 +1,127 @@
import unittest
import requests
from sglang.srt.utils import kill_process_tree
from sglang.test.ascend.test_ascend_utils import QWEN3_8B_WEIGHTS_PATH
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
popen_launch_server,
)
register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
class TestNPUHierarchicalCache(CustomTestCase):
"""Testcase: HierarchicalCache Test on Ascend NPU.
Cover scenarios:
1. Long identical texts: cache can be reused
2. Short identical texts: cache cannot be reused (page size limit)
3. Different long texts: cache cannot be reused (prefix mismatch)
[Test Category] HiCache
[Test Target] --enable-hierarchical-cache
"""
@classmethod
def setUpClass(cls):
cls.model = QWEN3_8B_WEIGHTS_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
cls.prefill_url = DEFAULT_URL_FOR_TEST
other_args = [
"--attention-backend",
"ascend",
"--disable-cuda-graph",
"--mem-fraction-static",
0.8,
"--tp-size",
1,
"--enable-hierarchical-cache",
]
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=other_args,
)
cls.base_url += "/v1"
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_hierarchical_cache_reused_long_identical(self):
"""Long identical texts should reuse HierarchicalCache"""
# Ultra-long repeated prompt (meets page size requirement)
long_text = "What is The capital of France?" * 36
for i in range(2):
response = requests.post(
f"{DEFAULT_URL_FOR_TEST}/generate",
json={
"text": long_text,
"sampling_params": {
"temperature": 0,
"max_new_tokens": 10,
},
},
)
self.assertEqual(response.status_code, 200)
cached_tokens = int(response.json()["meta_info"]["cached_tokens"])
if i == 0:
# First request: no cache
self.assertEqual(cached_tokens, 0)
else:
# Second request: cache reused
self.assertGreater(cached_tokens, 0)
def test_hierarchical_cache_not_reused_short_identical(self):
"""Short identical texts should NOT reuse HierarchicalCache (page size limit)"""
# Short text prompt (does not meet page size requirement)
short_text = "who am i?"
for i in range(2):
response = requests.post(
f"{DEFAULT_URL_FOR_TEST}/generate",
json={
"text": short_text,
"sampling_params": {
"temperature": 0,
"max_new_tokens": 10,
},
},
)
self.assertEqual(response.status_code, 200)
# No cache reuse for both requests
cached_tokens = int(response.json()["meta_info"]["cached_tokens"])
self.assertEqual(cached_tokens, 0)
def test_hierarchical_cache_not_reused_different_long(self):
"""Different long texts should NOT reuse HierarchicalCache (text uniqueness)"""
# Two different long text prompts (both meet the page size requirement)
texts = [
"Marie ordered one chicken meal that costs $12, 5 packs of milk that costs $3 each, 4 apples that cost $1.50 each, and some boxes of pizza. Marie paid a total of $50. How many boxes of pizza did Marie order if each box costs $8.50?"
* 8,
"Mishka bought 3 pairs of shorts, 3 pairs of pants, and 3 pairs of shoes. One pair of shorts costs $16.50. One pair of pants costs $22.50 and one pair of shoes costs $42. How many dollars did Mishka spend on all the clothing items?"
* 8,
]
for text in texts:
response = requests.post(
f"{DEFAULT_URL_FOR_TEST}/generate",
json={
"text": text,
"sampling_params": {
"temperature": 0,
"max_new_tokens": 10,
},
},
)
self.assertEqual(response.status_code, 200)
# No cache reuse for different text requests
cached_tokens = int(response.json()["meta_info"]["cached_tokens"])
self.assertEqual(cached_tokens, 0)
if __name__ == "__main__":
unittest.main()

97
test/registered/ascend/basic_function/HiCache/test_npu_hierarchical_cache_mla.py Normal file
View File

@@ -0,0 +1,97 @@
import unittest
from sglang.test.ascend.test_ascend_utils import (
DEEPSEEK_R1_0528_W8A8_WEIGHTS_PATH,
run_bench_serving,
)
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.test_utils import CustomTestCase
register_npu_ci(
est_time=400,
suite="nightly-16-npu-a3",
nightly=True,
disabled="run failed",
)
class TestNpuHierarchicalCacheMla(CustomTestCase):
"""The test used the DeepSeek-R1 model, with hierarchical cache enabled, and TTFT improved by 20%.
[Test Category] HiCache
[Test Target] --enable-hierarchical-cache
"""
def test_no_chunked_prefill_without_radix_cache(self):
TTFTS = []
model = DEEPSEEK_R1_0528_W8A8_WEIGHTS_PATH
common_args = [
[
"--trust-remote-code",
"--tp-size",
16,
"--mem-fraction-static",
0.8,
"--max-running-requests",
16,
"--disable-radix-cache",
"--chunked-prefill-size",
"512",
"--disable-cuda-graph",
"--quantization",
"modelslim",
"--attention-backend",
"ascend",
],
[
"--trust-remote-code",
"--tp-size",
16,
"--mem-fraction-static",
0.8,
"--max-running-requests",
16,
"--chunked-prefill-size",
"512",
"--disable-cuda-graph",
"--quantization",
"modelslim",
"--attention-backend",
"ascend",
"--enable-hierarchical-cache",
"--hicache-ratio",
5,
"--hicache-write-policy",
"write_back",
],
]
for common_arg in common_args:
other_args = common_arg + (
[
"--attention-backend",
"ascend",
]
)
res = run_bench_serving(
model=model,
dataset_name="generated-shared-prefix",
num_prompts=128,
random_input_len=3584,
random_output_len=1,
request_rate=float("inf"),
max_concurrency=16,
gsp_num_groups=1,
gsp_prompts_per_group=128,
gsp_system_prompt_len=1792,
gsp_question_len=1792,
gsp_output_len=1,
other_server_args=other_args,
)
TTFT = res["mean_ttft_ms"]
TTFTS.append(TTFT)
assert float(TTFTS[1]) <= 0.8 * float(TTFTS[0])
if __name__ == "__main__":
unittest.main()

68
test/registered/ascend/basic_function/HiCache/test_npu_hierarchical_cache_mutually_exclusive.py Normal file
View File

@@ -0,0 +1,68 @@
import os
import unittest
from sglang.test.ascend.test_ascend_utils import QWEN3_8B_WEIGHTS_PATH
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
popen_launch_server,
)
register_npu_ci(
est_time=400,
suite="nightly-1-npu-a3",
nightly=True,
)
class TestNpuHierarchicalCacheMutuallyExclusive(CustomTestCase):
"""Testcase: The test parameter disable-radix-cache and enable-hierarchical-cache
are mutually exclusive and cannot be used simultaneously.
[Test Category] HiCache
[Test Target] --disable-radix-cache; --enable-hierarchical-cache
"""
def test_hierarchical_cache_mutually_exclusive(self):
error_message = (
"The arguments enable-hierarchical-cache and disable-radix-cache are mutually exclusive and "
"cannot be used at the same time. Please use only one of them."
)
other_args = [
"--attention-backend",
"ascend",
"--disable-cuda-graph",
"--mem-fraction-static",
0.8,
"--tp-size",
2,
"--enable-hierarchical-cache",
"--disable-radix-cache",
]
out_log_file = open("./cache_out_log.txt", "w+", encoding="utf-8")
err_log_file = open("./cache_err_log.txt", "w+", encoding="utf-8")
try:
popen_launch_server(
QWEN3_8B_WEIGHTS_PATH,
DEFAULT_URL_FOR_TEST,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=other_args,
return_stdout_stderr=(out_log_file, err_log_file),
)
except Exception as e:
print(f"Server launch failed as expects:{e}")
finally:
err_log_file.seek(0)
content = err_log_file.read()
# error_message information is recorded in the error log
self.assertIn(error_message, content)
out_log_file.close()
err_log_file.close()
os.remove("./cache_out_log.txt")
os.remove("./cache_err_log.txt")
if __name__ == "__main__":
unittest.main()

89
test/registered/ascend/basic_function/HiCache/test_npu_hierarchical_cache_ttft_mha.py Normal file
View File

@@ -0,0 +1,89 @@
import unittest
from sglang.test.ascend.test_ascend_utils import (
QWEN3_32B_WEIGHTS_PATH,
run_bench_serving,
)
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.test_utils import CustomTestCase
register_npu_ci(
est_time=400,
suite="nightly-2-npu-a3",
nightly=True,
disabled="run failed",
)
class TestNpuHierarchicalCacheTTFT(CustomTestCase):
"""The test used the Qwen3-32B model, with hierarchical cache enabled, and TTFT improved by 40%.
[Test Category] HiCache
[Test Target] --enable-hierarchical-cache
"""
def test_no_chunked_prefill_without_radix_cache(self):
TTFTS = []
model = QWEN3_32B_WEIGHTS_PATH
common_args = [
[
"--trust-remote-code",
"--tp-size",
2,
"--mem-fraction-static",
0.8,
"--max-running-requests",
16,
"--disable-radix-cache",
"--chunked-prefill-size",
"-1",
"--disable-cuda-graph",
],
[
"--trust-remote-code",
"--tp-size",
2,
"--mem-fraction-static",
0.8,
"--max-running-requests",
16,
"--chunked-prefill-size",
"-1",
"--disable-cuda-graph",
"--enable-hierarchical-cache",
"--hicache-ratio",
5,
"--hicache-write-policy",
"write_back",
],
]
for common_arg in common_args:
other_args = common_arg + (
[
"--attention-backend",
"ascend",
]
)
res = run_bench_serving(
model=model,
dataset_name="generated-shared-prefix",
num_prompts=128,
random_input_len=3584,
random_output_len=1,
request_rate=float("inf"),
max_concurrency=16,
gsp_num_groups=1,
gsp_prompts_per_group=128,
gsp_system_prompt_len=1792,
gsp_question_len=1792,
gsp_output_len=1,
other_server_args=other_args,
)
TTFT = res["mean_ttft_ms"]
TTFTS.append(TTFT)
assert float(TTFTS[1]) <= 0.6 * float(TTFTS[0])
if __name__ == "__main__":
unittest.main()

132
test/registered/ascend/basic_function/HiCache/test_npu_radix_cache.py Normal file
View File

@@ -0,0 +1,132 @@
import unittest
import requests
from sglang.srt.utils import kill_process_tree
from sglang.test.ascend.test_ascend_utils import QWEN3_8B_WEIGHTS_PATH
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
popen_launch_server,
)
register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
class TestNPURadixCache(CustomTestCase):
"""Testcase: RadixCache Test on Ascend NPU.
Cover scenarios:
1. Long identical texts: cache can be reused
2. Short identical texts: cache cannot be reused (page size limit)
3. Different long texts: cache cannot be reused (prefix mismatch)
[Test Category] HiCache
"""
@classmethod
def setUpClass(cls):
cls.model = QWEN3_8B_WEIGHTS_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
other_args = [
"--attention-backend",
"ascend",
"--disable-cuda-graph",
"--mem-fraction-static",
0.8,
"--tp-size",
1,
]
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=other_args,
)
cls.base_url += "/v1"
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def tearDown(self):
try:
# Call the '/flush_cache' interface to clear RadixCache.
response = requests.post(f"{DEFAULT_URL_FOR_TEST}/flush_cache")
self.assertEqual(response.status_code, 200, "Failed to flush cache")
except Exception as e:
self.fail(f"Flush cache failed with error: {str(e)}")
def test_radix_cache_reused_long_identical(self):
"""Long identical texts should reuse RadixCache"""
# Ultra-long repeated prompt (meets page size requirement)
long_text = "What is The capital of France?" * 36
for i in range(2):
response = requests.post(
f"{DEFAULT_URL_FOR_TEST}/generate",
json={
"text": long_text,
"sampling_params": {
"temperature": 0,
"max_new_tokens": 10,
},
},
)
self.assertEqual(response.status_code, 200)
cached_tokens = int(response.json()["meta_info"]["cached_tokens"])
if i == 0:
# First request: no cache
self.assertEqual(cached_tokens, 0)
else:
# Second request: cache reused
self.assertGreater(cached_tokens, 0)
def test_radix_cache_not_reused_short_identical(self):
"""Short identical texts should NOT reuse RadixCache (page size limit)"""
# Short text prompt (does not meet page size requirement)
short_text = "who am i?"
for _ in range(2):
response = requests.post(
f"{DEFAULT_URL_FOR_TEST}/generate",
json={
"text": short_text,
"sampling_params": {
"temperature": 0,
"max_new_tokens": 10,
},
},
)
self.assertEqual(response.status_code, 200)
# No cache reuse for both requests
cached_tokens = int(response.json()["meta_info"]["cached_tokens"])
self.assertEqual(cached_tokens, 0)
def test_radix_cache_not_reused_different_long(self):
"""Different long texts should NOT reuse RadixCache (text uniqueness)"""
# Two different long text prompts (both meet the page size requirement)
texts = [
"Marie ordered one chicken meal that costs $12, 5 packs of milk that costs $3 each, 4 apples that cost $1.50 each, and some boxes of pizza. Marie paid a total of $50. How many boxes of pizza did Marie order if each box costs $8.50?"
* 8,
"Mishka bought 3 pairs of shorts, 3 pairs of pants, and 3 pairs of shoes. One pair of shorts costs $16.50. One pair of pants costs $22.50 and one pair of shoes costs $42. How many dollars did Mishka spend on all the clothing items?"
* 8,
]
for text in texts:
response = requests.post(
f"{DEFAULT_URL_FOR_TEST}/generate",
json={
"text": text,
"sampling_params": {
"temperature": 0,
"max_new_tokens": 10,
},
},
)
self.assertEqual(response.status_code, 200)
# No cache reuse for different text requests
cached_tokens = int(response.json()["meta_info"]["cached_tokens"])
self.assertEqual(cached_tokens, 0)
if __name__ == "__main__":
unittest.main()

197
test/registered/ascend/basic_function/interface/test_ascend_enable_thinking.py
View File

@@ -1,197 +0,0 @@
import json
import unittest
import requests
from sglang.srt.utils import kill_process_tree
from sglang.test.ascend.test_ascend_utils import QWEN3_30B_A3B_WEIGHTS_PATH
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
popen_launch_server,
)
register_npu_ci(est_time=400, suite="nightly-2-npu-a3", nightly=True)
class TestEnableThinking(CustomTestCase):
"""Testcase: Testing with the 'enable_thinking' feature enabled/disabled,
both streaming and non-streaming input requests successful
[Test Category] Interface
[Test Target] /v1/chat/completions
"""
@classmethod
def setUpClass(cls):
cls.model = QWEN3_30B_A3B_WEIGHTS_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
cls.api_key = "sk-1234"
cls.other_args = [
"--reasoning-parser",
"qwen3",
"--attention-backend",
"ascend",
"--disable-cuda-graph",
"--mem-fraction-static",
0.95,
"--tp",
2,
]
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
api_key=cls.api_key,
other_args=cls.other_args,
)
cls.additional_chat_kwargs = {}
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_chat_completion_with_reasoning(self):
# Test non-streaming with "enable_thinking": True, reasoning_content should not be empty
client = requests.post(
f"{self.base_url}/v1/chat/completions",
headers={"Authorization": f"Bearer {self.api_key}"},
json={
"model": self.model,
"messages": [{"role": "user", "content": "Hello"}],
"temperature": 0,
"separate_reasoning": True,
"chat_template_kwargs": {"enable_thinking": True},
**self.additional_chat_kwargs,
},
)
self.assertEqual(client.status_code, 200, f"Failed with: {client.text}")
data = client.json()
self.assertIn("choices", data)
self.assertTrue(len(data["choices"]) > 0)
self.assertIn("message", data["choices"][0])
self.assertIn("reasoning_content", data["choices"][0]["message"])
self.assertIsNotNone(data["choices"][0]["message"]["reasoning_content"])
def test_chat_completion_without_reasoning(self):
# Test non-streaming with "enable_thinking": False, reasoning_content should be empty
client = requests.post(
f"{self.base_url}/v1/chat/completions",
headers={"Authorization": f"Bearer {self.api_key}"},
json={
"model": self.model,
"messages": [{"role": "user", "content": "Hello"}],
"temperature": 0,
"separate_reasoning": True,
"chat_template_kwargs": {"enable_thinking": False},
**self.additional_chat_kwargs,
},
)
self.assertEqual(client.status_code, 200, f"Failed with: {client.text}")
data = client.json()
self.assertIn("choices", data)
self.assertTrue(len(data["choices"]) > 0)
self.assertIn("message", data["choices"][0])
if "reasoning_content" in data["choices"][0]["message"]:
self.assertIsNone(data["choices"][0]["message"]["reasoning_content"])
def test_stream_chat_completion_with_reasoning(self):
# Test streaming with "enable_thinking": True, reasoning_content should not be empty
response = requests.post(
f"{self.base_url}/v1/chat/completions",
headers={"Authorization": f"Bearer {self.api_key}"},
json={
"model": self.model,
"messages": [{"role": "user", "content": "Hello"}],
"temperature": 0,
"separate_reasoning": True,
"stream": True,
"chat_template_kwargs": {"enable_thinking": True},
**self.additional_chat_kwargs,
},
stream=True,
)
self.assertEqual(response.status_code, 200, f"Failed with: {response.text}")
has_reasoning = False
has_content = False
print("\n=== Stream With Reasoning ===")
for line in response.iter_lines():
if line:
line = line.decode("utf-8")
if line.startswith("data:") and not line.startswith("data: [DONE]"):
data = json.loads(line[6:])
if "choices" in data and len(data["choices"]) > 0:
delta = data["choices"][0].get("delta", {})
if "reasoning_content" in delta and delta["reasoning_content"]:
has_reasoning = True
if "content" in delta and delta["content"]:
has_content = True
self.assertTrue(
has_reasoning,
"The reasoning content is not included in the stream response",
)
self.assertTrue(
has_content, "The stream response does not contain normal content"
)
def test_stream_chat_completion_without_reasoning(self):
# Test streaming with "enable_thinking": False, reasoning_content should be empty
response = requests.post(
f"{self.base_url}/v1/chat/completions",
headers={"Authorization": f"Bearer {self.api_key}"},
json={
"model": self.model,
"messages": [{"role": "user", "content": "Hello"}],
"temperature": 0,
"separate_reasoning": True,
"stream": True,
"chat_template_kwargs": {"enable_thinking": False},
**self.additional_chat_kwargs,
},
stream=True,
)
self.assertEqual(response.status_code, 200, f"Failed with: {response.text}")
has_reasoning = False
has_content = False
print("\n=== Stream Without Reasoning ===")
for line in response.iter_lines():
if line:
line = line.decode("utf-8")
if line.startswith("data:") and not line.startswith("data: [DONE]"):
data = json.loads(line[6:])
if "choices" in data and len(data["choices"]) > 0:
delta = data["choices"][0].get("delta", {})
if "reasoning_content" in delta and delta["reasoning_content"]:
has_reasoning = True
if "content" in delta and delta["content"]:
has_content = True
self.assertFalse(
has_reasoning,
"The reasoning content should not be included in the stream response",
)
self.assertTrue(
has_content, "The stream response does not contain normal content"
)
if __name__ == "__main__":
unittest.main()

108
test/registered/ascend/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_auto_deepseek_v3_2_w8a8.py Normal file
View File

@@ -0,0 +1,108 @@
import os
import unittest
from types import SimpleNamespace
from sglang.srt.utils import kill_process_tree
from sglang.test.ascend.test_ascend_utils import DEEPSEEK_V3_2_W8A8_WEIGHTS_PATH
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.few_shot_gsm8k import run_eval as run_gsm8k
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_URL_FOR_TEST,
CustomTestCase,
popen_launch_server,
)
register_npu_ci(est_time=400, suite="nightly-16-npu-a3", nightly=True)
class TestDeepEpDeepseekV32(CustomTestCase):
"""Testcase: Verify that for the DeepSeek V3.2 model in the single-machine colocation scenario,
its inference accuracy on the MMLU and GSM8K dataset meets the preset standard when the parameter --deepep-mode auto is configured.
[Test Category] Expert Parallelism
[Test Target] --moe-a2a-backend deepep;--deepep-mode
"""
@classmethod
def setUpClass(cls):
cls.model = DEEPSEEK_V3_2_W8A8_WEIGHTS_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=6000,
other_args=[
"--trust-remote-code",
"--tp-size",
"16",
"--quantization",
"modelslim",
"--moe-a2a-backend",
"deepep",
"--deepep-mode",
"auto",
"--mem-fraction-static",
0.82,
"--disable-cuda-graph",
"--disable-radix-cache",
"--context-length",
40960,
"--max-prefill-tokens",
40960,
"--max-total-tokens",
40960,
],
env={
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"STREAMS_PER_DEVICE": "32",
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "16",
"HCCL_BUFFSIZE": "1600",
"HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_NPU_USE_MLAPO": "0",
"SGLANG_NPU_USE_MULTI_STREAM": "1",
"TASK_QUEUE_ENABLE": "0",
**os.environ,
},
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_mmlu(self):
expect_score = 0.85
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="mmlu",
num_examples=128,
num_threads=32,
)
print("Starting mmlu test...")
metrics = run_eval(args)
self.assertGreater(metrics["score"], expect_score)
def test_gsm8k(self):
expect_accuracy = 0.95
args = SimpleNamespace(
num_shots=8,
data_path=None,
timeout=60000,
num_questions=200,
max_new_tokens=512,
parallel=128,
host="http://127.0.0.1",
port=int(self.base_url.split(":")[-1]),
)
print("Starting gsm8k test...")
metrics = run_gsm8k(args)
self.assertGreaterEqual(
metrics["accuracy"],
expect_accuracy,
f'Accuracy of {self.model} is {str(metrics["accuracy"])}, is lower than {expect_accuracy}',
)
if __name__ == "__main__":
unittest.main()

128
test/registered/ascend/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_auto_qwen3_480b.py Normal file
View File

@@ -0,0 +1,128 @@
import os
import unittest
from types import SimpleNamespace
from sglang.srt.utils import kill_process_tree
from sglang.test.ascend.test_ascend_utils import (
QWEN3_CODER_480B_A35B_INSTRUCT_W8A8_QUAROT_WEIGHTS_PATH,
)
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.few_shot_gsm8k import run_eval as run_gsm8k
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
popen_launch_server,
)
register_npu_ci(est_time=200, suite="nightly-16-npu-a3", nightly=True)
class TestDeepEpQwen(CustomTestCase):
"""
Testcase:Test the Qwen3-Coder-480B-A35B-Instruct-w8a8-QuaRot model with DeepEP's auto mode enabled,
and verify that there is no drop in accuracy compared to when DeepEP is not enabled.
[Test Category] Expert Parallelism
[Test Target] --moe-a2a-backend, --deepep-mode
"""
@classmethod
def setUpClass(cls):
cls.model = QWEN3_CODER_480B_A35B_INSTRUCT_W8A8_QUAROT_WEIGHTS_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
"--trust-remote-code",
"--nnodes",
"1",
"--node-rank",
"0",
"--attention-backend",
"ascend",
"--device",
"npu",
"--quantization",
"modelslim",
"--max-running-requests",
96,
"--context-length",
8192,
"--dtype",
"bfloat16",
"--chunked-prefill-size",
28672,
"--max-prefill-tokens",
458880,
"--disable-radix-cache",
"--moe-a2a-backend",
"deepep",
"--deepep-mode",
"auto",
"--tp-size",
16,
"--dp-size",
4,
"--enable-dp-attention",
"--enable-dp-lm-head",
"--mem-fraction-static",
0.7,
"--cuda-graph-bs",
16,
20,
24,
],
env={
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600",
"HCCL_BUFFSIZE": "2100",
"HCCL_OP_EXPANSION_MODE": "AIV",
**os.environ,
},
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_mmlu(self):
expect_score = 0.61
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="mmlu",
num_examples=8,
num_threads=32,
)
metrics = run_eval(args)
self.assertGreater(metrics["score"], expect_score)
def test_gsm8k(self):
expect_accuracy = 0.91
host = "http://127.0.0.1"
port = int(self.base_url.split(":")[-1])
args = SimpleNamespace(
num_shots=8,
data_path=None,
num_questions=200,
max_new_tokens=512,
parallel=128,
host=host,
port=port,
)
metrics = run_gsm8k(args)
self.assertGreaterEqual(
metrics["accuracy"],
expect_accuracy,
f'Accuracy of {self.model} is {str(metrics["accuracy"])}, is lower than {expect_accuracy}',
)
if __name__ == "__main__":
unittest.main()

118
test/registered/ascend/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_auto_qwen3_next.py Normal file
View File

@@ -0,0 +1,118 @@
import os
import unittest
from types import SimpleNamespace
from sglang.srt.utils import kill_process_tree
from sglang.test.ascend.test_ascend_utils import (
QWEN3_NEXT_80B_A3B_INSTRUCT_WEIGHTS_PATH,
)
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.few_shot_gsm8k import run_eval as run_gsm8k
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
popen_launch_server,
)
register_npu_ci(
est_time=200,
suite="nightly-8-npu-a3",
nightly=True,
disabled="https://github.com/Ascend/sglang/issues/58",
)
class TestQwen3Next(CustomTestCase):
"""
Testcase:Test the Qwen3-Next-80B-A3B-Instruct-W8A8 model with DeepEP's auto mode enabled, and verify that there is
no drop in accuracy compared to when DeepEP is not enabled.
[Test Category] Parameter
[Test Target] --moe-a2a-backend deepep, --deepep-mode auto
"""
@classmethod
def setUpClass(cls):
cls.model = QWEN3_NEXT_80B_A3B_INSTRUCT_WEIGHTS_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
"--trust-remote-code",
"--attention-backend",
"ascend",
"--device",
"npu",
"--tp-size",
8,
"--mem-fraction-static",
0.8,
"--max-running-requests",
80,
"--watchdog-timeout",
9000,
"--disable-radix-cache",
"--disable-cuda-graph",
"--max-prefill-tokens",
28672,
"--max-total-tokens",
450560,
"--moe-a2a-backend",
"deepep",
"--deepep-mode",
"auto",
"--chunked-prefill-size",
-1,
],
env={
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"STREAMS_PER_DEVICE": "32",
"HCCL_OP_EXPANSION_MODE": "AIV",
"HCCL_ALGO": "level0:NA;level1:ring",
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "20",
"HCCL_BUFFSIZE": "2000",
**os.environ,
},
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_mmlu(self):
expect_score = 0.56
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="mmlu",
num_examples=8,
num_threads=32,
)
metrics = run_eval(args)
self.assertGreater(metrics["score"], expect_score)
def test_gsm8k(self):
expect_accuracy = 0.9
args = SimpleNamespace(
num_shots=5,
data_path=None,
num_questions=200,
max_new_tokens=512,
parallel=128,
host="http://127.0.0.1",
port=int(self.base_url.split(":")[-1]),
)
metrics = run_gsm8k(args)
self.assertGreaterEqual(
metrics["accuracy"],
expect_accuracy,
f'Accuracy of {self.model} is {str(metrics["accuracy"])}, is lower than {expect_accuracy}',
)
if __name__ == "__main__":
unittest.main()

107
test/registered/ascend/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_low_latency_deepseek_v3_2_w8a8.py Normal file
View File

@@ -0,0 +1,107 @@
import os
import unittest
from types import SimpleNamespace
from sglang.srt.utils import kill_process_tree
from sglang.test.ascend.test_ascend_utils import DEEPSEEK_V3_2_W8A8_WEIGHTS_PATH
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.few_shot_gsm8k import run_eval as run_gsm8k
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_URL_FOR_TEST,
CustomTestCase,
popen_launch_server,
)
register_npu_ci(est_time=200, suite="nightly-16-npu-a3", nightly=False)
class TestDeepEpDeepseekV32(CustomTestCase):
"""Testcase: Verify that for the DeepSeek V3.2 model in the single-machine colocation scenario,
its inference accuracy on the MMLU and GSM8K dataset meets the preset standard when the parameter --deepep-mode low_latency is configured.
[Test Category] Expert Parallelism
[Test Target] --moe-a2a-backend deepep;--deepep-mode
[Test Suggestions] Mixing deployment + low_latency mode is not recommended.
"""
@classmethod
def setUpClass(cls):
cls.model = DEEPSEEK_V3_2_W8A8_WEIGHTS_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=6000,
other_args=[
"--trust-remote-code",
"--tp-size",
"16",
"--quantization",
"modelslim",
"--moe-a2a-backend",
"deepep",
"--deepep-mode",
"low_latency",
"--mem-fraction-static",
0.82,
"--disable-cuda-graph",
"--disable-radix-cache",
"--context-length",
40960,
"--max-prefill-tokens",
128,
"--max-total-tokens",
40960,
],
env={
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"STREAMS_PER_DEVICE": "32",
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "128",
"HCCL_BUFFSIZE": "2048",
"HCCL_OP_EXPANSION_MODE": "AIV",
"TASK_QUEUE_ENABLE": "0",
**os.environ,
},
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_mmlu(self):
expect_score = 0.85
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="mmlu",
num_examples=128,
num_threads=32,
)
print("Starting mmlu test...")
metrics = run_eval(args)
self.assertGreater(metrics["score"], expect_score)
def test_gsm8k(self):
expect_accuracy = 0.95
args = SimpleNamespace(
num_shots=8,
data_path=None,
timeout=60000,
num_questions=200,
max_new_tokens=512,
parallel=128,
host="http://127.0.0.1",
port=int(self.base_url.split(":")[-1]),
)
print("Starting gsm8k test...")
metrics = run_gsm8k(args)
self.assertGreaterEqual(
metrics["accuracy"],
expect_accuracy,
f'Accuracy of {self.model} is {str(metrics["accuracy"])}, is lower than {expect_accuracy}',
)
if __name__ == "__main__":
unittest.main()

125
test/registered/ascend/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_low_latency_qwen3_480b.py Normal file
View File

@@ -0,0 +1,125 @@
import os
import unittest
from types import SimpleNamespace
from sglang.srt.utils import kill_process_tree
from sglang.test.ascend.test_ascend_utils import (
QWEN3_CODER_480B_A35B_INSTRUCT_W8A8_QUAROT_WEIGHTS_PATH,
)
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.few_shot_gsm8k import run_eval as run_gsm8k
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
popen_launch_server,
)
register_npu_ci(est_time=200, suite="nightly-16-npu-a3", nightly=False)
class TestDeepEpQwen(CustomTestCase):
"""
Testcase:Test the Qwen3-Coder-480B-A35B-Instruct-w8a8-QuaRot model with DeepEP's low_latency mode enabled,
and verify that there is no drop in accuracy compared to when DeepEP is not enabled.
[Test Category] Expert Parallelism
[Test Target] --moe-a2a-backend, --deepep-mode
[Test Suggestions] Mixing deployment + low_latency mode is not recommended.
"""
@classmethod
def setUpClass(cls):
cls.model = QWEN3_CODER_480B_A35B_INSTRUCT_W8A8_QUAROT_WEIGHTS_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
"--trust-remote-code",
"--nnodes",
"1",
"--node-rank",
"0",
"--attention-backend",
"ascend",
"--device",
"npu",
"--quantization",
"modelslim",
"--max-running-requests",
96,
"--context-length",
8192,
"--dtype",
"bfloat16",
"--chunked-prefill-size",
1024,
"--max-prefill-tokens",
458880,
"--disable-radix-cache",
"--moe-a2a-backend",
"deepep",
"--deepep-mode",
"low_latency",
"--tp-size",
16,
"--dp-size",
4,
"--enable-dp-attention",
"--enable-dp-lm-head",
"--mem-fraction-static",
0.7,
"--cuda-graph-bs",
16,
20,
24,
],
env={
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600",
"HCCL_BUFFSIZE": "2100",
"HCCL_OP_EXPANSION_MODE": "AIV",
**os.environ,
},
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_mmlu(self):
expect_score = 0.61
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="mmlu",
num_examples=8,
num_threads=32,
)
metrics = run_eval(args)
self.assertGreater(metrics["score"], expect_score)
def test_gsm8k(self):
expect_accuracy = 0.91
args = SimpleNamespace(
num_shots=8,
data_path=None,
num_questions=200,
max_new_tokens=512,
parallel=128,
host="http://127.0.0.1",
port=int(self.base_url.split(":")[-1]),
)
metrics = run_gsm8k(args)
self.assertGreaterEqual(
metrics["accuracy"],
expect_accuracy,
f'Accuracy of {self.model} is {str(metrics["accuracy"])}, is lower than {expect_accuracy}',
)
if __name__ == "__main__":
unittest.main()

118
test/registered/ascend/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_low_latency_qwen3_next.py Normal file
View File

@@ -0,0 +1,118 @@
import os
import unittest
from types import SimpleNamespace
from sglang.srt.utils import kill_process_tree
from sglang.test.ascend.test_ascend_utils import (
QWEN3_NEXT_80B_A3B_INSTRUCT_WEIGHTS_PATH,
)
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.few_shot_gsm8k import run_eval as run_gsm8k
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
popen_launch_server,
)
register_npu_ci(
est_time=200,
suite="nightly-8-npu-a3",
nightly=True,
disabled="https://github.com/Ascend/sglang/issues/58",
)
class TestQwen3Next(CustomTestCase):
"""
Testcase:Test the Qwen3-Next-80B-A3B-Instruct-W8A8 model with DeepEP's low_latency mode enabled, and verify that
there is no drop in accuracy compared to when DeepEP is not enabled.
[Test Category] Parameter
[Test Target] --moe-a2a-backend deepep, --deepep-mode low_latency
"""
@classmethod
def setUpClass(cls):
cls.model = QWEN3_NEXT_80B_A3B_INSTRUCT_WEIGHTS_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
"--trust-remote-code",
"--attention-backend",
"ascend",
"--device",
"npu",
"--tp-size",
8,
"--mem-fraction-static",
0.8,
"--max-running-requests",
80,
"--watchdog-timeout",
9000,
"--disable-radix-cache",
"--disable-cuda-graph",
"--chunked-prefill-size",
1024,
"--max-prefill-tokens",
28672,
"--max-total-tokens",
450560,
"--moe-a2a-backend",
"deepep",
"--deepep-mode",
"low_latency",
],
env={
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"STREAMS_PER_DEVICE": "32",
"HCCL_OP_EXPANSION_MODE": "AIV",
"HCCL_ALGO": "level0:NA;level1:ring",
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "20",
"HCCL_BUFFSIZE": "2048",
**os.environ,
},
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_mmlu(self):
expect_score = 0.56
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="mmlu",
num_examples=8,
num_threads=32,
)
metrics = run_eval(args)
self.assertGreater(metrics["score"], expect_score)
def test_gsm8k(self):
expect_accuracy = 0.9
args = SimpleNamespace(
num_shots=5,
data_path=None,
num_questions=200,
max_new_tokens=512,
parallel=128,
host="http://127.0.0.1",
port=int(self.base_url.split(":")[-1]),
)
metrics = run_gsm8k(args)
self.assertGreaterEqual(
metrics["accuracy"],
expect_accuracy,
f'Accuracy of {self.model} is {str(metrics["accuracy"])}, is lower than {expect_accuracy}',
)
if __name__ == "__main__":
unittest.main()

0
test/registered/ascend/basic_function/deepseek_coder.json → test/registered/ascend/basic_function/parameter/deepseek_coder.json
View File

15
test/registered/ascend/basic_function/test_ascend_fim_completion.py → test/registered/ascend/basic_function/parameter/test_npu_fim_completion.py
View File

@@ -4,6 +4,10 @@ import openai
from sglang.srt.utils import kill_process_tree
from sglang.srt.utils.hf_transformers_utils import get_tokenizer
from sglang.test.ascend.test_ascend_utils import (
DEEPSEEK_CODER_1_3_B_BASE_PATH,
DEEPSEEK_CODER_JSON_PATH,
)
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
@@ -12,7 +16,12 @@ from sglang.test.test_utils import (
popen_launch_server,
)
register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
register_npu_ci(
est_time=400,
suite="nightly-1-npu-a3",
nightly=True,
disabled="run failed",
)
class TestFimCompletion(CustomTestCase):
@@ -22,7 +31,7 @@ class TestFimCompletion(CustomTestCase):
[Test Target] --completion-template
"""
model = "/root/.cache/modelscope/hub/models/deepseek-ai/deepseek-coder-1.3b-base"
model = DEEPSEEK_CODER_1_3_B_BASE_PATH
other_args = [
"--completion-template",
"deepseek_coder",
@@ -86,7 +95,7 @@ class TestFimCompletion(CustomTestCase):
class TestFimCompletionJson(TestFimCompletion):
other_args = [
"--completion-template",
"./deepseek_coder.json",
DEEPSEEK_CODER_JSON_PATH,
"--attention-backend",
"ascend",
"--disable-cuda-graph",

0
test/registered/ascend/basic_function/parameter/test_ascend_log_level.py → test/registered/ascend/basic_function/parameter/test_npu_log_level.py
View File

0
test/registered/ascend/basic_function/parameter/test_ascend_no_chunked_prefill.py → test/registered/ascend/basic_function/parameter/test_npu_no_chunked_prefill.py
View File

0
test/registered/ascend/basic_function/parameter/test_ascend_no_overlap_scheduler.py → test/registered/ascend/basic_function/parameter/test_npu_no_overlap_scheduler.py
View File

0
test/registered/ascend/basic_function/parameter/test_ascend_original_logprobs.py → test/registered/ascend/basic_function/parameter/test_npu_original_logprobs.py
View File

0
test/registered/ascend/basic_function/parameter/test_ascend_warmups.py → test/registered/ascend/basic_function/parameter/test_npu_warmups.py
View File

101
test/registered/ascend/basic_function/speculative_inference/test_npu_eagle3.py Normal file
View File

@@ -0,0 +1,101 @@
import os
import unittest
from types import SimpleNamespace
from urllib.parse import urlparse
from sglang.srt.utils import kill_process_tree
from sglang.test.ascend.test_ascend_utils import (
QWEN3_8B_EAGLE3_WEIGHTS_PATH,
QWEN3_8B_WEIGHTS_PATH,
)
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
from sglang.test.test_utils import (
DEFAULT_URL_FOR_TEST,
CustomTestCase,
popen_launch_server,
)
register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
class TestNpuEagle3(CustomTestCase):
"""Testcase: Verify GSM8K inference accuracy ≥0.81 for model with specified EAGLE3 speculative inference parameters.
[Test Category] Speculative Decoding
[Test Target] --speculative-draft-model-quantization; --speculative-algorithm; --speculative-draft-model-path; --speculative-num-steps; --speculative-eagle-topk; --speculative-num-draft-tokens; --speculative-attention-mode
"""
@classmethod
def setUpClass(cls):
cls.model = QWEN3_8B_WEIGHTS_PATH
cls.accuracy = 0.81
cls.base_url = DEFAULT_URL_FOR_TEST
cls.url = urlparse(DEFAULT_URL_FOR_TEST)
cls.common_args = [
"--trust-remote-code",
"--attention-backend",
"ascend",
"--disable-radix-cache",
"--speculative-draft-model-quantization",
"unquant",
"--speculative-algorithm",
"EAGLE3",
"--speculative-draft-model-path",
QWEN3_8B_EAGLE3_WEIGHTS_PATH,
"--speculative-num-steps",
"4",
"--speculative-eagle-topk",
"1",
"--speculative-num-draft-tokens",
"5",
"--speculative-attention-mode",
"decode",
"--tp-size",
"1",
"--mem-fraction-static",
"0.7",
"--disable-cuda-graph",
"--dtype",
"bfloat16",
]
cls.extra_envs = {
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
}
os.environ.update(cls.extra_envs)
def test_gsm8k(self):
process = popen_launch_server(
self.model,
self.base_url,
timeout=1500,
other_args=[
*self.common_args,
],
)
try:
args = SimpleNamespace(
num_shots=5,
data_path=None,
num_questions=1319,
max_new_tokens=512,
parallel=128,
host=f"http://{self.url.hostname}",
port=int(self.url.port),
)
metrics = run_eval_few_shot_gsm8k(args)
self.assertGreaterEqual(
metrics["accuracy"],
self.accuracy,
)
finally:
kill_process_tree(process.pid)
if __name__ == "__main__":
unittest.main()