chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

Add NPU basic function testcases (#19382)

Browse Source 
Co-authored-by: cy <chenyang08056032@163.com>
Co-authored-by: Cherry_ming <136634645@qq.com>
This commit is contained in:
Sugar920
2026-03-16 15:09:56 +08:00
committed by GitHub
parent e96a3752a0
commit 895e56097c
87 changed files with 4587 additions and 333 deletions
Download Patch File Download Diff File Expand all files Collapse all files

127
test/registered/ascend/basic_function/HiCache/test_npu_hierarchical_cache.py Normal file
View File

@@ -0,0 +1,127 @@
import unittest
import requests
from sglang.srt.utils import kill_process_tree
from sglang.test.ascend.test_ascend_utils import QWEN3_8B_WEIGHTS_PATH
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
popen_launch_server,
)
register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
class TestNPUHierarchicalCache(CustomTestCase):
"""Testcase: HierarchicalCache Test on Ascend NPU.
Cover scenarios:
1. Long identical texts: cache can be reused
2. Short identical texts: cache cannot be reused (page size limit)
3. Different long texts: cache cannot be reused (prefix mismatch)
[Test Category] HiCache
[Test Target] --enable-hierarchical-cache
"""
@classmethod
def setUpClass(cls):
cls.model = QWEN3_8B_WEIGHTS_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
cls.prefill_url = DEFAULT_URL_FOR_TEST
other_args = [
"--attention-backend",
"ascend",
"--disable-cuda-graph",
"--mem-fraction-static",
0.8,
"--tp-size",
1,
"--enable-hierarchical-cache",
]
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=other_args,
)
cls.base_url += "/v1"
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_hierarchical_cache_reused_long_identical(self):
"""Long identical texts should reuse HierarchicalCache"""
# Ultra-long repeated prompt (meets page size requirement)
long_text = "What is The capital of France?" * 36
for i in range(2):
response = requests.post(
f"{DEFAULT_URL_FOR_TEST}/generate",
json={
"text": long_text,
"sampling_params": {
"temperature": 0,
"max_new_tokens": 10,
},
},
)
self.assertEqual(response.status_code, 200)
cached_tokens = int(response.json()["meta_info"]["cached_tokens"])
if i == 0:
# First request: no cache
self.assertEqual(cached_tokens, 0)
else:
# Second request: cache reused
self.assertGreater(cached_tokens, 0)
def test_hierarchical_cache_not_reused_short_identical(self):
"""Short identical texts should NOT reuse HierarchicalCache (page size limit)"""
# Short text prompt (does not meet page size requirement)
short_text = "who am i?"
for i in range(2):
response = requests.post(
f"{DEFAULT_URL_FOR_TEST}/generate",
json={
"text": short_text,
"sampling_params": {
"temperature": 0,
"max_new_tokens": 10,
},
},
)
self.assertEqual(response.status_code, 200)
# No cache reuse for both requests
cached_tokens = int(response.json()["meta_info"]["cached_tokens"])
self.assertEqual(cached_tokens, 0)
def test_hierarchical_cache_not_reused_different_long(self):
"""Different long texts should NOT reuse HierarchicalCache (text uniqueness)"""
# Two different long text prompts (both meet the page size requirement)
texts = [
"Marie ordered one chicken meal that costs $12, 5 packs of milk that costs $3 each, 4 apples that cost $1.50 each, and some boxes of pizza. Marie paid a total of $50. How many boxes of pizza did Marie order if each box costs $8.50?"
* 8,
"Mishka bought 3 pairs of shorts, 3 pairs of pants, and 3 pairs of shoes. One pair of shorts costs $16.50. One pair of pants costs $22.50 and one pair of shoes costs $42. How many dollars did Mishka spend on all the clothing items?"
* 8,
]
for text in texts:
response = requests.post(
f"{DEFAULT_URL_FOR_TEST}/generate",
json={
"text": text,
"sampling_params": {
"temperature": 0,
"max_new_tokens": 10,
},
},
)
self.assertEqual(response.status_code, 200)
# No cache reuse for different text requests
cached_tokens = int(response.json()["meta_info"]["cached_tokens"])
self.assertEqual(cached_tokens, 0)
if __name__ == "__main__":
unittest.main()

97
test/registered/ascend/basic_function/HiCache/test_npu_hierarchical_cache_mla.py Normal file
View File

@@ -0,0 +1,97 @@
import unittest
from sglang.test.ascend.test_ascend_utils import (
DEEPSEEK_R1_0528_W8A8_WEIGHTS_PATH,
run_bench_serving,
)
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.test_utils import CustomTestCase
register_npu_ci(
est_time=400,
suite="nightly-16-npu-a3",
nightly=True,
disabled="run failed",
)
class TestNpuHierarchicalCacheMla(CustomTestCase):
"""The test used the DeepSeek-R1 model, with hierarchical cache enabled, and TTFT improved by 20%.
[Test Category] HiCache
[Test Target] --enable-hierarchical-cache
"""
def test_no_chunked_prefill_without_radix_cache(self):
TTFTS = []
model = DEEPSEEK_R1_0528_W8A8_WEIGHTS_PATH
common_args = [
[
"--trust-remote-code",
"--tp-size",
16,
"--mem-fraction-static",
0.8,
"--max-running-requests",
16,
"--disable-radix-cache",
"--chunked-prefill-size",
"512",
"--disable-cuda-graph",
"--quantization",
"modelslim",
"--attention-backend",
"ascend",
],
[
"--trust-remote-code",
"--tp-size",
16,
"--mem-fraction-static",
0.8,
"--max-running-requests",
16,
"--chunked-prefill-size",
"512",
"--disable-cuda-graph",
"--quantization",
"modelslim",
"--attention-backend",
"ascend",
"--enable-hierarchical-cache",
"--hicache-ratio",
5,
"--hicache-write-policy",
"write_back",
],
]
for common_arg in common_args:
other_args = common_arg + (
[
"--attention-backend",
"ascend",
]
)
res = run_bench_serving(
model=model,
dataset_name="generated-shared-prefix",
num_prompts=128,
random_input_len=3584,
random_output_len=1,
request_rate=float("inf"),
max_concurrency=16,
gsp_num_groups=1,
gsp_prompts_per_group=128,
gsp_system_prompt_len=1792,
gsp_question_len=1792,
gsp_output_len=1,
other_server_args=other_args,
)
TTFT = res["mean_ttft_ms"]
TTFTS.append(TTFT)
assert float(TTFTS[1]) <= 0.8 * float(TTFTS[0])
if __name__ == "__main__":
unittest.main()

68
test/registered/ascend/basic_function/HiCache/test_npu_hierarchical_cache_mutually_exclusive.py Normal file
View File

@@ -0,0 +1,68 @@
import os
import unittest
from sglang.test.ascend.test_ascend_utils import QWEN3_8B_WEIGHTS_PATH
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
popen_launch_server,
)
register_npu_ci(
est_time=400,
suite="nightly-1-npu-a3",
nightly=True,
)
class TestNpuHierarchicalCacheMutuallyExclusive(CustomTestCase):
"""Testcase: The test parameter disable-radix-cache and enable-hierarchical-cache
are mutually exclusive and cannot be used simultaneously.
[Test Category] HiCache
[Test Target] --disable-radix-cache; --enable-hierarchical-cache
"""
def test_hierarchical_cache_mutually_exclusive(self):
error_message = (
"The arguments enable-hierarchical-cache and disable-radix-cache are mutually exclusive and "
"cannot be used at the same time. Please use only one of them."
)
other_args = [
"--attention-backend",
"ascend",
"--disable-cuda-graph",
"--mem-fraction-static",
0.8,
"--tp-size",
2,
"--enable-hierarchical-cache",
"--disable-radix-cache",
]
out_log_file = open("./cache_out_log.txt", "w+", encoding="utf-8")
err_log_file = open("./cache_err_log.txt", "w+", encoding="utf-8")
try:
popen_launch_server(
QWEN3_8B_WEIGHTS_PATH,
DEFAULT_URL_FOR_TEST,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=other_args,
return_stdout_stderr=(out_log_file, err_log_file),
)
except Exception as e:
print(f"Server launch failed as expects:{e}")
finally:
err_log_file.seek(0)
content = err_log_file.read()
# error_message information is recorded in the error log
self.assertIn(error_message, content)
out_log_file.close()
err_log_file.close()
os.remove("./cache_out_log.txt")
os.remove("./cache_err_log.txt")
if __name__ == "__main__":
unittest.main()

89
test/registered/ascend/basic_function/HiCache/test_npu_hierarchical_cache_ttft_mha.py Normal file
View File

@@ -0,0 +1,89 @@
import unittest
from sglang.test.ascend.test_ascend_utils import (
QWEN3_32B_WEIGHTS_PATH,
run_bench_serving,
)
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.test_utils import CustomTestCase
register_npu_ci(
est_time=400,
suite="nightly-2-npu-a3",
nightly=True,
disabled="run failed",
)
class TestNpuHierarchicalCacheTTFT(CustomTestCase):
"""The test used the Qwen3-32B model, with hierarchical cache enabled, and TTFT improved by 40%.
[Test Category] HiCache
[Test Target] --enable-hierarchical-cache
"""
def test_no_chunked_prefill_without_radix_cache(self):
TTFTS = []
model = QWEN3_32B_WEIGHTS_PATH
common_args = [
[
"--trust-remote-code",
"--tp-size",
2,
"--mem-fraction-static",
0.8,
"--max-running-requests",
16,
"--disable-radix-cache",
"--chunked-prefill-size",
"-1",
"--disable-cuda-graph",
],
[
"--trust-remote-code",
"--tp-size",
2,
"--mem-fraction-static",
0.8,
"--max-running-requests",
16,
"--chunked-prefill-size",
"-1",
"--disable-cuda-graph",
"--enable-hierarchical-cache",
"--hicache-ratio",
5,
"--hicache-write-policy",
"write_back",
],
]
for common_arg in common_args:
other_args = common_arg + (
[
"--attention-backend",
"ascend",
]
)
res = run_bench_serving(
model=model,
dataset_name="generated-shared-prefix",
num_prompts=128,
random_input_len=3584,
random_output_len=1,
request_rate=float("inf"),
max_concurrency=16,
gsp_num_groups=1,
gsp_prompts_per_group=128,
gsp_system_prompt_len=1792,
gsp_question_len=1792,
gsp_output_len=1,
other_server_args=other_args,
)
TTFT = res["mean_ttft_ms"]
TTFTS.append(TTFT)
assert float(TTFTS[1]) <= 0.6 * float(TTFTS[0])
if __name__ == "__main__":
unittest.main()

132
test/registered/ascend/basic_function/HiCache/test_npu_radix_cache.py Normal file
View File

@@ -0,0 +1,132 @@
import unittest
import requests
from sglang.srt.utils import kill_process_tree
from sglang.test.ascend.test_ascend_utils import QWEN3_8B_WEIGHTS_PATH
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
popen_launch_server,
)
register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
class TestNPURadixCache(CustomTestCase):
"""Testcase: RadixCache Test on Ascend NPU.
Cover scenarios:
1. Long identical texts: cache can be reused
2. Short identical texts: cache cannot be reused (page size limit)
3. Different long texts: cache cannot be reused (prefix mismatch)
[Test Category] HiCache
"""
@classmethod
def setUpClass(cls):
cls.model = QWEN3_8B_WEIGHTS_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
other_args = [
"--attention-backend",
"ascend",
"--disable-cuda-graph",
"--mem-fraction-static",
0.8,
"--tp-size",
1,
]
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=other_args,
)
cls.base_url += "/v1"
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def tearDown(self):
try:
# Call the '/flush_cache' interface to clear RadixCache.
response = requests.post(f"{DEFAULT_URL_FOR_TEST}/flush_cache")
self.assertEqual(response.status_code, 200, "Failed to flush cache")
except Exception as e:
self.fail(f"Flush cache failed with error: {str(e)}")
def test_radix_cache_reused_long_identical(self):
"""Long identical texts should reuse RadixCache"""
# Ultra-long repeated prompt (meets page size requirement)
long_text = "What is The capital of France?" * 36
for i in range(2):
response = requests.post(
f"{DEFAULT_URL_FOR_TEST}/generate",
json={
"text": long_text,
"sampling_params": {
"temperature": 0,
"max_new_tokens": 10,
},
},
)
self.assertEqual(response.status_code, 200)
cached_tokens = int(response.json()["meta_info"]["cached_tokens"])
if i == 0:
# First request: no cache
self.assertEqual(cached_tokens, 0)
else:
# Second request: cache reused
self.assertGreater(cached_tokens, 0)
def test_radix_cache_not_reused_short_identical(self):
"""Short identical texts should NOT reuse RadixCache (page size limit)"""
# Short text prompt (does not meet page size requirement)
short_text = "who am i?"
for _ in range(2):
response = requests.post(
f"{DEFAULT_URL_FOR_TEST}/generate",
json={
"text": short_text,
"sampling_params": {
"temperature": 0,
"max_new_tokens": 10,
},
},
)
self.assertEqual(response.status_code, 200)
# No cache reuse for both requests
cached_tokens = int(response.json()["meta_info"]["cached_tokens"])
self.assertEqual(cached_tokens, 0)
def test_radix_cache_not_reused_different_long(self):
"""Different long texts should NOT reuse RadixCache (text uniqueness)"""
# Two different long text prompts (both meet the page size requirement)
texts = [
"Marie ordered one chicken meal that costs $12, 5 packs of milk that costs $3 each, 4 apples that cost $1.50 each, and some boxes of pizza. Marie paid a total of $50. How many boxes of pizza did Marie order if each box costs $8.50?"
* 8,
"Mishka bought 3 pairs of shorts, 3 pairs of pants, and 3 pairs of shoes. One pair of shorts costs $16.50. One pair of pants costs $22.50 and one pair of shoes costs $42. How many dollars did Mishka spend on all the clothing items?"
* 8,
]
for text in texts:
response = requests.post(
f"{DEFAULT_URL_FOR_TEST}/generate",
json={
"text": text,
"sampling_params": {
"temperature": 0,
"max_new_tokens": 10,
},
},
)
self.assertEqual(response.status_code, 200)
# No cache reuse for different text requests
cached_tokens = int(response.json()["meta_info"]["cached_tokens"])
self.assertEqual(cached_tokens, 0)
if __name__ == "__main__":
unittest.main()