chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 11 Packages Projects Releases Wiki Activity

[NPU] NPU quantization refactoring & more quantization formats support (#14504)

Browse Source 
Co-authored-by: TamirBaydasov <mr.jeijy@gmail.com>
Co-authored-by: Tamir Baydasov <41994229+TamirBaydasov@users.noreply.github.com>
Co-authored-by: Савкин Артем <savkinartem@MacBook-Air-Viktoria.local>
Co-authored-by: Edward Shogulin <edward.shogulin@gmail.com>
This commit is contained in:
Артем Савкин
2026-01-14 23:25:15 +03:00
committed by GitHub
parent f091858304
commit 424a380077
30 changed files with 1962 additions and 771 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
test/manual/ascend/test_ascend_w8a8_quantization.py
View File

@@ -45,8 +45,6 @@ class TestAscendW8A8(CustomTestCase):
"npu",
"--attention-backend",
"ascend",
"--quantization",
"w8a8_int8",
],
)

2
test/srt/ascend/test_ascend_deepep.py
View File

@@ -34,8 +34,6 @@ class TestAscendDeepEP(CustomTestCase):
"--trust-remote-code",
"--attention-backend",
"ascend",
"--quantization",
"modelslim",
"--mem-fraction-static",
0.8,
"--disable-radix-cache",

2
test/srt/ascend/test_ascend_deepseek_mtp.py
View File

@@ -32,8 +32,6 @@ class TestAscendDeepSeekMTP(CustomTestCase):
"--trust-remote-code",
"--attention-backend",
"ascend",
"--quantization",
"modelslim",
"--mem-fraction-static",
0.8,
"--disable-radix-cache",

2
test/srt/ascend/test_ascend_hicache_mla.py
View File

@@ -35,8 +35,6 @@ class TestAscendMlaHicache(CustomTestCase):
0.8,
"--attention-backend",
"ascend",
"--quantization",
"modelslim",
"--tp-size",
4,
"--enable-hierarchical-cache",

2
test/srt/ascend/test_ascend_mla_fia_w8a8int8.py
View File

@@ -37,8 +37,6 @@ class TestAscendMlaW8A8Int8(CustomTestCase):
0.8,
"--attention-backend",
"ascend",
"--quantization",
"modelslim",
"--tp-size",
2,
"--disable-radix-cache",

2
test/srt/ascend/test_ascend_mla_w8a8int8.py
View File

@@ -36,8 +36,6 @@ class TestAscendMlaW8A8Int8(CustomTestCase):
0.8,
"--attention-backend",
"ascend",
"--quantization",
"modelslim",
"--tp-size",
4,
"--disable-radix-cache",

108
test/srt/ascend/test_ascend_w4a4_quantization.py Normal file
View File

@@ -0,0 +1,108 @@
"""
Usage:
python3 -m unittest test_ascend_w4a4_quantization.TestAscendW4A4.test_gsm8k
"""
import os
import time
import unittest
from types import SimpleNamespace
from urllib.parse import urlparse
import requests
from sglang.srt.utils import kill_process_tree
from sglang.test.few_shot_gsm8k import run_eval
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
is_in_ci,
popen_launch_server,
)
if "ASCEND_RT_VISIBLE_DEVICES" not in os.environ:
os.environ["ASCEND_RT_VISIBLE_DEVICES"] = "0,1,2,3"
DEFAULT_PORT_FOR_SRT_TEST_RUNNER = (
7000 + int(os.environ.get("ASCEND_RT_VISIBLE_DEVICES", "0")[0]) * 100
)
DEFAULT_URL_FOR_TEST = f"http://127.0.0.1:{DEFAULT_PORT_FOR_SRT_TEST_RUNNER + 1000}"
class TestAscendW4A4(CustomTestCase):
@classmethod
def setUpClass(cls):
cls.model = "/root/.cache/modelscope/hub/models/Eco-Tech/Qwen3-32B-w4a4-LAOS"
cls.base_url = DEFAULT_URL_FOR_TEST
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
"--trust-remote-code",
"--device",
"npu",
"--attention-backend",
"ascend",
"--tp-size",
"4",
"--mem-fraction-static",
"0.8",
"--cuda-graph-bs",
"64",
"--disable-radix-cache",
],
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_gsm8k(self):
base_url = DEFAULT_URL_FOR_TEST
url = urlparse(base_url)
args = SimpleNamespace(
num_shots=5,
data_path=None,
num_questions=1319,
max_new_tokens=512,
parallel=64,
host=f"http://{url.hostname}",
port=int(url.port),
)
metrics = run_eval(args)
print(metrics)
self.assertGreaterEqual(metrics["accuracy"], 0.80)
self.assertGreaterEqual(metrics["output_throughput"], 1000)
def run_decode(self, max_new_tokens):
response = requests.post(
self.base_url + "/generate",
json={
"text": "The capital of France is",
"sampling_params": {
"temperature": 0,
"max_new_tokens": max_new_tokens,
},
"ignore_eos": True,
},
)
return response.json()
def test_throughput(self):
max_tokens = 256
tic = time.perf_counter()
res = self.run_decode(max_tokens)
tok = time.perf_counter()
print(res["text"])
throughput = max_tokens / (tok - tic)
print(f"Throughput: {throughput} tokens/s")
if is_in_ci():
self.assertGreaterEqual(throughput, 35)
if __name__ == "__main__":
unittest.main()

103
test/srt/ascend/test_ascend_w8a8_quantization.py Normal file
View File

@@ -0,0 +1,103 @@
"""
Usage:
python3 -m unittest test_ascend_w8a8_quantization.TestAscendW8A8.test_gsm8k
"""
import os
import time
import unittest
from types import SimpleNamespace
from urllib.parse import urlparse
import requests
from sglang.srt.utils import kill_process_tree
from sglang.test.few_shot_gsm8k import run_eval
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
is_in_ci,
popen_launch_server,
)
if "ASCEND_RT_VISIBLE_DEVICES" not in os.environ:
os.environ["ASCEND_RT_VISIBLE_DEVICES"] = "0,1"
DEFAULT_PORT_FOR_SRT_TEST_RUNNER = (
7000 + int(os.environ.get("ASCEND_RT_VISIBLE_DEVICES", "0")[0]) * 100
)
DEFAULT_URL_FOR_TEST = f"http://127.0.0.1:{DEFAULT_PORT_FOR_SRT_TEST_RUNNER + 1000}"
class TestAscendW8A8CompressedTensors(CustomTestCase):
@classmethod
def setUpClass(cls):
# TODO: Move model to CI or Modelscope
cls.model = "RedHatAI/Qwen2.5-0.5B-Instruct-quantized.w8a8"
cls.base_url = DEFAULT_URL_FOR_TEST
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
"--trust-remote-code",
"--disable-cuda-graph",
"--device",
"npu",
"--attention-backend",
"ascend",
],
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_gsm8k(self):
base_url = DEFAULT_URL_FOR_TEST
url = urlparse(base_url)
args = SimpleNamespace(
num_shots=5,
data_path=None,
num_questions=200,
max_new_tokens=512,
parallel=128,
host=f"http://{url.hostname}",
port=int(url.port),
)
metrics = run_eval(args)
print(metrics)
self.assertGreaterEqual(metrics["accuracy"], 0.3)
self.assertGreaterEqual(metrics["output_throughput"], 700)
def run_decode(self, max_new_tokens):
response = requests.post(
self.base_url + "/generate",
json={
"text": "The capital of France is",
"sampling_params": {
"temperature": 0,
"max_new_tokens": max_new_tokens,
},
"ignore_eos": True,
},
)
return response.json()
def test_throughput(self):
max_tokens = 256
tic = time.perf_counter()
res = self.run_decode(max_tokens)
tok = time.perf_counter()
print(res["text"])
throughput = max_tokens / (tok - tic)
print(f"Throughput: {throughput} tokens/s")
if is_in_ci():
self.assertGreaterEqual(throughput, 25)
if __name__ == "__main__":
unittest.main()

6
test/srt/run_suite.py
View File

@@ -160,6 +160,7 @@ suite_ascend = {
TestFile("ascend/test_ascend_sampling_backend.py", 400),
TestFile("ascend/test_ascend_tp1_bf16.py", 400),
TestFile("ascend/test_ascend_compile_graph_tp1_bf16.py", 400),
TestFile("ascend/test_ascend_w8a8_quantization.py", 400),
TestFile("test_embed_interpolate_unittest.py", 400),
],
"per-commit-2-npu-a2": [
@@ -174,8 +175,9 @@ suite_ascend = {
TestFile("ascend/test_ascend_tp4_bf16.py", 400),
],
"per-commit-16-npu-a3": [
TestFile("ascend/test_ascend_deepep.py", 400),
TestFile("ascend/test_ascend_deepseek_mtp.py", 400),
TestFile("ascend/test_ascend_deepep.py", 3600),
TestFile("ascend/test_ascend_deepseek_mtp.py", 2800),
TestFile("ascend/test_ascend_w4a4_quantization.py", 600),
],
}