chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[Feature][NPU]: add runtime support for AutoRound quantized models (#16699)

Browse Source 
Co-authored-by: root <root@localhost.localdomain>
Co-authored-by: ronnie_zheng <zl19940307@163.com>
This commit is contained in:
YeChang Guo
2026-03-07 23:03:55 +08:00
committed by GitHub
parent fd79cd8d9c
commit 13bdc7bf4a
5 changed files with 181 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
python/sglang/srt/layers/linear.py
View File

@@ -64,10 +64,9 @@ WEIGHT_LOADER_V2_SUPPORTED = [
"GPTQMarlin24LinearMethod",
"TPUInt8LinearMethod",
"GPTQLinearMethod",
"GPTQLinearAscendMethod",
"GPTQMoEAscendMethod",
"FBGEMMFp8LinearMethod",
"GPTQLinearAscendMethod",
"GPTQMoEAscendMethod",
"ModelOptFp8LinearMethod",
"ModelOptFp4LinearMethod",
"IPEXAWQLinearMethod",

26
python/sglang/srt/layers/quantization/auto_round.py
View File

@@ -14,6 +14,9 @@ from sglang.srt.layers.quantization.utils import get_scalar_types
ScalarType, scalar_types = get_scalar_types()
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.utils import is_npu
_is_npu = is_npu()
class AutoRoundConfig(QuantizationConfig):
@@ -301,6 +304,11 @@ class AutoRoundConfig(QuantizationConfig):
def apply_gptq_quant_layer(self, layer, prefix: str, backend: str = "auto"):
from sglang.srt.layers.linear import LinearBase
from sglang.srt.layers.moe.fused_moe_triton import FusedMoE
from sglang.srt.layers.quantization.gptq import (
GPTQConfig,
GPTQLinearAscendMethod,
GPTQMoEAscendMethod,
)
from sglang.srt.layers.quantization.marlin_utils import (
check_marlin_supported,
check_moe_marlin_supports_layer,
@@ -323,6 +331,24 @@ class AutoRoundConfig(QuantizationConfig):
group_size,
sym,
)
if _is_npu:
quant_args = GPTQConfig(
weight_bits=weight_bits,
group_size=group_size,
lm_head_quantized=False,
desc_act=False,
dynamic={},
)
quant_args.sym = sym
if isinstance(layer, FusedMoE):
return GPTQMoEAscendMethod(quant_args)
if isinstance(layer, (LinearBase, ParallelLMHead)):
return GPTQLinearAscendMethod(quant_args)
return None
if backend == "auto" or "marlin" in backend:
GPTQ_TYPE_MAP = {
(4, True): scalar_types.uint4b8,

76
test/srt/ascend/test_ascend_autoround_dense.py Normal file
View File

@@ -0,0 +1,76 @@
import logging
import unittest
from types import SimpleNamespace
from urllib.parse import urlparse
from sglang.srt.utils import kill_process_tree
from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
popen_launch_server,
)
logger = logging.getLogger(__name__)
TEST_MODEL_MATRIX = {
"/root/.cache/modelscope/hub/models/Intel/Qwen3-8B-int4-AutoRound": {
"accuracy": 0.85,
},
}
class TestAscendAutoRoundDense(CustomTestCase):
@classmethod
def setUpClass(cls):
cls.models = TEST_MODEL_MATRIX.keys()
cls.base_url = DEFAULT_URL_FOR_TEST
cls.url = urlparse(DEFAULT_URL_FOR_TEST)
cls.common_args = [
"--trust-remote-code",
"--mem-fraction-static",
0.8,
"--attention-backend",
"ascend",
"--quantization",
"auto-round",
]
def test_a_gsm8k(self):
for model in self.models:
with self.subTest(model=model):
logger.info(f"##=== Testing accuracy: {model} ===##")
process = popen_launch_server(
model,
self.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
*self.common_args,
],
)
try:
args = SimpleNamespace(
num_shots=5,
data_path=None,
num_questions=1319,
max_new_tokens=512,
parallel=128,
host=f"http://{self.url.hostname}",
port=int(self.url.port),
)
metrics = run_eval_few_shot_gsm8k(args)
self.assertGreaterEqual(
metrics["accuracy"],
TEST_MODEL_MATRIX[model]["accuracy"],
)
finally:
kill_process_tree(process.pid)
if __name__ == "__main__":
unittest.main()

76
test/srt/ascend/test_ascend_autoround_moe.py Normal file
View File

@@ -0,0 +1,76 @@
import logging
import unittest
from types import SimpleNamespace
from urllib.parse import urlparse
from sglang.srt.utils import kill_process_tree
from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
popen_launch_server,
)
logger = logging.getLogger(__name__)
TEST_MODEL_MATRIX = {
"/root/.cache/modelscope/hub/models/Intel/Qwen3-30B-A3B-Instruct-2507-int4-AutoRound": {
"accuracy": 0.85,
},
}
class TestAscendAutoRoundMoE(CustomTestCase):
@classmethod
def setUpClass(cls):
cls.models = TEST_MODEL_MATRIX.keys()
cls.base_url = DEFAULT_URL_FOR_TEST
cls.url = urlparse(DEFAULT_URL_FOR_TEST)
cls.common_args = [
"--trust-remote-code",
"--mem-fraction-static",
0.8,
"--attention-backend",
"ascend",
"--quantization",
"auto-round",
]
def test_a_gsm8k(self):
for model in self.models:
with self.subTest(model=model):
logger.info(f"##=== Testing accuracy: {model} ===##")
process = popen_launch_server(
model,
self.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
*self.common_args,
],
)
try:
args = SimpleNamespace(
num_shots=5,
data_path=None,
num_questions=1319,
max_new_tokens=512,
parallel=128,
host=f"http://{self.url.hostname}",
port=int(self.url.port),
)
metrics = run_eval_few_shot_gsm8k(args)
self.assertGreaterEqual(
metrics["accuracy"],
TEST_MODEL_MATRIX[model]["accuracy"],
)
finally:
kill_process_tree(process.pid)
if __name__ == "__main__":
unittest.main()

2
test/srt/run_suite.py
View File

@@ -83,6 +83,8 @@ suite_xpu = {
# NOTE: please sort the test cases alphabetically by the test file name
suite_ascend = {
"per-commit-1-npu-a2": [
TestFile("ascend/test_ascend_autoround_dense.py", 400),
TestFile("ascend/test_ascend_autoround_moe.py", 400),
TestFile("ascend/test_ascend_gptq.py", 400),
TestFile("ascend/test_ascend_gptq_moe.py", 400),
TestFile("ascend/test_ascend_graph_tp1_bf16.py", 400),