From 13bdc7bf4a22917dc0b20fca06f77e9ec803c113 Mon Sep 17 00:00:00 2001 From: YeChang Guo <52730608+YChange01@users.noreply.github.com> Date: Sat, 7 Mar 2026 23:03:55 +0800 Subject: [PATCH] [Feature][NPU]: add runtime support for AutoRound quantized models (#16699) Co-authored-by: root Co-authored-by: ronnie_zheng --- python/sglang/srt/layers/linear.py | 3 +- .../srt/layers/quantization/auto_round.py | 26 +++++++ .../srt/ascend/test_ascend_autoround_dense.py | 76 +++++++++++++++++++ test/srt/ascend/test_ascend_autoround_moe.py | 76 +++++++++++++++++++ test/srt/run_suite.py | 2 + 5 files changed, 181 insertions(+), 2 deletions(-) create mode 100644 test/srt/ascend/test_ascend_autoround_dense.py create mode 100644 test/srt/ascend/test_ascend_autoround_moe.py diff --git a/python/sglang/srt/layers/linear.py b/python/sglang/srt/layers/linear.py index 02d11cc45..924ef64fd 100644 --- a/python/sglang/srt/layers/linear.py +++ b/python/sglang/srt/layers/linear.py @@ -64,10 +64,9 @@ WEIGHT_LOADER_V2_SUPPORTED = [ "GPTQMarlin24LinearMethod", "TPUInt8LinearMethod", "GPTQLinearMethod", - "GPTQLinearAscendMethod", - "GPTQMoEAscendMethod", "FBGEMMFp8LinearMethod", "GPTQLinearAscendMethod", + "GPTQMoEAscendMethod", "ModelOptFp8LinearMethod", "ModelOptFp4LinearMethod", "IPEXAWQLinearMethod", diff --git a/python/sglang/srt/layers/quantization/auto_round.py b/python/sglang/srt/layers/quantization/auto_round.py index 74c4f0231..4690bbb9e 100644 --- a/python/sglang/srt/layers/quantization/auto_round.py +++ b/python/sglang/srt/layers/quantization/auto_round.py @@ -14,6 +14,9 @@ from sglang.srt.layers.quantization.utils import get_scalar_types ScalarType, scalar_types = get_scalar_types() from sglang.srt.layers.quantization.base_config import QuantizationConfig +from sglang.srt.utils import is_npu + +_is_npu = is_npu() class AutoRoundConfig(QuantizationConfig): @@ -301,6 +304,11 @@ class AutoRoundConfig(QuantizationConfig): def apply_gptq_quant_layer(self, layer, prefix: str, backend: str = "auto"): from sglang.srt.layers.linear import LinearBase from sglang.srt.layers.moe.fused_moe_triton import FusedMoE + from sglang.srt.layers.quantization.gptq import ( + GPTQConfig, + GPTQLinearAscendMethod, + GPTQMoEAscendMethod, + ) from sglang.srt.layers.quantization.marlin_utils import ( check_marlin_supported, check_moe_marlin_supports_layer, @@ -323,6 +331,24 @@ class AutoRoundConfig(QuantizationConfig): group_size, sym, ) + if _is_npu: + quant_args = GPTQConfig( + weight_bits=weight_bits, + group_size=group_size, + lm_head_quantized=False, + desc_act=False, + dynamic={}, + ) + quant_args.sym = sym + + if isinstance(layer, FusedMoE): + return GPTQMoEAscendMethod(quant_args) + + if isinstance(layer, (LinearBase, ParallelLMHead)): + return GPTQLinearAscendMethod(quant_args) + + return None + if backend == "auto" or "marlin" in backend: GPTQ_TYPE_MAP = { (4, True): scalar_types.uint4b8, diff --git a/test/srt/ascend/test_ascend_autoround_dense.py b/test/srt/ascend/test_ascend_autoround_dense.py new file mode 100644 index 000000000..475311b6c --- /dev/null +++ b/test/srt/ascend/test_ascend_autoround_dense.py @@ -0,0 +1,76 @@ +import logging +import unittest +from types import SimpleNamespace +from urllib.parse import urlparse + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + +logger = logging.getLogger(__name__) + +TEST_MODEL_MATRIX = { + "/root/.cache/modelscope/hub/models/Intel/Qwen3-8B-int4-AutoRound": { + "accuracy": 0.85, + }, +} + + +class TestAscendAutoRoundDense(CustomTestCase): + + @classmethod + def setUpClass(cls): + cls.models = TEST_MODEL_MATRIX.keys() + cls.base_url = DEFAULT_URL_FOR_TEST + cls.url = urlparse(DEFAULT_URL_FOR_TEST) + cls.common_args = [ + "--trust-remote-code", + "--mem-fraction-static", + 0.8, + "--attention-backend", + "ascend", + "--quantization", + "auto-round", + ] + + def test_a_gsm8k(self): + for model in self.models: + with self.subTest(model=model): + logger.info(f"##=== Testing accuracy: {model} ===##") + + process = popen_launch_server( + model, + self.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + *self.common_args, + ], + ) + + try: + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=1319, + max_new_tokens=512, + parallel=128, + host=f"http://{self.url.hostname}", + port=int(self.url.port), + ) + + metrics = run_eval_few_shot_gsm8k(args) + self.assertGreaterEqual( + metrics["accuracy"], + TEST_MODEL_MATRIX[model]["accuracy"], + ) + finally: + kill_process_tree(process.pid) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/ascend/test_ascend_autoround_moe.py b/test/srt/ascend/test_ascend_autoround_moe.py new file mode 100644 index 000000000..b0b8f6960 --- /dev/null +++ b/test/srt/ascend/test_ascend_autoround_moe.py @@ -0,0 +1,76 @@ +import logging +import unittest +from types import SimpleNamespace +from urllib.parse import urlparse + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + +logger = logging.getLogger(__name__) + +TEST_MODEL_MATRIX = { + "/root/.cache/modelscope/hub/models/Intel/Qwen3-30B-A3B-Instruct-2507-int4-AutoRound": { + "accuracy": 0.85, + }, +} + + +class TestAscendAutoRoundMoE(CustomTestCase): + + @classmethod + def setUpClass(cls): + cls.models = TEST_MODEL_MATRIX.keys() + cls.base_url = DEFAULT_URL_FOR_TEST + cls.url = urlparse(DEFAULT_URL_FOR_TEST) + cls.common_args = [ + "--trust-remote-code", + "--mem-fraction-static", + 0.8, + "--attention-backend", + "ascend", + "--quantization", + "auto-round", + ] + + def test_a_gsm8k(self): + for model in self.models: + with self.subTest(model=model): + logger.info(f"##=== Testing accuracy: {model} ===##") + + process = popen_launch_server( + model, + self.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + *self.common_args, + ], + ) + + try: + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=1319, + max_new_tokens=512, + parallel=128, + host=f"http://{self.url.hostname}", + port=int(self.url.port), + ) + + metrics = run_eval_few_shot_gsm8k(args) + self.assertGreaterEqual( + metrics["accuracy"], + TEST_MODEL_MATRIX[model]["accuracy"], + ) + finally: + kill_process_tree(process.pid) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/run_suite.py b/test/srt/run_suite.py index 8f8de3885..c27decf36 100644 --- a/test/srt/run_suite.py +++ b/test/srt/run_suite.py @@ -83,6 +83,8 @@ suite_xpu = { # NOTE: please sort the test cases alphabetically by the test file name suite_ascend = { "per-commit-1-npu-a2": [ + TestFile("ascend/test_ascend_autoround_dense.py", 400), + TestFile("ascend/test_ascend_autoround_moe.py", 400), TestFile("ascend/test_ascend_gptq.py", 400), TestFile("ascend/test_ascend_gptq_moe.py", 400), TestFile("ascend/test_ascend_graph_tp1_bf16.py", 400),