[NPU]add nightly-test-npu (#14143)

test/nightly/ascend/llm_models/gsm8k_ascend_mixin.py

@@ -0,0 +1,68 @@
+import os
+from abc import ABC
+from types import SimpleNamespace
+
+from sglang.srt.utils import kill_process_tree
+from sglang.test.few_shot_gsm8k import run_eval
+from sglang.test.test_utils import (
+    DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+    DEFAULT_URL_FOR_TEST,
+    popen_launch_server,
+)
+
+
+class GSM8KAscendMixin(ABC):
+    model = ""
+    accuracy = 0.00
+    other_args = [
+        "--trust-remote-code",
+        "--mem-fraction-static",
+        "0.8",
+        "--attention-backend",
+        "ascend",
+        "--disable-cuda-graph",
+    ]
+
+    @classmethod
+    def setUpClass(cls):
+        cls.base_url = DEFAULT_URL_FOR_TEST
+        os.environ["PYTORCH_NPU_ALLOC_CONF"] = "expandable_segments:True"
+        os.environ["ASCEND_MF_STORE_URL"] = "tcp://127.0.0.1:24666"
+        os.environ["HCCL_BUFFSIZE"] = "200"
+        os.environ["SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK"] = "24"
+        os.environ["USE_VLLM_CUSTOM_ALLREDUCE"] = "1"
+        os.environ["HCCL_EXEC_TIMEOUT"] = "200"
+        os.environ["STREAMS_PER_DEVICE"] = "32"
+        os.environ["SGLANG_ENBLE_TORCH_COMILE"] = "1"
+        os.environ["AUTO_USE_UC_MEMORY"] = "0"
+        os.environ["P2P_HCCL_BUFFSIZE"] = "20"
+        env = os.environ.copy()
+
+        cls.process = popen_launch_server(
+            cls.model,
+            cls.base_url,
+            timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+            other_args=cls.other_args,
+            env=env,
+        )
+
+    @classmethod
+    def tearDownClass(cls):
+        kill_process_tree(cls.process.pid)
+
+    def test_gsm8k(self):
+        args = SimpleNamespace(
+            num_shots=5,
+            data_path=None,
+            num_questions=200,
+            max_new_tokens=512,
+            parallel=128,
+            host="http://127.0.0.1",
+            port=int(self.base_url.split(":")[-1]),
+        )
+        metrics = run_eval(args)
+        self.assertGreater(
+            metrics["accuracy"],
+            self.accuracy,
+            f'Accuracy of {self.model} is {str(metrics["accuracy"])}, is lower than {self.accuracy}',
+        )

test/nightly/ascend/llm_models/test_ascend_afm_4_5b.py

@@ -0,0 +1,17 @@
+import unittest
+
+from gsm8k_ascend_mixin import GSM8KAscendMixin
+
+from sglang.test.ci.ci_register import register_npu_ci
+from sglang.test.test_utils import CustomTestCase
+
+register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
+
+
+class TestMistral7B(GSM8KAscendMixin, CustomTestCase):
+    model = "/root/.cache/modelscope/hub/models/arcee-ai/AFM-4.5B-Base"
+    accuracy = 0.00
+
+
+if __name__ == "__main__":
+    unittest.main()

test/nightly/ascend/llm_models/test_ascend_baichuan2_13b_chat.py

@@ -0,0 +1,22 @@
+import unittest
+
+from gsm8k_ascend_mixin import GSM8KAscendMixin
+
+from sglang.test.ci.ci_register import register_npu_ci
+from sglang.test.test_utils import CustomTestCase
+
+register_npu_ci(
+    est_time=400,
+    suite="nightly-1-npu-a3",
+    nightly=True,
+    disabled="The accuracy test result is 0.",
+)
+
+
+class TestMistral7B(GSM8KAscendMixin, CustomTestCase):
+    model = "/root/.cache/modelscope/hub/models/baichuan-inc/Baichuan2-13B-Chat"
+    accuracy = 0.00
+
+
+if __name__ == "__main__":
+    unittest.main()

test/nightly/ascend/llm_models/test_ascend_c4ai_command_r_v01.py

@@ -0,0 +1,91 @@
+import os
+import unittest
+from types import SimpleNamespace
+
+from sglang.srt.utils import kill_process_tree
+from sglang.test.ci.ci_register import register_npu_ci
+from sglang.test.few_shot_gsm8k import run_eval
+from sglang.test.test_utils import (
+    DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+    DEFAULT_URL_FOR_TEST,
+    CustomTestCase,
+    popen_launch_server,
+)
+
+register_npu_ci(
+    est_time=400,
+    suite="nightly-2-npu-a3",
+    nightly=True,
+    disabled="The accuracy test result is 0.",
+)
+
+
+class TestC4AI(CustomTestCase):
+    model = "/root/.cache/modelscope/hub/models/CohereForAI/c4ai-command-r-v01"
+    accuracy = 0.05
+
+    @classmethod
+    def setUpClass(cls):
+        cls.base_url = DEFAULT_URL_FOR_TEST
+        chat_template_path = "/__w/sglang/sglang/test/nightly/ascend/llm_models/tool_chat_template_c4ai_command_r_v01.jinja"
+
+        other_args = [
+            "--trust-remote-code",
+            "--mem-fraction-static",
+            "0.8",
+            "--attention-backend",
+            "ascend",
+            "--disable-cuda-graph",
+            "--chat-template",
+            chat_template_path,
+            "--tp-size",
+            "2",
+            "--dtype",
+            "bfloat16",
+        ]
+        env = os.environ.copy()
+        env.update(
+            {
+                "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
+                "ASCEND_MF_STORE_URL": "tcp://127.0.0.1:24666",
+                "HCCL_BUFFSIZE": "200",
+                "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "24",
+                "USE_VLLM_CUSTOM_ALLREDUCE": "1",
+                "HCCL_EXEC_TIMEOUT": "200",
+                "STREAMS_PER_DEVICE": "32",
+                "SGLANG_ENABLE_TORCH_COMPILE": "1",
+            }
+        )
+
+        cls.process = popen_launch_server(
+            cls.model,
+            cls.base_url,
+            timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+            other_args=other_args,
+            env=env,
+        )
+
+    @classmethod
+    def tearDownClass(cls):
+        kill_process_tree(cls.process.pid)
+
+    def test_gsm8k(self):
+        args = SimpleNamespace(
+            num_shots=5,
+            data_path=None,
+            num_questions=200,
+            max_new_tokens=512,
+            parallel=128,
+            host="http://127.0.0.1",
+            port=int(self.base_url.split(":")[-1]),
+        )
+        metrics = run_eval(args)
+        self.assertGreater(
+            metrics["accuracy"],
+            self.accuracy,
+            f'Accuracy of {self.model} is {str(metrics["accuracy"])}, is lower than {self.accuracy}',
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()

test/nightly/ascend/llm_models/test_ascend_charglm2_6b.py

@@ -0,0 +1,27 @@
+import unittest
+
+from gsm8k_ascend_mixin import GSM8KAscendMixin
+
+from sglang.test.ci.ci_register import register_npu_ci
+from sglang.test.test_utils import CustomTestCase
+
+register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
+
+
+class TestMistral7B(GSM8KAscendMixin, CustomTestCase):
+    model = "/root/.cache/modelscope/hub/models/ZhipuAI/chatglm2-6b"
+    accuracy = 0.25
+    other_args = [
+        "--trust-remote-code",
+        "--mem-fraction-static",
+        "0.8",
+        "--attention-backend",
+        "ascend",
+        "--disable-cuda-graph",
+        "--dtype",
+        "bfloat16",
+    ]
+
+
+if __name__ == "__main__":
+    unittest.main()

test/nightly/ascend/llm_models/test_ascend_exaone_3.py

@@ -0,0 +1,27 @@
+import unittest
+
+from gsm8k_ascend_mixin import GSM8KAscendMixin
+
+from sglang.test.ci.ci_register import register_npu_ci
+from sglang.test.test_utils import CustomTestCase
+
+register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
+
+
+class TestMistral7B(GSM8KAscendMixin, CustomTestCase):
+    model = "/root/.cache/modelscope/hub/models/LGAI-EXAONE/EXAONE-3.5-7.8B-Instruct"
+    accuracy = 0.00
+    other_args = [
+        "--trust-remote-code",
+        "--mem-fraction-static",
+        "0.8",
+        "--attention-backend",
+        "ascend",
+        "--disable-cuda-graph",
+        "--dtype",
+        "bfloat16",
+    ]
+
+
+if __name__ == "__main__":
+    unittest.main()

test/nightly/ascend/llm_models/test_ascend_gemma_3_1b_it.py

@@ -0,0 +1,22 @@
+import unittest
+
+from gsm8k_ascend_mixin import GSM8KAscendMixin
+
+from sglang.test.ci.ci_register import register_npu_ci
+from sglang.test.test_utils import CustomTestCase
+
+register_npu_ci(
+    est_time=400,
+    suite="nightly-1-npu-a3",
+    nightly=True,
+    disabled="The accuracy test result is 0.",
+)
+
+
+class TestMistral7B(GSM8KAscendMixin, CustomTestCase):
+    model = "/root/.cache/modelscope/hub/models/LLM-Research/gemma-3-1b-it"
+    accuracy = 0.00
+
+
+if __name__ == "__main__":
+    unittest.main()

test/nightly/ascend/llm_models/test_ascend_glm4_9b_chat.py

@@ -0,0 +1,17 @@
+import unittest
+
+from gsm8k_ascend_mixin import GSM8KAscendMixin
+
+from sglang.test.ci.ci_register import register_npu_ci
+from sglang.test.test_utils import CustomTestCase
+
+register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
+
+
+class TestGLM49BChat(GSM8KAscendMixin, CustomTestCase):
+    model = "/root/.cache/modelscope/hub/models/ZhipuAI/glm-4-9b-chat"
+    accuracy = 0.00
+
+
+if __name__ == "__main__":
+    unittest.main()

test/nightly/ascend/llm_models/test_ascend_granite_3_0_3b_a800m.py

@@ -0,0 +1,19 @@
+import unittest
+
+from gsm8k_ascend_mixin import GSM8KAscendMixin
+
+from sglang.test.ci.ci_register import register_npu_ci
+from sglang.test.test_utils import CustomTestCase
+
+register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
+
+
+class TestMistral7B(GSM8KAscendMixin, CustomTestCase):
+    model = (
+        "/root/.cache/modelscope/hub/models/ibm-granite/granite-3.0-3b-a800m-instruct"
+    )
+    accuracy = 0.00
+
+
+if __name__ == "__main__":
+    unittest.main()

test/nightly/ascend/llm_models/test_ascend_granite_3_1_8b.py

@@ -0,0 +1,17 @@
+import unittest
+
+from gsm8k_ascend_mixin import GSM8KAscendMixin
+
+from sglang.test.ci.ci_register import register_npu_ci
+from sglang.test.test_utils import CustomTestCase
+
+register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
+
+
+class TestMistral7B(GSM8KAscendMixin, CustomTestCase):
+    model = "/root/.cache/modelscope/hub/models/ibm-granite/granite-3.1-8b-instruct"
+    accuracy = 0.695
+
+
+if __name__ == "__main__":
+    unittest.main()

test/nightly/ascend/llm_models/test_ascend_internlm2_7b.py

@@ -0,0 +1,17 @@
+import unittest
+
+from gsm8k_ascend_mixin import GSM8KAscendMixin
+
+from sglang.test.ci.ci_register import register_npu_ci
+from sglang.test.test_utils import CustomTestCase
+
+register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
+
+
+class TestMistral7B(GSM8KAscendMixin, CustomTestCase):
+    model = "/root/.cache/modelscope/hub/models/Shanghai_AI_Laboratory/internlm2-7b"
+    accuracy = 0.6
+
+
+if __name__ == "__main__":
+    unittest.main()

test/nightly/ascend/llm_models/test_ascend_ling_lite.py

@@ -0,0 +1,17 @@
+import unittest
+
+from gsm8k_ascend_mixin import GSM8KAscendMixin
+
+from sglang.test.ci.ci_register import register_npu_ci
+from sglang.test.test_utils import CustomTestCase
+
+register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
+
+
+class TestMistral7B(GSM8KAscendMixin, CustomTestCase):
+    model = "/root/.cache/modelscope/hub/models/inclusionAI/Ling-lite"
+    accuracy = 0.75
+
+
+if __name__ == "__main__":
+    unittest.main()

test/nightly/ascend/llm_models/test_ascend_llama_2_7b.py

@@ -0,0 +1,17 @@
+import unittest
+
+from gsm8k_ascend_mixin import GSM8KAscendMixin
+
+from sglang.test.ci.ci_register import register_npu_ci
+from sglang.test.test_utils import CustomTestCase
+
+register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
+
+
+class TestMistral7B(GSM8KAscendMixin, CustomTestCase):
+    model = "/root/.cache/modelscope/hub/models/LLM-Research/Llama-2-7B"
+    accuracy = 0.18
+
+
+if __name__ == "__main__":
+    unittest.main()

test/nightly/ascend/llm_models/test_ascend_mimo_7b_rl.py

@@ -0,0 +1,17 @@
+import unittest
+
+from gsm8k_ascend_mixin import GSM8KAscendMixin
+
+from sglang.test.ci.ci_register import register_npu_ci
+from sglang.test.test_utils import CustomTestCase
+
+register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
+
+
+class TestMistral7B(GSM8KAscendMixin, CustomTestCase):
+    model = "/root/.cache/modelscope/hub/models/XiaomiMiMo/MiMo-7B-RL"
+    accuracy = 0.75
+
+
+if __name__ == "__main__":
+    unittest.main()

test/nightly/ascend/llm_models/test_ascend_mistral_7b.py

@@ -0,0 +1,17 @@
+import unittest
+
+from gsm8k_ascend_mixin import GSM8KAscendMixin
+
+from sglang.test.ci.ci_register import register_npu_ci
+from sglang.test.test_utils import CustomTestCase
+
+register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
+
+
+class TestMistral7B(GSM8KAscendMixin, CustomTestCase):
+    model = "/root/.cache/modelscope/hub/models/mistralai/Mistral-7B-Instruct-v0.2"
+    accuracy = 0.375
+
+
+if __name__ == "__main__":
+    unittest.main()

test/nightly/ascend/llm_models/test_ascend_persimmon_8b_chat.py

@@ -0,0 +1,17 @@
+import unittest
+
+from gsm8k_ascend_mixin import GSM8KAscendMixin
+
+from sglang.test.ci.ci_register import register_npu_ci
+from sglang.test.test_utils import CustomTestCase
+
+register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
+
+
+class TestMistral7B(GSM8KAscendMixin, CustomTestCase):
+    model = "/root/.cache/modelscope/hub/models/Howeee/persimmon-8b-chat"
+    accuracy = 0.17
+
+
+if __name__ == "__main__":
+    unittest.main()

test/nightly/ascend/llm_models/test_ascend_phi_4_multimodal.py

@@ -0,0 +1,17 @@
+import unittest
+
+from gsm8k_ascend_mixin import GSM8KAscendMixin
+
+from sglang.test.ci.ci_register import register_npu_ci
+from sglang.test.test_utils import CustomTestCase
+
+register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
+
+
+class TestMistral7B(GSM8KAscendMixin, CustomTestCase):
+    model = "/root/.cache/modelscope/hub/models/LLM-Research/Phi-4-multimodal-instruct"
+    accuracy = 0.8
+
+
+if __name__ == "__main__":
+    unittest.main()

test/nightly/ascend/llm_models/test_ascend_smollm_1_7b.py

@@ -0,0 +1,27 @@
+import unittest
+
+from gsm8k_ascend_mixin import GSM8KAscendMixin
+
+from sglang.test.ci.ci_register import register_npu_ci
+from sglang.test.test_utils import CustomTestCase
+
+register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
+
+
+class TestMistral7B(GSM8KAscendMixin, CustomTestCase):
+    model = "/root/.cache/modelscope/hub/models/HuggingFaceTB/SmolLM-1.7B"
+    accuracy = 0.05
+    other_args = [
+        "--trust-remote-code",
+        "--mem-fraction-static",
+        "0.8",
+        "--attention-backend",
+        "ascend",
+        "--disable-cuda-graph",
+        "--dtype",
+        "bfloat16",
+    ]
+
+
+if __name__ == "__main__":
+    unittest.main()

test/nightly/ascend/llm_models/tool_chat_template_c4ai_command_r_v01.jinja

@@ -0,0 +1 @@
+{{ bos_token }}{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% elif false == true %}{% set loop_messages = messages %}{% set system_message = 'You are Command-R, a brilliant, sophisticated, AI-assistant trained to assist human users by providing thorough responses. You are trained by Cohere.' %}{% else %}{% set loop_messages = messages %}{% set system_message = false %}{% endif %}{% if system_message != false %}{{ '<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|>' + system_message + '<|END_OF_TURN_TOKEN|>' }}{% endif %}{% for message in loop_messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ '<|START_OF_TURN_TOKEN|><|USER_TOKEN|>' + content.strip() + '<|END_OF_TURN_TOKEN|>' }}{% elif message['role'] == 'assistant' %}{{ '<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>'  + content.strip() + '<|END_OF_TURN_TOKEN|>' }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>' }}{% endif %}