From 1eb40d8d458c5364d9c36204a866c77730ac4d4f Mon Sep 17 00:00:00 2001 From: Cherry_ming <136634645@qq.com> Date: Sat, 28 Feb 2026 17:06:03 +0800 Subject: [PATCH] [NPU]Optimize the PR pipeline to reduce E2E runtime (#18767) Co-authored-by: Sugar920 <121632458+Sugar920@users.noreply.github.com> --- .github/workflows/pr-test-npu.yml | 28 +++++++++---------- .../test_ascend_compile_graph_tp1_bf16.py | 22 --------------- test/srt/ascend/test_ascend_deepep.py | 24 +--------------- test/srt/ascend/test_ascend_graph_tp1_bf16.py | 22 --------------- test/srt/ascend/test_ascend_graph_tp2_bf16.py | 22 --------------- test/srt/ascend/test_ascend_hicache_mha.py | 22 --------------- test/srt/ascend/test_ascend_hicache_mla.py | 22 --------------- .../ascend/test_ascend_mla_fia_w8a8int8.py | 22 --------------- test/srt/ascend/test_ascend_mla_w8a8int8.py | 22 --------------- test/srt/ascend/test_ascend_tp2_bf16.py | 22 --------------- test/srt/ascend/test_ascend_tp4_bf16.py | 22 --------------- test/srt/run_suite.py | 5 ++-- 12 files changed, 17 insertions(+), 238 deletions(-) diff --git a/.github/workflows/pr-test-npu.yml b/.github/workflows/pr-test-npu.yml index 7169f4718..6a8085809 100644 --- a/.github/workflows/pr-test-npu.yml +++ b/.github/workflows/pr-test-npu.yml @@ -79,6 +79,10 @@ jobs: needs: [check-changes, pr-gate] if: needs.check-changes.outputs.main_package == 'true' runs-on: linux-aarch64-a2-1 + strategy: + fail-fast: false + matrix: + part: [ 0, 1 ] container: image: swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:8.5.0-910b-ubuntu22.04-py3.11 steps: @@ -113,7 +117,7 @@ jobs: STREAMS_PER_DEVICE: 32 run: | cd test/srt - python3 run_suite.py --suite per-commit-1-npu-a2 + python3 run_suite.py --suite per-commit-1-npu-a2 --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 per-commit-2-npu-a2: needs: [check-changes, pr-gate] @@ -122,7 +126,7 @@ jobs: strategy: fail-fast: true matrix: - part: [0, 1, 2] + part: [0, 1] container: image: swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:8.5.0-910b-ubuntu22.04-py3.11 steps: @@ -157,14 +161,14 @@ jobs: STREAMS_PER_DEVICE: 32 run: | cd test/srt - python3 run_suite.py --suite per-commit-2-npu-a2 --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 + python3 run_suite.py --suite per-commit-2-npu-a2 --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 - per-commit-4-npu-a2: + per-commit-4-npu-a3: needs: [check-changes, pr-gate] if: needs.check-changes.outputs.main_package == 'true' - runs-on: linux-aarch64-a2-4 + runs-on: linux-aarch64-a3-4 container: - image: swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:8.5.0-910b-ubuntu22.04-py3.11 + image: swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:8.5.0-a3-ubuntu22.04-py3.11 steps: - name: Checkout code uses: actions/checkout@v4 @@ -180,7 +184,7 @@ jobs: pip config set global.extra-index-url "https://pypi.tuna.tsinghua.edu.cn/simple" pip config set global.trusted-host "${CACHING_URL} pypi.tuna.tsinghua.edu.cn" - bash scripts/ci/npu/npu_ci_install_dependency.sh 910b + bash scripts/ci/npu/npu_ci_install_dependency.sh a3 # copy required file from our daily cache cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp # copy download through proxy @@ -197,16 +201,12 @@ jobs: STREAMS_PER_DEVICE: 32 run: | cd test/srt - python3 run_suite.py --suite per-commit-4-npu-a2 --timeout-per-file 3600 + python3 run_suite.py --suite per-commit-4-npu-a3 --timeout-per-file 3600 per-commit-16-npu-a3: needs: [check-changes, pr-gate] if: needs.check-changes.outputs.main_package == 'true' runs-on: linux-aarch64-a3-16 - strategy: - fail-fast: true - matrix: - part: [0, 1] container: image: swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:8.5.0-a3-ubuntu22.04-py3.11 steps: @@ -241,12 +241,12 @@ jobs: STREAMS_PER_DEVICE: 32 run: | cd test/srt - python3 run_suite.py --suite per-commit-16-npu-a3 --timeout-per-file 3600 --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 + python3 run_suite.py --suite per-commit-16-npu-a3 --timeout-per-file 3600 multimodal-gen-test-1-npu-a3: needs: [check-changes, pr-gate] if: needs.check-changes.outputs.multimodal_gen == 'true' - runs-on: linux-aarch64-a3-16 + runs-on: linux-aarch64-a3-2 container: image: swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:8.3.rc2-a3-ubuntu22.04-py3.11 steps: diff --git a/test/srt/ascend/test_ascend_compile_graph_tp1_bf16.py b/test/srt/ascend/test_ascend_compile_graph_tp1_bf16.py index 4c6a1e8d0..e3a51499b 100644 --- a/test/srt/ascend/test_ascend_compile_graph_tp1_bf16.py +++ b/test/srt/ascend/test_ascend_compile_graph_tp1_bf16.py @@ -9,9 +9,7 @@ from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, CustomTestCase, - is_in_ci, popen_launch_server, - run_bench_offline_throughput, ) TEST_MODEL_MATRIX = { @@ -77,26 +75,6 @@ class TestAscendTp1Bf16(CustomTestCase): finally: kill_process_tree(process.pid) - def test_b_throughput(self): - for model in self.models: - with self.subTest(model=model): - print(f"##=== Testing throughput: {model} ===##") - - output_throughput = run_bench_offline_throughput( - model, - [ - *self.common_args, - ], - ) - - print(f"##=== {model} throughput: {output_throughput} ===##") - - if is_in_ci(): - self.assertGreater( - output_throughput, - TEST_MODEL_MATRIX[model]["output_throughput"], - ) - if __name__ == "__main__": unittest.main() diff --git a/test/srt/ascend/test_ascend_deepep.py b/test/srt/ascend/test_ascend_deepep.py index e17281094..19330b862 100644 --- a/test/srt/ascend/test_ascend_deepep.py +++ b/test/srt/ascend/test_ascend_deepep.py @@ -8,9 +8,7 @@ from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k from sglang.test.test_utils import ( DEFAULT_URL_FOR_TEST, CustomTestCase, - is_in_ci, popen_launch_server, - run_bench_offline_throughput, ) TEST_MODEL_MATRIX = { @@ -78,7 +76,7 @@ class TestAscendDeepEP(CustomTestCase): args = SimpleNamespace( num_shots=5, data_path=None, - num_questions=1319, + num_questions=500, max_new_tokens=512, parallel=128, host=f"http://{self.url.hostname}", @@ -93,26 +91,6 @@ class TestAscendDeepEP(CustomTestCase): finally: kill_process_tree(process.pid) - def test_b_throughput(self): - for model in self.models: - with self.subTest(model=model): - print(f"##=== Testing throughput: {model} ===##") - - output_throughput = run_bench_offline_throughput( - model, - [ - *self.common_args, - ], - ) - - print(f"##=== {model} throughput: {output_throughput} ===##") - - if is_in_ci(): - self.assertGreater( - output_throughput, - TEST_MODEL_MATRIX[model]["output_throughput"], - ) - if __name__ == "__main__": unittest.main() diff --git a/test/srt/ascend/test_ascend_graph_tp1_bf16.py b/test/srt/ascend/test_ascend_graph_tp1_bf16.py index e181f7aac..4f8d4b4aa 100644 --- a/test/srt/ascend/test_ascend_graph_tp1_bf16.py +++ b/test/srt/ascend/test_ascend_graph_tp1_bf16.py @@ -8,9 +8,7 @@ from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, CustomTestCase, - is_in_ci, popen_launch_server, - run_bench_offline_throughput, ) TEST_MODEL_MATRIX = { @@ -70,26 +68,6 @@ class TestAscendGraphTp1Bf16(CustomTestCase): finally: kill_process_tree(process.pid) - def test_b_throughput(self): - for model in self.models: - with self.subTest(model=model): - print(f"##=== Testing throughput: {model} ===##") - - output_throughput = run_bench_offline_throughput( - model, - [ - *self.common_args, - ], - ) - - print(f"##=== {model} throughput: {output_throughput} ===##") - - if is_in_ci(): - self.assertGreater( - output_throughput, - TEST_MODEL_MATRIX[model]["output_throughput"], - ) - if __name__ == "__main__": unittest.main() diff --git a/test/srt/ascend/test_ascend_graph_tp2_bf16.py b/test/srt/ascend/test_ascend_graph_tp2_bf16.py index da65f5c6a..d4bf902a2 100644 --- a/test/srt/ascend/test_ascend_graph_tp2_bf16.py +++ b/test/srt/ascend/test_ascend_graph_tp2_bf16.py @@ -8,9 +8,7 @@ from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, CustomTestCase, - is_in_ci, popen_launch_server, - run_bench_offline_throughput, ) TEST_MODEL_MATRIX = { @@ -72,26 +70,6 @@ class TestAscendGraphTp2Bf16(CustomTestCase): finally: kill_process_tree(process.pid) - def test_b_throughput(self): - for model in self.models: - with self.subTest(model=model): - print(f"##=== Testing throughput: {model} ===##") - - output_throughput = run_bench_offline_throughput( - model, - [ - *self.common_args, - ], - ) - - print(f"##=== {model} throughput: {output_throughput} ===##") - - if is_in_ci(): - self.assertGreater( - output_throughput, - TEST_MODEL_MATRIX[model]["output_throughput"], - ) - if __name__ == "__main__": unittest.main() diff --git a/test/srt/ascend/test_ascend_hicache_mha.py b/test/srt/ascend/test_ascend_hicache_mha.py index 2a91af17c..521537e05 100644 --- a/test/srt/ascend/test_ascend_hicache_mha.py +++ b/test/srt/ascend/test_ascend_hicache_mha.py @@ -8,9 +8,7 @@ from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, CustomTestCase, - is_in_ci, popen_launch_server, - run_bench_offline_throughput, ) TEST_MODEL_MATRIX = { @@ -73,26 +71,6 @@ class TestAscendMhaHicache(CustomTestCase): finally: kill_process_tree(process.pid) - def test_b_throughput(self): - for model in self.models: - with self.subTest(model=model): - print(f"##=== Testing throughput: {model} ===##") - - output_throughput = run_bench_offline_throughput( - model, - [ - *self.common_args, - ], - ) - - print(f"##=== {model} throughput: {output_throughput} ===##") - - if is_in_ci(): - self.assertGreater( - output_throughput, - TEST_MODEL_MATRIX[model]["output_throughput"], - ) - if __name__ == "__main__": unittest.main() diff --git a/test/srt/ascend/test_ascend_hicache_mla.py b/test/srt/ascend/test_ascend_hicache_mla.py index d0bc1f378..4bb355d37 100644 --- a/test/srt/ascend/test_ascend_hicache_mla.py +++ b/test/srt/ascend/test_ascend_hicache_mla.py @@ -8,9 +8,7 @@ from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, CustomTestCase, - is_in_ci, popen_launch_server, - run_bench_offline_throughput, ) TEST_MODEL_MATRIX = { @@ -75,26 +73,6 @@ class TestAscendMlaHicache(CustomTestCase): finally: kill_process_tree(process.pid) - def test_b_throughput(self): - for model in self.models: - with self.subTest(model=model): - print(f"##=== Testing throughput: {model} ===##") - - output_throughput = run_bench_offline_throughput( - model, - [ - *self.common_args, - ], - ) - - print(f"##=== {model} throughput: {output_throughput} ===##") - - if is_in_ci(): - self.assertGreater( - output_throughput, - TEST_MODEL_MATRIX[model]["output_throughput"], - ) - if __name__ == "__main__": unittest.main() diff --git a/test/srt/ascend/test_ascend_mla_fia_w8a8int8.py b/test/srt/ascend/test_ascend_mla_fia_w8a8int8.py index bdab4ea05..4001df6f6 100644 --- a/test/srt/ascend/test_ascend_mla_fia_w8a8int8.py +++ b/test/srt/ascend/test_ascend_mla_fia_w8a8int8.py @@ -9,9 +9,7 @@ from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, CustomTestCase, - is_in_ci, popen_launch_server, - run_bench_offline_throughput, ) TEST_MODEL_MATRIX = { @@ -76,26 +74,6 @@ class TestAscendMlaW8A8Int8(CustomTestCase): finally: kill_process_tree(process.pid) - def test_b_throughput(self): - for model in self.models: - with self.subTest(model=model): - print(f"##=== Testing throughput: {model} ===##") - - output_throughput = run_bench_offline_throughput( - model, - [ - *self.common_args, - ], - ) - - print(f"##=== {model} throughput: {output_throughput} ===##") - - if is_in_ci(): - self.assertGreater( - output_throughput, - TEST_MODEL_MATRIX[model]["output_throughput"], - ) - if __name__ == "__main__": unittest.main() diff --git a/test/srt/ascend/test_ascend_mla_w8a8int8.py b/test/srt/ascend/test_ascend_mla_w8a8int8.py index 3c3e73366..177af099a 100644 --- a/test/srt/ascend/test_ascend_mla_w8a8int8.py +++ b/test/srt/ascend/test_ascend_mla_w8a8int8.py @@ -8,9 +8,7 @@ from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, CustomTestCase, - is_in_ci, popen_launch_server, - run_bench_offline_throughput, ) TEST_MODEL_MATRIX = { @@ -74,26 +72,6 @@ class TestAscendMlaW8A8Int8(CustomTestCase): finally: kill_process_tree(process.pid) - def test_b_throughput(self): - for model in self.models: - with self.subTest(model=model): - print(f"##=== Testing throughput: {model} ===##") - - output_throughput = run_bench_offline_throughput( - model, - [ - *self.common_args, - ], - ) - - print(f"##=== {model} throughput: {output_throughput} ===##") - - if is_in_ci(): - self.assertGreater( - output_throughput, - TEST_MODEL_MATRIX[model]["output_throughput"], - ) - if __name__ == "__main__": unittest.main() diff --git a/test/srt/ascend/test_ascend_tp2_bf16.py b/test/srt/ascend/test_ascend_tp2_bf16.py index 64f52cd4d..e1f736e9b 100644 --- a/test/srt/ascend/test_ascend_tp2_bf16.py +++ b/test/srt/ascend/test_ascend_tp2_bf16.py @@ -8,9 +8,7 @@ from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, CustomTestCase, - is_in_ci, popen_launch_server, - run_bench_offline_throughput, ) TEST_MODEL_MATRIX = { @@ -73,26 +71,6 @@ class TestAscendTp2Bf16(CustomTestCase): finally: kill_process_tree(process.pid) - def test_b_throughput(self): - for model in self.models: - with self.subTest(model=model): - print(f"##=== Testing throughput: {model} ===##") - - output_throughput = run_bench_offline_throughput( - model, - [ - *self.common_args, - ], - ) - - print(f"##=== {model} throughput: {output_throughput} ===##") - - if is_in_ci(): - self.assertGreater( - output_throughput, - TEST_MODEL_MATRIX[model]["output_throughput"], - ) - if __name__ == "__main__": unittest.main() diff --git a/test/srt/ascend/test_ascend_tp4_bf16.py b/test/srt/ascend/test_ascend_tp4_bf16.py index 9730ffeb3..79efa2445 100644 --- a/test/srt/ascend/test_ascend_tp4_bf16.py +++ b/test/srt/ascend/test_ascend_tp4_bf16.py @@ -7,9 +7,7 @@ from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k from sglang.test.test_utils import ( DEFAULT_URL_FOR_TEST, CustomTestCase, - is_in_ci, popen_launch_server, - run_bench_offline_throughput, ) TEST_MODEL_MATRIX = { @@ -76,26 +74,6 @@ class TestAscendTp4Bf16(CustomTestCase): finally: kill_process_tree(process.pid) - def test_b_throughput(self): - for model in self.models: - with self.subTest(model=model): - print(f"##=== Testing throughput: {model} ===##") - - output_throughput = run_bench_offline_throughput( - model, - [ - *self.common_args, - ], - ) - - print(f"##=== {model} throughput: {output_throughput} ===##") - - if is_in_ci(): - self.assertGreater( - output_throughput, - TEST_MODEL_MATRIX[model]["output_throughput"], - ) - if __name__ == "__main__": unittest.main() diff --git a/test/srt/run_suite.py b/test/srt/run_suite.py index c75eae0b5..a5dc7a145 100644 --- a/test/srt/run_suite.py +++ b/test/srt/run_suite.py @@ -99,15 +99,14 @@ suite_ascend = { TestFile("ascend/test_ascend_tp2_bf16.py", 400), TestFile("ascend/test_ascend_tp2_fia_bf16.py", 400), ], - "per-commit-4-npu-a2": [ + "per-commit-4-npu-a3": [ TestFile("ascend/test_ascend_mla_w8a8int8.py", 400), TestFile("ascend/test_ascend_hicache_mla.py", 400), TestFile("ascend/test_ascend_tp4_bf16.py", 400), + TestFile("ascend/test_ascend_w4a4_quantization.py", 600), ], "per-commit-16-npu-a3": [ TestFile("ascend/test_ascend_deepep.py", 3600), - # TestFile("ascend/test_ascend_deepseek_mtp.py", 2800), - TestFile("ascend/test_ascend_w4a4_quantization.py", 600), ], }