From b86bbf841e813716fa82c9e1cd08e3c267feb682 Mon Sep 17 00:00:00 2001 From: Hubert Lu <55214931+hubertlu-tw@users.noreply.github.com> Date: Wed, 7 Jan 2026 01:43:11 -0800 Subject: [PATCH] [AMD] Add 8-GPU MX35X test running DSR1-MXFP4 model for AMD CI (#13602) --- .github/workflows/pr-test-amd.yml | 70 ++++++++++ scripts/ci/amd_ci_exec.sh | 3 +- scripts/ci/amd_ci_install_dependency.sh | 5 +- test/srt/run_suite.py | 3 + test/srt/test_deepseek_r1_mxfp4_8gpu.py | 165 ++++++++++++++++++++++++ 5 files changed, 243 insertions(+), 3 deletions(-) create mode 100644 test/srt/test_deepseek_r1_mxfp4_8gpu.py diff --git a/.github/workflows/pr-test-amd.yml b/.github/workflows/pr-test-amd.yml index 3a98c5504..63b6c1fd4 100644 --- a/.github/workflows/pr-test-amd.yml +++ b/.github/workflows/pr-test-amd.yml @@ -543,6 +543,45 @@ jobs: run: | bash scripts/ci/amd_ci_exec.sh python3 run_suite.py --suite per-commit-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 12 + unit-test-backend-1-gpu-amd-mi35x: + needs: [check-changes, stage-a-test-1-amd] + if: | + always() && + ( + (inputs.target_stage == 'unit-test-backend-1-gpu-amd-mi35x') || + ( + !inputs.target_stage && + (!failure() && !cancelled()) && + ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) + ) + ) + strategy: + fail-fast: false + matrix: + runner: [linux-mi35x-gpu-1] + runs-on: ${{matrix.runner}} + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} + + - name: Ensure VRAM is clear + run: bash scripts/ensure_vram_clear.sh rocm + + - name: Start CI container + run: bash scripts/ci/amd_ci_start_container.sh + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: bash scripts/ci/amd_ci_install_dependency.sh + + - name: Run test + timeout-minutes: 15 + run: | + bash scripts/ci/amd_ci_exec.sh python3 run_suite.py --suite per-commit-amd-mi35x + unit-test-backend-2-gpu-amd: needs: [check-changes, stage-a-test-1-amd] if: | @@ -631,6 +670,35 @@ jobs: run: | bash scripts/ci/amd_ci_exec.sh python3 run_suite.py --suite per-commit-8-gpu-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 --timeout-per-file 3600 + unit-test-backend-8-gpu-amd-mi35x: + needs: [check-changes, unit-test-backend-2-gpu-amd] + if: always() && !failure() && !cancelled() && + ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) + strategy: + fail-fast: false + matrix: + runner: [linux-mi35x-gpu-8] + runs-on: ${{matrix.runner}} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Ensure VRAM is clear + run: bash scripts/ensure_vram_clear.sh rocm + + - name: Start CI container + run: bash scripts/ci/amd_ci_start_container.sh + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: bash scripts/ci/amd_ci_install_dependency.sh + + - name: Run test + timeout-minutes: 30 + run: | + bash scripts/ci/amd_ci_exec.sh python3 run_suite.py --suite per-commit-8-gpu-amd-mi35x --timeout-per-file 1800 + performance-test-1-gpu-part-1-amd: needs: [check-changes, stage-a-test-1-amd] if: | @@ -897,8 +965,10 @@ jobs: stage-b-test-small-1-gpu-amd, stage-b-test-large-2-gpu-amd, unit-test-backend-1-gpu-amd, + unit-test-backend-1-gpu-amd-mi35x, unit-test-backend-2-gpu-amd, unit-test-backend-8-gpu-amd, + unit-test-backend-8-gpu-amd-mi35x, performance-test-1-gpu-part-1-amd, performance-test-1-gpu-part-2-amd, performance-test-2-gpu-amd, diff --git a/scripts/ci/amd_ci_exec.sh b/scripts/ci/amd_ci_exec.sh index ff30a4f13..84b1f347b 100755 --- a/scripts/ci/amd_ci_exec.sh +++ b/scripts/ci/amd_ci_exec.sh @@ -60,9 +60,10 @@ if docker exec \ "${ENV_ARGS[@]}" \ ci_sglang "$@"; then exit 0 +else + FIRST_EXIT_CODE=$? fi -FIRST_EXIT_CODE=$? echo "First attempt failed with exit code $FIRST_EXIT_CODE" echo "Retrying with HF_HUB_OFFLINE=1 (offline mode)..." diff --git a/scripts/ci/amd_ci_install_dependency.sh b/scripts/ci/amd_ci_install_dependency.sh index d2081e573..8875b7e92 100755 --- a/scripts/ci/amd_ci_install_dependency.sh +++ b/scripts/ci/amd_ci_install_dependency.sh @@ -99,10 +99,11 @@ case "${GPU_ARCH}" in mi35x) echo "Runner uses ${GPU_ARCH}; will fetch mi35x image." docker exec ci_sglang rm -rf python/pyproject.toml && mv python/pyproject_other.toml python/pyproject.toml - install_with_retry docker exec ci_sglang pip install --cache-dir=/sgl-data/pip-cache -e "python[${EXTRAS}]" --no-deps # TODO: only for mi35x + # Follow the same dependency installation flow as mi30x/mi300/mi325. + install_with_retry docker exec ci_sglang pip install --cache-dir=/sgl-data/pip-cache -e "python[${EXTRAS}]" # For lmms_evals evaluating MMMU docker exec -w / ci_sglang git clone --branch v0.4.1 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git - install_with_retry docker exec -w /lmms-eval ci_sglang pip install --cache-dir=/sgl-data/pip-cache -e . --no-deps + install_with_retry docker exec -w /lmms-eval ci_sglang pip install --cache-dir=/sgl-data/pip-cache -e . ;; mi30x|mi300|mi325) echo "Runner uses ${GPU_ARCH}; will fetch mi30x image." diff --git a/test/srt/run_suite.py b/test/srt/run_suite.py index e3dd9befb..e091abb96 100644 --- a/test/srt/run_suite.py +++ b/test/srt/run_suite.py @@ -157,6 +157,9 @@ suite_amd = { TestFile("test_deepseek_v3_basic.py", 275), TestFile("test_deepseek_v3_mtp.py", 275), ], + "per-commit-8-gpu-amd-mi35x": [ + TestFile("test_deepseek_r1_mxfp4_8gpu.py", 1800), + ], "nightly-amd": [ TestFile("nightly/test_gsm8k_eval_amd.py"), ], diff --git a/test/srt/test_deepseek_r1_mxfp4_8gpu.py b/test/srt/test_deepseek_r1_mxfp4_8gpu.py new file mode 100644 index 000000000..fc825b9b4 --- /dev/null +++ b/test/srt/test_deepseek_r1_mxfp4_8gpu.py @@ -0,0 +1,165 @@ +import os +import unittest +from types import SimpleNamespace + +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.send_one import BenchArgs, send_one_prompt +from sglang.test.test_utils import ( + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, + write_github_step_summary, +) + +DEEPSEEK_R1_MODEL_PATH = "amd/DeepSeek-R1-MXFP4-Preview" +SERVER_LAUNCH_TIMEOUT = 1000 + + +class TestDeepseekR1MXFP4(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEEPSEEK_R1_MODEL_PATH + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = [ + "--tp", + "8", + "--chunked-prefill-size", + "131072", + "--model-loader-extra-config", + '{"enable_multithread_load": true}', + ] + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=SERVER_LAUNCH_TIMEOUT, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_a_gsm8k( + self, + ): # Append an "a" to make this test run first (alphabetically) to warm up the server + requests.get(self.base_url + "/flush_cache") + + args = SimpleNamespace( + num_shots=8, + data_path=None, + num_questions=1319, + parallel=1319, + max_new_tokens=512, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(f"{metrics=}") + + if is_in_ci(): + write_github_step_summary( + f"### test_gsm8k (deepseek-r1-mxfp4)\n" f'{metrics["accuracy"]=:.3f}\n' + ) + self.assertGreater(metrics["accuracy"], 0.94) + + def test_bs_1_speed(self): + args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048) + _, speed = send_one_prompt(args) + + print(f"{speed=:.2f}") + + if is_in_ci(): + write_github_step_summary( + f"### test_bs_1_speed (deepseek-r1-mxfp4)\n" f"{speed=:.2f} token/s\n" + ) + self.assertGreater(speed, 75) + + +class TestDeepseekR1MXFP4MTP(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEEPSEEK_R1_MODEL_PATH + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = [ + "--tp", + "8", + "--chunked-prefill-size", + "131072", + "--speculative-algorithm", + "EAGLE", + "--speculative-num-steps", + "3", + "--speculative-eagle-topk", + "1", + "--speculative-num-draft-tokens", + "4", + "--model-loader-extra-config", + '{"enable_multithread_load": true}', + ] + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=SERVER_LAUNCH_TIMEOUT, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + if "SGLANG_ENABLE_SPEC_V2" in os.environ: + del os.environ["SGLANG_ENABLE_SPEC_V2"] + + def test_a_gsm8k( + self, + ): # Append an "a" to make this test run first (alphabetically) to warm up the server + requests.get(self.base_url + "/flush_cache") + + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(f"{metrics=}") + + server_info = requests.get(self.base_url + "/get_server_info") + avg_spec_accept_length = server_info.json()["internal_states"][0][ + "avg_spec_accept_length" + ] + print(f"{avg_spec_accept_length=}") + + if is_in_ci(): + write_github_step_summary( + f"### test_gsm8k (deepseek-r1-mxfp4 mtp)\n" + f'{metrics["accuracy"]=:.3f}\n' + f"{avg_spec_accept_length=:.2f}\n" + ) + self.assertGreater(metrics["accuracy"], 0.94) + self.assertGreater(avg_spec_accept_length, 2.04) + + def test_bs_1_speed(self): + args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048) + acc_length, speed = send_one_prompt(args) + + print(f"{acc_length=:.2f} {speed=:.2f}") + + if is_in_ci(): + write_github_step_summary( + f"### test_bs_1_speed (deepseek-r1-mxfp4 mtp)\n" + f"{acc_length=:.2f}\n" + f"{speed=:.2f} token/s\n" + ) + self.assertGreater(acc_length, 2.04) + self.assertGreater(speed, 150) + + +if __name__ == "__main__": + unittest.main()