diff --git a/.github/workflows/pr-test-amd.yml b/.github/workflows/pr-test-amd.yml index 432a1f1e9..c2211b9fc 100644 --- a/.github/workflows/pr-test-amd.yml +++ b/.github/workflows/pr-test-amd.yml @@ -315,12 +315,12 @@ jobs: run: | bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-small-1-gpu-amd-mi35x - stage-b-test-large-2-gpu-amd: + stage-b-test-large-1-gpu-amd: needs: [check-changes, stage-a-test-1-amd] if: | always() && ( - (inputs.target_stage == 'stage-b-test-large-2-gpu-amd') || + (inputs.target_stage == 'stage-b-test-large-1-gpu-amd') || ( !inputs.target_stage && (!failure() && !cancelled()) && @@ -330,7 +330,8 @@ jobs: strategy: fail-fast: false matrix: - runner: [linux-mi325-gpu-2] + runner: [linux-mi325-gpu-1] + part: [0, 1] runs-on: ${{matrix.runner}} steps: - name: Checkout code @@ -352,7 +353,47 @@ jobs: - name: Run test timeout-minutes: 30 run: | - bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-large-2-gpu-amd + bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-large-1-gpu-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 --timeout-per-file 1800 + + stage-b-test-large-2-gpu-amd: + needs: [check-changes, stage-a-test-1-amd] + if: | + always() && + ( + (inputs.target_stage == 'stage-b-test-large-2-gpu-amd') || + ( + !inputs.target_stage && + (!failure() && !cancelled()) && + ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) + ) + ) + strategy: + fail-fast: false + matrix: + runner: [linux-mi325-gpu-2] + part: [0, 1] + runs-on: ${{matrix.runner}} + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} + + - name: Ensure VRAM is clear + run: bash scripts/ensure_vram_clear.sh rocm + + - name: Start CI container + run: bash scripts/ci/amd/amd_ci_start_container.sh + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: bash scripts/ci/amd/amd_ci_install_dependency.sh + + - name: Run test + timeout-minutes: 30 + run: | + bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-large-2-gpu-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 --timeout-per-file 1800 multimodal-gen-test-1-gpu-amd: needs: [check-changes] @@ -611,7 +652,7 @@ jobs: fail-fast: false matrix: runner: [linux-mi325-gpu-8] - part: [0, 1] + part: [0, 1, 2] runs-on: ${{matrix.runner}} steps: - name: Checkout code @@ -639,7 +680,7 @@ jobs: - name: Run test timeout-minutes: 60 run: | - bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-c-test-large-8-gpu-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 --timeout-per-file 3600 + bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-c-test-large-8-gpu-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --timeout-per-file 3600 stage-c-test-large-8-gpu-amd-mi35x: needs: [check-changes, call-gate, stage-b-test-small-1-gpu-amd, stage-b-test-large-2-gpu-amd] @@ -657,7 +698,7 @@ jobs: fail-fast: false matrix: runner: [linux-mi35x-gpu-8] - part: [0, 1, 2] + part: [0] runs-on: ${{matrix.runner}} steps: - name: Checkout code @@ -679,203 +720,7 @@ jobs: - name: Run test timeout-minutes: 60 run: | - bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-c-test-large-8-gpu-amd-mi35x --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --timeout-per-file 3600 - - stage-b-test-small-1-gpu-performance-amd: - needs: [check-changes, call-gate, stage-a-test-1-amd] - if: | - always() && - ( - (inputs.target_stage == 'stage-b-test-small-1-gpu-performance-amd') || - ( - !inputs.target_stage && - (!failure() && !cancelled()) && - ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) - ) - ) - strategy: - fail-fast: false - matrix: - runner: [linux-mi325-gpu-1] - runs-on: ${{matrix.runner}} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ensure_vram_clear.sh rocm - - - name: Start CI container - run: bash scripts/ci/amd/amd_ci_start_container.sh - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: bash scripts/ci/amd/amd_ci_install_dependency.sh - - - name: Run test - timeout-minutes: 30 - run: | - bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-small-1-gpu-performance-amd --timeout-per-file 1200 - - stage-b-test-large-1-gpu-performance-amd: - needs: [check-changes, call-gate, stage-a-test-1-amd] - if: | - always() && - ( - (inputs.target_stage == 'stage-b-test-large-1-gpu-performance-amd') || - ( - !inputs.target_stage && - (!failure() && !cancelled()) && - ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) - ) - ) - strategy: - fail-fast: false - matrix: - runner: [linux-mi325-gpu-1] - part: [0, 1] - runs-on: ${{matrix.runner}} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ensure_vram_clear.sh rocm - - - name: Start CI container - run: bash scripts/ci/amd/amd_ci_start_container.sh - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: bash scripts/ci/amd/amd_ci_install_dependency.sh - - - name: Run test - timeout-minutes: 30 - run: | - bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-large-1-gpu-performance-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 --timeout-per-file 1200 - - stage-b-test-large-2-gpu-performance-amd: - needs: [check-changes, call-gate, stage-a-test-1-amd] - if: | - always() && - ( - (inputs.target_stage == 'stage-b-test-large-2-gpu-performance-amd') || - ( - !inputs.target_stage && - (!failure() && !cancelled()) && - ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) - ) - ) - strategy: - fail-fast: false - matrix: - runner: [linux-mi325-gpu-2] - runs-on: ${{matrix.runner}} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ensure_vram_clear.sh rocm - - - name: Start CI container - run: bash scripts/ci/amd/amd_ci_start_container.sh - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: bash scripts/ci/amd/amd_ci_install_dependency.sh - - - name: Run test - timeout-minutes: 30 - run: | - bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-large-2-gpu-performance-amd --timeout-per-file 1200 - - stage-b-test-small-1-gpu-accuracy-amd: - needs: [check-changes, stage-a-test-1-amd] - if: | - always() && - ( - (inputs.target_stage == 'stage-b-test-small-1-gpu-accuracy-amd') || - ( - !inputs.target_stage && - (!failure() && !cancelled()) && - ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) - ) - ) - strategy: - fail-fast: false - matrix: - runner: [linux-mi325-gpu-1] - runs-on: ${{matrix.runner}} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ensure_vram_clear.sh rocm - - - name: Start CI container - run: bash scripts/ci/amd/amd_ci_start_container.sh - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: bash scripts/ci/amd/amd_ci_install_dependency.sh - - - name: Run test - timeout-minutes: 30 - run: | - bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" -e SGLANG_USE_AITER=0 python3 run_suite.py --hw amd --suite stage-b-test-small-1-gpu-accuracy-amd --timeout-per-file 1800 - - stage-b-test-large-2-gpu-accuracy-amd: - needs: [check-changes, stage-a-test-1-amd] - if: | - always() && - ( - (inputs.target_stage == 'stage-b-test-large-2-gpu-accuracy-amd') || - ( - !inputs.target_stage && - (!failure() && !cancelled()) && - ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) - ) - ) - strategy: - fail-fast: false - matrix: - runner: [linux-mi325-gpu-2] - runs-on: ${{matrix.runner}} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} - - - name: Ensure VRAM is clear - run: bash scripts/ensure_vram_clear.sh rocm - - - name: Start CI container - run: bash scripts/ci/amd/amd_ci_start_container.sh - env: - GITHUB_WORKSPACE: ${{ github.workspace }} - - - name: Install dependencies - run: bash scripts/ci/amd/amd_ci_install_dependency.sh - - - name: Run test - timeout-minutes: 30 - run: | - bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" -e SGLANG_USE_AITER_AR=0 -e SGLANG_USE_AITER=0 -e HF_HUB_ENABLE_HF_TRANSFER=0 python3 run_suite.py --hw amd --suite stage-b-test-large-2-gpu-accuracy-amd --timeout-per-file 1800 + bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-c-test-large-8-gpu-amd-mi35x --auto-partition-id ${{ matrix.part }} --auto-partition-size 1 --timeout-per-file 3600 pr-test-amd-finish: needs: @@ -891,12 +736,8 @@ jobs: stage-a-test-1-amd, stage-b-test-small-1-gpu-amd, stage-b-test-small-1-gpu-amd-mi35x, + stage-b-test-large-1-gpu-amd, stage-b-test-large-2-gpu-amd, - stage-b-test-small-1-gpu-performance-amd, - stage-b-test-large-1-gpu-performance-amd, - stage-b-test-large-2-gpu-performance-amd, - stage-b-test-small-1-gpu-accuracy-amd, - stage-b-test-large-2-gpu-accuracy-amd, stage-c-test-large-8-gpu-amd, stage-c-test-large-8-gpu-amd-mi35x, ] diff --git a/scripts/ci/utils/slash_command_handler.py b/scripts/ci/utils/slash_command_handler.py index e787b35a9..66340bebc 100644 --- a/scripts/ci/utils/slash_command_handler.py +++ b/scripts/ci/utils/slash_command_handler.py @@ -253,19 +253,16 @@ def handle_rerun_stage( # Valid AMD stage names that support target_stage amd_stages = [ "sgl-kernel-unit-test-amd", + "sgl-kernel-unit-test-2-gpu-amd", "stage-a-test-1-amd", "stage-b-test-small-1-gpu-amd", "stage-b-test-small-1-gpu-amd-mi35x", + "stage-b-test-large-1-gpu-amd", "stage-b-test-large-2-gpu-amd", - "stage-b-test-small-1-gpu-performance-amd", - "stage-b-test-large-1-gpu-performance-amd", - "stage-b-test-large-2-gpu-performance-amd", + "multimodal-gen-test-1-gpu-amd", + "multimodal-gen-test-2-gpu-amd", + "stage-c-test-large-8-gpu-amd", "stage-c-test-large-8-gpu-amd-mi35x", - "unit-test-backend-1-gpu-amd", - "unit-test-backend-2-gpu-amd", - "unit-test-backend-8-gpu-amd", - "accuracy-test-1-gpu-amd", - "accuracy-test-2-gpu-amd", ] valid_stages = nvidia_stages + amd_stages diff --git a/test/registered/amd/test_deepseek_v32_basic.py b/test/registered/amd/test_deepseek_v32_basic.py index 1d2e35d48..cd849897e 100644 --- a/test/registered/amd/test_deepseek_v32_basic.py +++ b/test/registered/amd/test_deepseek_v32_basic.py @@ -15,11 +15,7 @@ from sglang.test.test_utils import ( write_github_step_summary, ) -register_amd_ci( - est_time=3600, - suite="stage-c-test-large-8-gpu-amd-mi35x", - disabled="move to nightly for saving time", -) +register_amd_ci(est_time=1800, suite="stage-c-test-large-8-gpu-amd") DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2" @@ -159,7 +155,7 @@ class TestDeepseekV32TP(CustomTestCase): f"### test_bs_1_speed (deepseek-v32)\n" f"{speed=:.2f} token/s\n" ) if is_in_amd_ci(): - self.assertGreater(speed, 20) + self.assertGreater(speed, 15) else: self.assertGreater(speed, 70) diff --git a/test/registered/amd/test_deepseek_v32_mtp.py b/test/registered/amd/test_deepseek_v32_mtp.py index 24e0143e0..87e4e6923 100644 --- a/test/registered/amd/test_deepseek_v32_mtp.py +++ b/test/registered/amd/test_deepseek_v32_mtp.py @@ -17,7 +17,12 @@ from sglang.test.test_utils import ( write_github_step_summary, ) -register_amd_ci(est_time=3600, suite="stage-c-test-large-8-gpu-amd-mi35x") +register_amd_ci( + est_time=3600, + suite="stage-c-test-large-8-gpu-amd-mi35x", + disabled="move to nightly for saving time", +) + FULL_DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2" diff --git a/test/registered/amd/test_kimi_k2_instruct.py b/test/registered/amd/test_kimi_k2_instruct.py index 34761396e..896d3b154 100644 --- a/test/registered/amd/test_kimi_k2_instruct.py +++ b/test/registered/amd/test_kimi_k2_instruct.py @@ -11,12 +11,13 @@ from sglang.test.send_one import BenchArgs, send_one_prompt from sglang.test.test_utils import ( DEFAULT_URL_FOR_TEST, CustomTestCase, + is_in_amd_ci, is_in_ci, popen_launch_server, write_github_step_summary, ) -register_amd_ci(est_time=3600, suite="stage-c-test-large-8-gpu-amd-mi35x") +register_amd_ci(est_time=3600, suite="stage-c-test-large-8-gpu-amd") KIMI_K2_MODEL_PATH = "moonshotai/Kimi-K2-Instruct-0905" SERVER_LAUNCH_TIMEOUT = 3600 @@ -88,7 +89,10 @@ class TestKimiK2Instruct0905(CustomTestCase): f"### test_bs_1_speed (Kimi-K2-Instruct-0905)\n" f"{speed=:.2f} token/s\n" ) - self.assertGreater(speed, 45) + if is_in_amd_ci(): + self.assertGreater(speed, 30) + else: + self.assertGreater(speed, 45) if __name__ == "__main__": diff --git a/test/registered/attention/test_triton_sliding_window.py b/test/registered/attention/test_triton_sliding_window.py index afad309c7..3f4df13a5 100644 --- a/test/registered/attention/test_triton_sliding_window.py +++ b/test/registered/attention/test_triton_sliding_window.py @@ -10,6 +10,7 @@ from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, CustomTestCase, + is_in_amd_ci, is_in_ci, popen_launch_server, ) @@ -62,7 +63,10 @@ class TestSlidingWindowAttentionTriton(CustomTestCase): metrics = run_eval(args) print(f"MMLU metrics with sliding window: {metrics}") - self.assertGreaterEqual(metrics["score"], 0.60) + if is_in_amd_ci(): + self.assertGreaterEqual(metrics["score"], 0.55) + else: + self.assertGreaterEqual(metrics["score"], 0.60) def _test_short_context_generation(self): response = requests.post( diff --git a/test/registered/eval/test_eval_accuracy_large.py b/test/registered/eval/test_eval_accuracy_large.py index 486e45afa..e7328c29f 100644 --- a/test/registered/eval/test_eval_accuracy_large.py +++ b/test/registered/eval/test_eval_accuracy_large.py @@ -14,6 +14,7 @@ from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, CustomTestCase, + is_in_amd_ci, is_in_ci, popen_launch_server, write_github_step_summary, @@ -71,7 +72,10 @@ class TestEvalAccuracyLarge(CustomTestCase): f"### test_human_eval\n" f'{metrics["score"]=:.4f}\n' ) - self.assertGreater(metrics["score"], 0.64) + if is_in_amd_ci(): + self.assertGreater(metrics["score"], 0.60) + else: + self.assertGreater(metrics["score"], 0.64) def test_mgsm_en(self): args = SimpleNamespace( diff --git a/test/registered/eval/test_moe_eval_accuracy_large.py b/test/registered/eval/test_moe_eval_accuracy_large.py index ae90f768e..369835ef4 100644 --- a/test/registered/eval/test_moe_eval_accuracy_large.py +++ b/test/registered/eval/test_moe_eval_accuracy_large.py @@ -3,6 +3,7 @@ Usage: python -m unittest test_moe_eval_accuracy_large.TestMoEEvalAccuracyLarge.test_mmlu """ +import os import unittest from types import SimpleNamespace @@ -14,6 +15,7 @@ from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, CustomTestCase, + is_in_amd_ci, is_in_ci, popen_launch_server, write_github_step_summary, @@ -28,6 +30,15 @@ class TestMoEEvalAccuracyLarge(CustomTestCase): def setUpClass(cls): cls.model = DEFAULT_MOE_MODEL_NAME_FOR_TEST cls.base_url = DEFAULT_URL_FOR_TEST + + # Disable AITER for AMD CI to ensure consistent results + env = None + if is_in_amd_ci(): + env = os.environ.copy() + env["SGLANG_USE_AITER"] = "0" + env["SGLANG_USE_AITER_AR"] = "0" + env["HF_HUB_ENABLE_HF_TRANSFER"] = "0" + cls.process = popen_launch_server( cls.model, cls.base_url, @@ -38,6 +49,7 @@ class TestMoEEvalAccuracyLarge(CustomTestCase): "--tp", "2", ], + env=env, ) @classmethod diff --git a/test/run_suite.py b/test/run_suite.py index 45cd6299f..f682bdbc9 100644 --- a/test/run_suite.py +++ b/test/run_suite.py @@ -24,11 +24,6 @@ PER_COMMIT_SUITES = { "stage-b-test-small-1-gpu-amd-mi35x", "stage-b-test-large-1-gpu-amd", "stage-b-test-large-2-gpu-amd", - "stage-b-test-small-1-gpu-performance-amd", - "stage-b-test-large-1-gpu-performance-amd", - "stage-b-test-large-2-gpu-performance-amd", - "stage-b-test-small-1-gpu-accuracy-amd", - "stage-b-test-large-2-gpu-accuracy-amd", "stage-c-test-large-8-gpu-amd-mi35x", ], HWBackend.CUDA: [