From c1da4207990477e2b75e82710070af1933d998af Mon Sep 17 00:00:00 2001 From: Lianmin Zheng Date: Fri, 20 Mar 2026 02:55:16 -0700 Subject: [PATCH] ci: run Stage A CUDA tests as stage-a-test-small-1-gpu on 5090 (#20988) --- .github/actions/wait-for-jobs/action.yml | 2 +- .github/workflows/pr-test-amd-rocm720.yml | 10 +- .github/workflows/pr-test-amd.yml | 22 ++-- .github/workflows/pr-test.yml | 103 +++++++----------- scripts/ci/utils/slash_command_handler.py | 11 +- test/README.md | 8 +- .../attention/test_wave_attention_kernels.py | 2 +- .../debug_utils/test_cuda_coredump_smoke.py | 2 +- test/registered/quant/test_awq_dequant.py | 2 +- .../quant/test_fused_rms_fp8_group_quant.py | 2 +- test/registered/test_srt_backend.py | 8 +- test/run_suite.py | 6 +- 12 files changed, 78 insertions(+), 100 deletions(-) diff --git a/.github/actions/wait-for-jobs/action.yml b/.github/actions/wait-for-jobs/action.yml index 2b5d4652e..36169fd20 100644 --- a/.github/actions/wait-for-jobs/action.yml +++ b/.github/actions/wait-for-jobs/action.yml @@ -8,7 +8,7 @@ inputs: jobs: description: | JSON array of job specs to wait for. Each element is either: - - a string: exact job name (e.g. "stage-a-test-1") + - a string: exact job name (e.g. "stage-a-test-small-1-gpu") - an object { "prefix": "...", "expected_count": N }: for matrix jobs required: true max-wait-minutes: diff --git a/.github/workflows/pr-test-amd-rocm720.yml b/.github/workflows/pr-test-amd-rocm720.yml index fd78f9a8b..5e21e505b 100644 --- a/.github/workflows/pr-test-amd-rocm720.yml +++ b/.github/workflows/pr-test-amd-rocm720.yml @@ -35,7 +35,7 @@ on: - '' - sgl-kernel-unit-test-amd - sgl-kernel-unit-test-2-gpu-amd - - stage-a-test-1-amd + - stage-a-test-small-1-gpu-amd - jit-kernel-unit-test-amd - stage-b-test-small-1-gpu-amd - stage-b-test-small-1-gpu-amd-nondeterministic @@ -246,12 +246,12 @@ jobs: # =============================================== primary ==================================================== - stage-a-test-1-amd: + stage-a-test-small-1-gpu-amd: needs: [check-changes] if: | always() && ( - (contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',stage-a-test-1-amd,')) || + (contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',stage-a-test-small-1-gpu-amd,')) || ( !(inputs.target_stage || inputs.target_stage_select) && (!failure() && !cancelled()) && @@ -283,7 +283,7 @@ jobs: - name: Run test timeout-minutes: 10 run: | - bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-a-test-1-amd ${{ inputs.continue_on_error && '--continue-on-error' || '' }} + bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-a-test-small-1-gpu-amd ${{ inputs.continue_on_error && '--continue-on-error' || '' }} jit-kernel-unit-test-amd: needs: [check-changes] @@ -984,7 +984,7 @@ jobs: multimodal-gen-test-1-gpu-amd, multimodal-gen-test-2-gpu-amd, - stage-a-test-1-amd, + stage-a-test-small-1-gpu-amd, jit-kernel-unit-test-amd, stage-b-test-small-1-gpu-amd, stage-b-test-small-1-gpu-amd-nondeterministic, diff --git a/.github/workflows/pr-test-amd.yml b/.github/workflows/pr-test-amd.yml index 3e821b8d8..d231527ae 100644 --- a/.github/workflows/pr-test-amd.yml +++ b/.github/workflows/pr-test-amd.yml @@ -33,7 +33,7 @@ on: - '' - sgl-kernel-unit-test-amd - sgl-kernel-unit-test-2-gpu-amd - - stage-a-test-1-amd + - stage-a-test-small-1-gpu-amd - jit-kernel-unit-test-amd - stage-b-test-small-1-gpu-amd - stage-b-test-small-1-gpu-amd-nondeterministic @@ -246,12 +246,12 @@ jobs: # =============================================== primary ==================================================== - stage-a-test-1-amd: + stage-a-test-small-1-gpu-amd: needs: [check-changes] if: | always() && ( - (contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',stage-a-test-1-amd,')) || + (contains(format(',{0},', inputs.target_stage || inputs.target_stage_select), ',stage-a-test-small-1-gpu-amd,')) || ( !(inputs.target_stage || inputs.target_stage_select) && (!failure() && !cancelled()) && @@ -284,7 +284,7 @@ jobs: - name: Run test timeout-minutes: 10 run: | - bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-a-test-1-amd ${{ inputs.continue_on_error && '--continue-on-error' || '' }} + bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-a-test-small-1-gpu-amd ${{ inputs.continue_on_error && '--continue-on-error' || '' }} jit-kernel-unit-test-amd: needs: [check-changes] @@ -326,7 +326,7 @@ jobs: bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout" python3 -m pytest -q python/sglang/jit_kernel/tests/test_store_cache.py stage-b-test-small-1-gpu-amd: - needs: [check-changes, stage-a-test-1-amd] + needs: [check-changes, stage-a-test-small-1-gpu-amd] if: | always() && ( @@ -366,7 +366,7 @@ jobs: bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-small-1-gpu-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 14 --timeout-per-file 1800 ${{ inputs.continue_on_error && '--continue-on-error' || '' }} stage-b-test-small-1-gpu-amd-nondeterministic: - needs: [check-changes, stage-a-test-1-amd] + needs: [check-changes, stage-a-test-small-1-gpu-amd] if: | always() && ( @@ -405,7 +405,7 @@ jobs: bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-small-1-gpu-amd-nondeterministic --timeout-per-file 1800 ${{ inputs.continue_on_error && '--continue-on-error' || '' }} stage-b-test-small-1-gpu-amd-mi35x: - needs: [check-changes, stage-a-test-1-amd] + needs: [check-changes, stage-a-test-small-1-gpu-amd] if: | always() && ( @@ -444,7 +444,7 @@ jobs: bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-small-1-gpu-amd-mi35x ${{ inputs.continue_on_error && '--continue-on-error' || '' }} stage-b-test-large-1-gpu-amd: - needs: [check-changes, stage-a-test-1-amd] + needs: [check-changes, stage-a-test-small-1-gpu-amd] if: | always() && ( @@ -484,7 +484,7 @@ jobs: bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-large-1-gpu-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 --timeout-per-file 1800 ${{ inputs.continue_on_error && '--continue-on-error' || '' }} stage-b-test-large-2-gpu-amd: - needs: [check-changes, stage-a-test-1-amd] + needs: [check-changes, stage-a-test-small-1-gpu-amd] if: | always() && ( @@ -868,7 +868,7 @@ jobs: # =============================================== Disaggregation ==================================================== stage-b-test-large-8-gpu-35x-disaggregation-amd: - needs: [check-changes, stage-a-test-1-amd] + needs: [check-changes, stage-a-test-small-1-gpu-amd] if: | always() && ( @@ -989,7 +989,7 @@ jobs: multimodal-gen-test-1-gpu-amd, multimodal-gen-test-2-gpu-amd, - stage-a-test-1-amd, + stage-a-test-small-1-gpu-amd, jit-kernel-unit-test-amd, stage-b-test-small-1-gpu-amd, stage-b-test-small-1-gpu-amd-nondeterministic, diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index 33b603d18..18102043d 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -333,7 +333,7 @@ jobs: id: wait with: stage-name: stage-a - jobs: '["stage-a-test-1", "stage-a-cpu-only"]' + jobs: '["stage-a-test-small-1-gpu", "stage-a-cpu-only"]' max-wait-minutes: '240' wait-for-stage-b: @@ -783,20 +783,23 @@ jobs: # =============================================== primary ==================================================== - stage-a-test-1: + # Runs on 5090 (32GB, SM120) + stage-a-test-small-1-gpu: needs: [check-changes, call-gate, sgl-kernel-build-wheels] if: | always() && ( - (inputs.target_stage == 'stage-a-test-1') || + (inputs.target_stage == 'stage-a-test-small-1-gpu') || ( !inputs.target_stage && ((github.event_name == 'schedule' || inputs.test_parallel_dispatch == true) || (!failure() && !cancelled())) && ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) ) ) - runs-on: 1-gpu-runner + runs-on: 1-gpu-5090 timeout-minutes: 240 + env: + IS_BLACKWELL: "1" steps: - name: Checkout code uses: actions/checkout@v4 @@ -818,13 +821,11 @@ jobs: - name: Run test timeout-minutes: 10 + env: + CONTINUE_ON_ERROR_FLAG: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} run: | cd test/ - CONTINUE_ON_ERROR_FLAG="" - if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then - CONTINUE_ON_ERROR_FLAG="--continue-on-error" - fi - python3 run_suite.py --hw cuda --suite stage-a-test-1 $CONTINUE_ON_ERROR_FLAG + python3 run_suite.py --hw cuda --suite stage-a-test-small-1-gpu $CONTINUE_ON_ERROR_FLAG - uses: ./.github/actions/upload-cuda-coredumps if: always() @@ -866,12 +867,10 @@ jobs: - name: Run test timeout-minutes: 10 + env: + CONTINUE_ON_ERROR_FLAG: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} run: | cd test/ - CONTINUE_ON_ERROR_FLAG="" - if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then - CONTINUE_ON_ERROR_FLAG="--continue-on-error" - fi python3 run_suite.py --hw cpu --suite stage-a-cpu-only $CONTINUE_ON_ERROR_FLAG # Runs on 5090 (32GB, SM120) @@ -921,13 +920,11 @@ jobs: - name: Run test timeout-minutes: 30 + env: + CONTINUE_ON_ERROR_FLAG: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} run: | source /etc/profile.d/sglang-ci.sh cd test/ - CONTINUE_ON_ERROR_FLAG="" - if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then - CONTINUE_ON_ERROR_FLAG="--continue-on-error" - fi python3 run_suite.py --hw cuda --suite stage-b-test-small-1-gpu --auto-partition-id ${{ matrix.partition }} --auto-partition-size 8 $CONTINUE_ON_ERROR_FLAG - uses: ./.github/actions/upload-cuda-coredumps @@ -976,12 +973,10 @@ jobs: - name: Run test timeout-minutes: 30 + env: + CONTINUE_ON_ERROR_FLAG: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} run: | cd test/ - CONTINUE_ON_ERROR_FLAG="" - if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then - CONTINUE_ON_ERROR_FLAG="--continue-on-error" - fi python3 run_suite.py --hw cuda --suite stage-b-test-large-1-gpu --auto-partition-id ${{ matrix.partition }} --auto-partition-size 14 --timeout-per-file 1800 $CONTINUE_ON_ERROR_FLAG - uses: ./.github/actions/upload-cuda-coredumps @@ -1031,12 +1026,10 @@ jobs: - name: Run test timeout-minutes: 30 + env: + CONTINUE_ON_ERROR_FLAG: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} run: | cd test/ - CONTINUE_ON_ERROR_FLAG="" - if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then - CONTINUE_ON_ERROR_FLAG="--continue-on-error" - fi python3 run_suite.py --hw cuda --suite stage-b-test-large-2-gpu --auto-partition-id ${{ matrix.partition }} --auto-partition-size 4 $CONTINUE_ON_ERROR_FLAG - uses: ./.github/actions/upload-cuda-coredumps @@ -1082,12 +1075,10 @@ jobs: - name: Run test timeout-minutes: 30 + env: + CONTINUE_ON_ERROR_FLAG: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} run: | cd test - CONTINUE_ON_ERROR_FLAG="" - if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then - CONTINUE_ON_ERROR_FLAG="--continue-on-error" - fi IS_BLACKWELL=1 python3 run_suite.py --hw cuda --suite stage-b-test-4-gpu-b200 $CONTINUE_ON_ERROR_FLAG - name: Run FA4 jit_kernel tests (SM100+) @@ -1138,12 +1129,9 @@ jobs: timeout-minutes: 240 env: RUNAI_STREAMER_MEMORY_LIMIT: 0 + CONTINUE_ON_ERROR_FLAG: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} run: | cd python - CONTINUE_ON_ERROR_FLAG="" - if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then - CONTINUE_ON_ERROR_FLAG="--continue-on-error" - fi python3 sglang/multimodal_gen/test/run_suite.py \ --suite 1-gpu \ --partition-id ${{ matrix.part }} \ @@ -1196,12 +1184,9 @@ jobs: timeout-minutes: 240 env: RUNAI_STREAMER_MEMORY_LIMIT: 0 + CONTINUE_ON_ERROR_FLAG: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} run: | cd python - CONTINUE_ON_ERROR_FLAG="" - if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then - CONTINUE_ON_ERROR_FLAG="--continue-on-error" - fi python3 sglang/multimodal_gen/test/run_suite.py \ --suite 2-gpu \ --partition-id ${{ matrix.part }} \ @@ -1291,12 +1276,10 @@ jobs: - name: Run test timeout-minutes: 30 + env: + CONTINUE_ON_ERROR_FLAG: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} run: | cd test - CONTINUE_ON_ERROR_FLAG="" - if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then - CONTINUE_ON_ERROR_FLAG="--continue-on-error" - fi python3 run_suite.py --hw cuda --suite stage-c-test-4-gpu-h100 --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 $CONTINUE_ON_ERROR_FLAG - uses: ./.github/actions/upload-cuda-coredumps @@ -1357,12 +1340,10 @@ jobs: - name: Run test timeout-minutes: 30 + env: + CONTINUE_ON_ERROR_FLAG: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} run: | cd test - CONTINUE_ON_ERROR_FLAG="" - if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then - CONTINUE_ON_ERROR_FLAG="--continue-on-error" - fi python3 run_suite.py --hw cuda --suite stage-c-test-8-gpu-h200 --auto-partition-id ${{ matrix.part }} --auto-partition-size 4 $CONTINUE_ON_ERROR_FLAG - uses: ./.github/actions/upload-cuda-coredumps @@ -1411,12 +1392,10 @@ jobs: - name: Run test timeout-minutes: 30 + env: + CONTINUE_ON_ERROR_FLAG: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} run: | cd test - CONTINUE_ON_ERROR_FLAG="" - if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then - CONTINUE_ON_ERROR_FLAG="--continue-on-error" - fi python3 run_suite.py --hw cuda --suite stage-c-test-8-gpu-h20 --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 $CONTINUE_ON_ERROR_FLAG - uses: ./.github/actions/upload-cuda-coredumps @@ -1471,12 +1450,10 @@ jobs: - name: Run test timeout-minutes: 30 + env: + CONTINUE_ON_ERROR_FLAG: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} run: | cd test - CONTINUE_ON_ERROR_FLAG="" - if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then - CONTINUE_ON_ERROR_FLAG="--continue-on-error" - fi python3 run_suite.py --hw cuda --suite stage-c-test-deepep-4-gpu $CONTINUE_ON_ERROR_FLAG - uses: ./.github/actions/upload-cuda-coredumps @@ -1530,12 +1507,10 @@ jobs: - name: Run test timeout-minutes: 45 + env: + CONTINUE_ON_ERROR_FLAG: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} run: | cd test - CONTINUE_ON_ERROR_FLAG="" - if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then - CONTINUE_ON_ERROR_FLAG="--continue-on-error" - fi python3 run_suite.py --hw cuda --suite stage-c-test-deepep-8-gpu-h200 $CONTINUE_ON_ERROR_FLAG - uses: ./.github/actions/upload-cuda-coredumps @@ -1581,12 +1556,10 @@ jobs: - name: Run test timeout-minutes: 30 + env: + CONTINUE_ON_ERROR_FLAG: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} run: | cd test - CONTINUE_ON_ERROR_FLAG="" - if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then - CONTINUE_ON_ERROR_FLAG="--continue-on-error" - fi IS_BLACKWELL=1 python3 run_suite.py --hw cuda --suite stage-c-test-4-gpu-b200 --auto-partition-id ${{ matrix.part }} --auto-partition-size 4 --timeout-per-file 1800 $CONTINUE_ON_ERROR_FLAG - uses: ./.github/actions/upload-cuda-coredumps @@ -1633,12 +1606,10 @@ jobs: # # - name: Run test # timeout-minutes: 45 + # env: + # CONTINUE_ON_ERROR_FLAG: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} # run: | # cd test - # CONTINUE_ON_ERROR_FLAG="" - # if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then - # CONTINUE_ON_ERROR_FLAG="--continue-on-error" - # fi # python3 run_suite.py --hw cuda --suite stage-c-test-4-gpu-gb200 --timeout-per-file 3600 $CONTINUE_ON_ERROR_FLAG # # - uses: ./.github/actions/upload-cuda-coredumps @@ -1667,7 +1638,7 @@ jobs: multimodal-gen-test-1-gpu, multimodal-gen-test-2-gpu, - stage-a-test-1, + stage-a-test-small-1-gpu, stage-a-cpu-only, stage-b-test-small-1-gpu, stage-b-test-large-1-gpu, diff --git a/scripts/ci/utils/slash_command_handler.py b/scripts/ci/utils/slash_command_handler.py index 7d53dd0b0..65c44ccf9 100644 --- a/scripts/ci/utils/slash_command_handler.py +++ b/scripts/ci/utils/slash_command_handler.py @@ -253,7 +253,7 @@ def handle_rerun_stage( # Valid NVIDIA stage names that support target_stage nvidia_stages = [ - "stage-a-test-1", + "stage-a-test-small-1-gpu", "stage-a-cpu-only", "stage-b-test-small-1-gpu", "stage-b-test-large-1-gpu", @@ -274,7 +274,7 @@ def handle_rerun_stage( amd_stages = [ "sgl-kernel-unit-test-amd", "sgl-kernel-unit-test-2-gpu-amd", - "stage-a-test-1-amd", + "stage-a-test-small-1-gpu-amd", "stage-b-test-small-1-gpu-amd", "stage-b-test-small-1-gpu-amd-nondeterministic", "stage-b-test-small-1-gpu-amd-mi35x", @@ -334,7 +334,10 @@ def handle_rerun_stage( f"Triggering {workflow_name} workflow on ref: {ref}, PR head SHA: {pr_head_sha}" ) if is_amd_stage: - inputs = {"target_stage": stage_name, "pr_head_sha": pr_head_sha} + inputs = { + "target_stage": stage_name, + "pr_head_sha": pr_head_sha, + } else: inputs = { "version": "release", @@ -410,7 +413,7 @@ def handle_rerun_stage( CUDA_SUITE_TO_RUNNER = { - "stage-a-test-1": "1-gpu-runner", + "stage-a-test-small-1-gpu": "1-gpu-5090", "stage-a-cpu-only": "ubuntu-latest", "stage-b-test-small-1-gpu": "1-gpu-5090", "stage-b-test-large-1-gpu": "1-gpu-runner", diff --git a/test/README.md b/test/README.md index 1dc3c6d35..e7621d194 100644 --- a/test/README.md +++ b/test/README.md @@ -60,7 +60,7 @@ register_cuda_ci(est_time=200, suite="nightly-1-gpu", nightly=True) # Multi-backend test register_cuda_ci(est_time=80, suite="stage-b-test-small-1-gpu") -register_amd_ci(est_time=120, suite="stage-a-test-1") +register_amd_ci(est_time=120, suite="stage-a-test-small-1-gpu-amd") # Temporarily disabled test register_cuda_ci(est_time=80, suite="stage-b-test-small-1-gpu", disabled="flaky - see #12345") @@ -72,6 +72,8 @@ When adding 1-GPU tests, choose the appropriate suite based on hardware compatib | Suite | Runner | GPU | When to Use | |-------|--------|-----|-------------| +| `stage-a-test-small-1-gpu` | `1-gpu-5090` | RTX 5090 (32GB, SM120) | Stage A per-commit smoke on 5090 (CUDA) | +| `stage-a-test-small-1-gpu-amd` | AMD CI runners | ROCm | Stage A per-commit smoke (AMD) | | `stage-b-test-small-1-gpu` | `1-gpu-5090` | RTX 5090 (32GB, SM120) | 5090-compatible tests (preferred) | | `stage-b-test-large-1-gpu` | `1-gpu-runner` | H100 (80GB, SM90) | Large models or 5090-incompatible tests | @@ -98,13 +100,13 @@ If a test cannot run on 5090 due to any of the above, use `stage-b-test-large-1- ### Available Suites **Per-Commit (CUDA)**: -- Stage A: `stage-a-test-1` (locked), `stage-a-test-2`, `stage-a-test-cpu` +- Stage A: `stage-a-test-small-1-gpu` (5090), `stage-a-test-2`, `stage-a-test-cpu` - Stage B: `stage-b-test-small-1-gpu` (5090), `stage-b-test-large-1-gpu` (H100), `stage-b-test-large-2-gpu` - Stage C (4-GPU): `stage-c-test-4-gpu-h100`, `stage-c-test-4-gpu-b200`, `stage-c-test-4-gpu-gb200`, `stage-c-test-deepep-4-gpu` - Stage C (8-GPU): `stage-c-test-8-gpu-h20`, `stage-c-test-8-gpu-h200`, `stage-c-test-8-gpu-b200`, `stage-c-test-deepep-8-gpu-h200` **Per-Commit (AMD)**: -- `stage-a-test-1`, `stage-b-test-small-1-gpu-amd`, `stage-b-test-large-2-gpu-amd` +- `stage-a-test-small-1-gpu-amd`, `stage-b-test-small-1-gpu-amd`, `stage-b-test-large-2-gpu-amd` **Nightly**: - `nightly-1-gpu`, `nightly-2-gpu`, `nightly-4-gpu`, `nightly-8-gpu`, etc. diff --git a/test/registered/attention/test_wave_attention_kernels.py b/test/registered/attention/test_wave_attention_kernels.py index 93ac52e3c..43051bd5a 100644 --- a/test/registered/attention/test_wave_attention_kernels.py +++ b/test/registered/attention/test_wave_attention_kernels.py @@ -25,7 +25,7 @@ from sglang.srt.utils import get_device from sglang.test.ci.ci_register import register_amd_ci # Wave attention kernel unit tests (AMD only - requires wave_lang) -register_amd_ci(est_time=60, suite="stage-a-test-1-amd") +register_amd_ci(est_time=60, suite="stage-a-test-small-1-gpu-amd") class TestWaveAttention(unittest.TestCase): diff --git a/test/registered/debug_utils/test_cuda_coredump_smoke.py b/test/registered/debug_utils/test_cuda_coredump_smoke.py index 09bde37a5..c90ecd38e 100644 --- a/test/registered/debug_utils/test_cuda_coredump_smoke.py +++ b/test/registered/debug_utils/test_cuda_coredump_smoke.py @@ -12,7 +12,7 @@ from sglang.test.ci.ci_register import register_cuda_ci register_cuda_ci( est_time=10, - suite="stage-a-test-1", + suite="stage-a-test-small-1-gpu", disabled="Manual only: triggers intentional CUDA crash for coredump verification", ) diff --git a/test/registered/quant/test_awq_dequant.py b/test/registered/quant/test_awq_dequant.py index 3f9ac10e9..589aad2c4 100644 --- a/test/registered/quant/test_awq_dequant.py +++ b/test/registered/quant/test_awq_dequant.py @@ -21,7 +21,7 @@ from sglang.srt.utils import get_device from sglang.test.ci.ci_register import register_amd_ci from sglang.test.test_utils import CustomTestCase -register_amd_ci(est_time=2, suite="stage-a-test-1-amd") +register_amd_ci(est_time=2, suite="stage-a-test-small-1-gpu-amd") device = get_device() diff --git a/test/registered/quant/test_fused_rms_fp8_group_quant.py b/test/registered/quant/test_fused_rms_fp8_group_quant.py index 5750908e8..c9aac1de0 100644 --- a/test/registered/quant/test_fused_rms_fp8_group_quant.py +++ b/test/registered/quant/test_fused_rms_fp8_group_quant.py @@ -7,7 +7,7 @@ import torch.nn.functional as F from sglang.test.ci.ci_register import register_amd_ci from sglang.test.test_utils import CustomTestCase -register_amd_ci(est_time=10, suite="stage-a-test-1-amd") +register_amd_ci(est_time=10, suite="stage-a-test-small-1-gpu-amd") def _fp8_available() -> bool: diff --git a/test/registered/test_srt_backend.py b/test/registered/test_srt_backend.py index 535794d63..e1e9d20d5 100644 --- a/test/registered/test_srt_backend.py +++ b/test/registered/test_srt_backend.py @@ -19,8 +19,8 @@ from sglang.test.test_programs import ( ) from sglang.test.test_utils import DEFAULT_MODEL_NAME_FOR_TEST, CustomTestCase -register_cuda_ci(est_time=80, suite="stage-a-test-1") -register_amd_ci(est_time=120, suite="stage-a-test-1-amd") +register_cuda_ci(est_time=80, suite="stage-a-test-small-1-gpu") +register_amd_ci(est_time=120, suite="stage-a-test-small-1-gpu-amd") class TestSRTBackend(CustomTestCase): @@ -29,7 +29,9 @@ class TestSRTBackend(CustomTestCase): @classmethod def setUpClass(cls): cls.backend = sgl.Runtime( - model_path=DEFAULT_MODEL_NAME_FOR_TEST, cuda_graph_max_bs=4 + model_path=DEFAULT_MODEL_NAME_FOR_TEST, + cuda_graph_max_bs=4, + mem_fraction_static=0.7, ) sgl.set_default_backend(cls.backend) diff --git a/test/run_suite.py b/test/run_suite.py index 5b5d1e1c6..89cd529ee 100644 --- a/test/run_suite.py +++ b/test/run_suite.py @@ -24,7 +24,7 @@ HW_MAPPING = { PER_COMMIT_SUITES = { HWBackend.CPU: ["stage-a-cpu-only"], HWBackend.AMD: [ - "stage-a-test-1-amd", + "stage-a-test-small-1-gpu-amd", "stage-b-test-small-1-gpu-amd", "stage-b-test-small-1-gpu-amd-nondeterministic", "stage-b-test-small-1-gpu-amd-mi35x", @@ -35,7 +35,7 @@ PER_COMMIT_SUITES = { "stage-c-test-large-8-gpu-amd-mi35x", ], HWBackend.CUDA: [ - "stage-a-test-1", + "stage-a-test-small-1-gpu", "stage-b-test-small-1-gpu", "stage-b-test-large-1-gpu", "stage-b-test-large-2-gpu", @@ -49,7 +49,7 @@ PER_COMMIT_SUITES = { "stage-c-test-deepep-8-gpu-h200", ], HWBackend.NPU: [ - "stage-a-test-1", + "stage-a-test-small-1-gpu", "stage-b-test-1-npu-a2", "stage-b-test-2-npu-a2", "stage-b-test-4-npu-a3",