From d7dd0b8832293437f8652a9a8fadb8a0ccbcc4d0 Mon Sep 17 00:00:00 2001 From: Alison Shao <54658187+alisonshao@users.noreply.github.com> Date: Thu, 22 Jan 2026 22:31:44 -0800 Subject: [PATCH] Re-enable unit-test-deepep-8-gpu and unit-test-backend-4-gpu-gb200 (#17438) --- .github/workflows/pr-test.yml | 202 +++++++++++++++++----------------- test/srt/run_suite.py | 9 +- 2 files changed, 103 insertions(+), 108 deletions(-) diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index 953e2e9d3..c6da837ef 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -251,7 +251,6 @@ jobs: needs: [check-changes, call-gate, wait-for-stage-a] # Only run for PRs (not scheduled) and when not targeting a specific stage # Skip if call-gate failed (stage-b jobs will be skipped, nothing to wait for) - # !cancelled() ensures this job respects workflow cancellation from concurrency group if: | always() && !cancelled() && @@ -1645,54 +1644,54 @@ jobs: fi python3 run_suite.py --suite per-commit-4-gpu-deepep $RETRY_FLAG $CONTINUE_ON_ERROR_FLAG - # Disabled: IBGDA/cudaHostRegister environment issues on 8-GPU runner, see #17175 - # unit-test-deepep-8-gpu: - # needs: [check-changes, call-gate, stage-b-test-small-1-gpu, stage-b-test-large-1-gpu, stage-b-test-large-2-gpu, stage-b-test-4-gpu-b200] - # if: | - # always() && - # ( - # (inputs.target_stage == 'unit-test-deepep-8-gpu') || - # ( - # !inputs.target_stage && - # (github.event_name == 'schedule' || (!failure() && !cancelled())) && - # ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) - # ) - # ) - # runs-on: 8-gpu-h200 - # env: - # RUNNER_LABELS: 8-gpu-h200 - # steps: - # - name: Checkout code - # uses: actions/checkout@v4 - # with: - # ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} - # - # - name: Download artifacts - # if: needs.check-changes.outputs.sgl_kernel == 'true' - # uses: actions/download-artifact@v4 - # with: - # path: sgl-kernel/dist/ - # merge-multiple: true - # pattern: wheel-python3.10-cuda12.9 - # - # - name: Install dependencies - # timeout-minutes: 10 - # run: | - # CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/ci_install_deepep.sh - # - # - name: Run test - # timeout-minutes: 20 - # run: | - # cd test/srt - # RETRY_FLAG="" - # if [[ "${{ needs.check-changes.outputs.enable_retry }}" == "true" ]]; then - # RETRY_FLAG="--enable-retry" - # fi - # CONTINUE_ON_ERROR_FLAG="" - # if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then - # CONTINUE_ON_ERROR_FLAG="--continue-on-error" - # fi - # python3 run_suite.py --suite per-commit-8-gpu-h200-deepep $RETRY_FLAG $CONTINUE_ON_ERROR_FLAG + unit-test-deepep-8-gpu: + needs: [check-changes, call-gate, wait-for-stage-b] + if: | + always() && + ( + (inputs.target_stage == 'unit-test-deepep-8-gpu') || + ( + !inputs.target_stage && + ((github.event_name == 'schedule' || inputs.test_parallel_dispatch == true) || (!failure() && !cancelled())) && + ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) + ) + ) + runs-on: 8-gpu-h200 + timeout-minutes: 60 + env: + RUNNER_LABELS: 8-gpu-h200 + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} + + - name: Download artifacts + if: needs.check-changes.outputs.sgl_kernel == 'true' + uses: actions/download-artifact@v4 + with: + path: sgl-kernel/dist/ + merge-multiple: true + pattern: wheel-python3.10-cuda12.9 + + - name: Install dependencies + timeout-minutes: 10 + run: | + CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/ci_install_deepep.sh + + - name: Run test + timeout-minutes: 45 + run: | + cd test/srt + RETRY_FLAG="" + if [[ "${{ needs.check-changes.outputs.enable_retry }}" == "true" ]]; then + RETRY_FLAG="--enable-retry" + fi + CONTINUE_ON_ERROR_FLAG="" + if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then + CONTINUE_ON_ERROR_FLAG="--continue-on-error" + fi + python3 run_suite.py --suite per-commit-8-gpu-h200-deepep $RETRY_FLAG $CONTINUE_ON_ERROR_FLAG unit-test-backend-4-gpu-b200: needs: [check-changes, call-gate, wait-for-stage-b] @@ -1748,57 +1747,56 @@ jobs: fi IS_BLACKWELL=1 python3 run_suite.py --suite per-commit-4-gpu-b200 --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --timeout-per-file 1800 $RETRY_FLAG $CONTINUE_ON_ERROR_FLAG - # Disabled: GB200 runner needs repair - # unit-test-backend-4-gpu-gb200: - # needs: [check-changes, call-gate, wait-for-stage-b, sgl-kernel-build-wheels-arm] - # if: | - # always() && - # ( - # (inputs.target_stage == 'unit-test-backend-4-gpu-gb200') || - # ( - # !inputs.target_stage && - # ((github.event_name == 'schedule' || inputs.test_parallel_dispatch == true) || (!failure() && !cancelled())) && - # ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) - # ) - # ) - # runs-on: 4-gpu-gb200 - # timeout-minutes: 60 - # env: - # RUNNER_LABELS: 4-gpu-gb200 - # strategy: - # fail-fast: false - # steps: - # - name: Checkout code - # uses: actions/checkout@v4 - # with: - # ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} - # - # - name: Download artifacts - # if: needs.check-changes.outputs.sgl_kernel == 'true' - # uses: actions/download-artifact@v4 - # with: - # path: sgl-kernel/dist/ - # merge-multiple: true - # pattern: wheel-python3.10-cuda12.9-aarch64 - # - # - name: Install dependencies - # timeout-minutes: 10 - # run: | - # CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} IS_BLACKWELL=1 GRACE_BLACKWELL=1 bash scripts/ci/ci_install_deepep.sh - # - # - name: Run test - # timeout-minutes: 45 - # run: | - # cd test/srt - # RETRY_FLAG="" - # if [[ "${{ needs.check-changes.outputs.enable_retry }}" == "true" ]]; then - # RETRY_FLAG="--enable-retry" - # fi - # CONTINUE_ON_ERROR_FLAG="" - # if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then - # CONTINUE_ON_ERROR_FLAG="--continue-on-error" - # fi - # python3 run_suite.py --suite per-commit-4-gpu-gb200 --auto-partition-id 0 --auto-partition-size 1 --timeout-per-file 3600 $RETRY_FLAG $CONTINUE_ON_ERROR_FLAG + unit-test-backend-4-gpu-gb200: + needs: [check-changes, call-gate, wait-for-stage-b, sgl-kernel-build-wheels-arm] + if: | + always() && + ( + (inputs.target_stage == 'unit-test-backend-4-gpu-gb200') || + ( + !inputs.target_stage && + ((github.event_name == 'schedule' || inputs.test_parallel_dispatch == true) || (!failure() && !cancelled())) && + ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) + ) + ) + runs-on: 4-gpu-gb200 + timeout-minutes: 60 + env: + RUNNER_LABELS: 4-gpu-gb200 + strategy: + fail-fast: false + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} + + - name: Download artifacts + if: needs.check-changes.outputs.sgl_kernel == 'true' + uses: actions/download-artifact@v4 + with: + path: sgl-kernel/dist/ + merge-multiple: true + pattern: wheel-python3.10-cuda12.9-aarch64 + + - name: Install dependencies + timeout-minutes: 10 + run: | + CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} IS_BLACKWELL=1 GRACE_BLACKWELL=1 bash scripts/ci/ci_install_deepep.sh + + - name: Run test + timeout-minutes: 45 + run: | + cd test/srt + RETRY_FLAG="" + if [[ "${{ needs.check-changes.outputs.enable_retry }}" == "true" ]]; then + RETRY_FLAG="--enable-retry" + fi + CONTINUE_ON_ERROR_FLAG="" + if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then + CONTINUE_ON_ERROR_FLAG="--continue-on-error" + fi + python3 run_suite.py --suite per-commit-4-gpu-gb200 --auto-partition-id 0 --auto-partition-size 1 --timeout-per-file 3600 $RETRY_FLAG $CONTINUE_ON_ERROR_FLAG pr-test-finish: needs: @@ -1836,9 +1834,9 @@ jobs: unit-test-backend-8-gpu-h20, unit-test-backend-8-gpu-h200, unit-test-deepep-4-gpu, - # unit-test-deepep-8-gpu, # Disabled, see #17175 + unit-test-deepep-8-gpu, unit-test-backend-4-gpu-b200, - # unit-test-backend-4-gpu-gb200, # Disabled: GB200 runner needs repair + unit-test-backend-4-gpu-gb200, ] if: always() runs-on: ubuntu-latest diff --git a/test/srt/run_suite.py b/test/srt/run_suite.py index 6a373319b..305266525 100644 --- a/test/srt/run_suite.py +++ b/test/srt/run_suite.py @@ -47,11 +47,9 @@ suites = { TestFile("ep/test_deepep_small.py", 531), TestFile("ep/test_mooncake_ep_small.py", 660), ], - # Disabled: IBGDA/cudaHostRegister environment issues on 8-GPU runner, see #17175 - # 4-GPU DeepEP tests provide sufficient coverage - # "per-commit-8-gpu-h200-deepep": [ - # TestFile("ep/test_deepep_large.py", 563), - # ], + "per-commit-8-gpu-h200-deepep": [ + TestFile("ep/test_deepep_large.py", 563), + ], # quantization_test suite migrated to test/registered/quant/ "__not_in_ci__": [ TestFile("test_release_memory_occupation.py", 200), # Temporarily disabled @@ -64,7 +62,6 @@ suites = { TestFile( "models/test_qwen3_next_models_pcg.py" ), # Disabled: intermittent failures, see #17039 - TestFile("ep/test_deepep_large.py", 563), # Disabled: see #17175 ], }