diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index 76c0b96d5..f4e195a74 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -1614,53 +1614,54 @@ jobs: fi python3 run_suite.py --suite per-commit-4-gpu-deepep $RETRY_FLAG $CONTINUE_ON_ERROR_FLAG - unit-test-deepep-8-gpu: - needs: [check-changes, call-gate, stage-b-test-small-1-gpu, stage-b-test-large-1-gpu, stage-b-test-large-2-gpu, stage-b-test-4-gpu-b200] - if: | - always() && - ( - (inputs.target_stage == 'unit-test-deepep-8-gpu') || - ( - !inputs.target_stage && - (github.event_name == 'schedule' || (!failure() && !cancelled())) && - ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) - ) - ) - runs-on: 8-gpu-h200 - env: - RUNNER_LABELS: 8-gpu-h200 - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} - - - name: Download artifacts - if: needs.check-changes.outputs.sgl_kernel == 'true' - uses: actions/download-artifact@v4 - with: - path: sgl-kernel/dist/ - merge-multiple: true - pattern: wheel-python3.10-cuda12.9 - - - name: Install dependencies - timeout-minutes: 10 - run: | - CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/ci_install_deepep.sh - - - name: Run test - timeout-minutes: 20 - run: | - cd test/srt - RETRY_FLAG="" - if [[ "${{ needs.check-changes.outputs.enable_retry }}" == "true" ]]; then - RETRY_FLAG="--enable-retry" - fi - CONTINUE_ON_ERROR_FLAG="" - if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then - CONTINUE_ON_ERROR_FLAG="--continue-on-error" - fi - python3 run_suite.py --suite per-commit-8-gpu-h200-deepep $RETRY_FLAG $CONTINUE_ON_ERROR_FLAG + # Disabled: IBGDA/cudaHostRegister environment issues on 8-GPU runner, see #17175 + # unit-test-deepep-8-gpu: + # needs: [check-changes, call-gate, stage-b-test-small-1-gpu, stage-b-test-large-1-gpu, stage-b-test-large-2-gpu, stage-b-test-4-gpu-b200] + # if: | + # always() && + # ( + # (inputs.target_stage == 'unit-test-deepep-8-gpu') || + # ( + # !inputs.target_stage && + # (github.event_name == 'schedule' || (!failure() && !cancelled())) && + # ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) + # ) + # ) + # runs-on: 8-gpu-h200 + # env: + # RUNNER_LABELS: 8-gpu-h200 + # steps: + # - name: Checkout code + # uses: actions/checkout@v4 + # with: + # ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} + # + # - name: Download artifacts + # if: needs.check-changes.outputs.sgl_kernel == 'true' + # uses: actions/download-artifact@v4 + # with: + # path: sgl-kernel/dist/ + # merge-multiple: true + # pattern: wheel-python3.10-cuda12.9 + # + # - name: Install dependencies + # timeout-minutes: 10 + # run: | + # CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/ci_install_deepep.sh + # + # - name: Run test + # timeout-minutes: 20 + # run: | + # cd test/srt + # RETRY_FLAG="" + # if [[ "${{ needs.check-changes.outputs.enable_retry }}" == "true" ]]; then + # RETRY_FLAG="--enable-retry" + # fi + # CONTINUE_ON_ERROR_FLAG="" + # if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then + # CONTINUE_ON_ERROR_FLAG="--continue-on-error" + # fi + # python3 run_suite.py --suite per-commit-8-gpu-h200-deepep $RETRY_FLAG $CONTINUE_ON_ERROR_FLAG unit-test-backend-4-gpu-b200: needs: [check-changes, call-gate, stage-b-test-small-1-gpu, stage-b-test-large-1-gpu, stage-b-test-large-2-gpu, stage-b-test-4-gpu-b200] @@ -1800,7 +1801,7 @@ jobs: accuracy-test-1-gpu, accuracy-test-2-gpu, unit-test-deepep-4-gpu, - unit-test-deepep-8-gpu, + # unit-test-deepep-8-gpu, # Disabled, see #17175 unit-test-backend-4-gpu-b200, unit-test-backend-4-gpu-gb200, ] diff --git a/test/srt/run_suite.py b/test/srt/run_suite.py index b00d0a297..fbc7c8154 100644 --- a/test/srt/run_suite.py +++ b/test/srt/run_suite.py @@ -48,9 +48,11 @@ suites = { TestFile("ep/test_deepep_small.py", 531), TestFile("ep/test_mooncake_ep_small.py", 660), ], - "per-commit-8-gpu-h200-deepep": [ - TestFile("ep/test_deepep_large.py", 563), - ], + # Disabled: IBGDA/cudaHostRegister environment issues on 8-GPU runner, see #17175 + # 4-GPU DeepEP tests provide sufficient coverage + # "per-commit-8-gpu-h200-deepep": [ + # TestFile("ep/test_deepep_large.py", 563), + # ], "quantization_test": [ TestFile("quant/test_awq.py", 163), TestFile("quant/test_marlin_moe.py", 200), @@ -74,6 +76,7 @@ suites = { TestFile( "models/test_qwen3_next_models_pcg.py" ), # Disabled: intermittent failures, see #17039 + TestFile("ep/test_deepep_large.py", 563), # Disabled: see #17175 ], }