From 9479eca75dc971ae87b76b1b97703e6457f40d38 Mon Sep 17 00:00:00 2001 From: Alison Shao <54658187+alisonshao@users.noreply.github.com> Date: Tue, 13 Jan 2026 20:16:32 -0800 Subject: [PATCH] [CI] Fix max_parallel for scheduled runs (#17046) --- .github/workflows/pr-test.yml | 106 ++++++++++++++++++---------------- 1 file changed, 55 insertions(+), 51 deletions(-) diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index 793a3b3d9..76c0b96d5 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -110,12 +110,16 @@ jobs: - "python/*.toml" - ".github/workflows/pr-test.yml" - - name: Set max-parallel based on high-priority label + - name: Set max-parallel based on run type id: set-parallel run: | - if [[ "${{ github.event_name }}" == "pull_request" && "${{ contains(github.event.pull_request.labels.*.name, 'high priority') }}" == "true" ]]; then - echo "max_parallel=15" >> $GITHUB_OUTPUT - echo "High priority PR detected, setting max_parallel to 15" + # Scheduled runs and high-priority PRs get full parallelism + if [[ "${{ github.event_name }}" == "schedule" ]]; then + echo "max_parallel=14" >> $GITHUB_OUTPUT + echo "Scheduled run detected, setting max_parallel to 14" + elif [[ "${{ github.event_name }}" == "pull_request" && "${{ contains(github.event.pull_request.labels.*.name, 'high priority') }}" == "true" ]]; then + echo "max_parallel=14" >> $GITHUB_OUTPUT + echo "High priority PR detected, setting max_parallel to 14" else echo "max_parallel=3" >> $GITHUB_OUTPUT echo "Using default max_parallel of 3" @@ -772,6 +776,53 @@ jobs: fi python3 run_suite.py --hw cuda --suite stage-b-test-large-2-gpu --auto-partition-id ${{ matrix.partition }} --auto-partition-size 2 $CONTINUE_ON_ERROR_FLAG + stage-b-test-4-gpu-b200: + needs: [check-changes, call-gate, stage-a-test-1, sgl-kernel-build-wheels] + if: | + always() && + ( + (inputs.target_stage == 'stage-b-test-4-gpu-b200') || + ( + !inputs.target_stage && + (github.event_name == 'schedule' || (!failure() && !cancelled())) && + ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) + ) + ) + runs-on: ${{ needs.check-changes.outputs.b200_runner }} + env: + RUNNER_LABELS: ${{ needs.check-changes.outputs.b200_runner }} + strategy: + fail-fast: false + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} + + - name: Download artifacts + if: needs.check-changes.outputs.sgl_kernel == 'true' + uses: actions/download-artifact@v6 + with: + path: sgl-kernel/dist/ + merge-multiple: true + pattern: wheel-python3.10-cuda12.9 + + - name: Install dependencies + timeout-minutes: 10 + run: | + CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} IS_BLACKWELL=1 bash scripts/ci/ci_install_dependency.sh + + - name: Run test + timeout-minutes: 30 + run: | + cd test + CONTINUE_ON_ERROR_FLAG="" + if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then + CONTINUE_ON_ERROR_FLAG="--continue-on-error" + fi + IS_BLACKWELL=1 python3 run_suite.py --hw cuda --suite stage-b-test-4-gpu-b200 $CONTINUE_ON_ERROR_FLAG + stage-c-test-large-4-gpu: needs: [check-changes, call-gate, stage-b-test-small-1-gpu, stage-b-test-large-1-gpu, stage-b-test-large-2-gpu, sgl-kernel-build-wheels] if: | @@ -1002,53 +1053,6 @@ jobs: fi python3 run_suite.py --suite quantization_test $RETRY_FLAG $CONTINUE_ON_ERROR_FLAG - stage-b-test-4-gpu-b200: - needs: [check-changes, call-gate, stage-a-test-1, sgl-kernel-build-wheels] - if: | - always() && - ( - (inputs.target_stage == 'stage-b-test-4-gpu-b200') || - ( - !inputs.target_stage && - (github.event_name == 'schedule' || (!failure() && !cancelled())) && - ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) - ) - ) - runs-on: ${{ needs.check-changes.outputs.b200_runner }} - env: - RUNNER_LABELS: ${{ needs.check-changes.outputs.b200_runner }} - strategy: - fail-fast: false - - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} - - - name: Download artifacts - if: needs.check-changes.outputs.sgl_kernel == 'true' - uses: actions/download-artifact@v6 - with: - path: sgl-kernel/dist/ - merge-multiple: true - pattern: wheel-python3.10-cuda12.9 - - - name: Install dependencies - timeout-minutes: 10 - run: | - CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} IS_BLACKWELL=1 bash scripts/ci/ci_install_dependency.sh - - - name: Run test - timeout-minutes: 30 - run: | - cd test - CONTINUE_ON_ERROR_FLAG="" - if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then - CONTINUE_ON_ERROR_FLAG="--continue-on-error" - fi - IS_BLACKWELL=1 python3 run_suite.py --hw cuda --suite stage-b-test-4-gpu-b200 $CONTINUE_ON_ERROR_FLAG - unit-test-backend-4-gpu: needs: [check-changes, call-gate, stage-b-test-small-1-gpu, stage-b-test-large-1-gpu, stage-b-test-large-2-gpu, stage-b-test-4-gpu-b200] if: |