diff --git a/.github/actions/check-maintenance/action.yml b/.github/actions/check-maintenance/action.yml new file mode 100644 index 000000000..b59938ef2 --- /dev/null +++ b/.github/actions/check-maintenance/action.yml @@ -0,0 +1,52 @@ +name: Check Maintenance Mode +description: Blocks CI when maintenance mode is active (issue #21065 is open), unless the PR has the bypass-maintenance label. + +inputs: + github-token: + description: GitHub token for API access + required: true + +runs: + using: composite + steps: + - name: Check maintenance mode + shell: bash + env: + GH_TOKEN: ${{ inputs.github-token }} + run: | + MAINTENANCE_ISSUE=21065 + REPO="${{ github.repository }}" + PR_NUMBER="${{ github.event.pull_request.number }}" + + # Check if maintenance issue is open (fail-open: if API errors, allow CI to proceed) + ISSUE_STATE=$(gh issue view "$MAINTENANCE_ISSUE" --repo "$REPO" --json state --jq '.state' 2>/dev/null || echo "UNKNOWN") + + if [[ "$ISSUE_STATE" != "OPEN" ]]; then + echo "✅ Maintenance mode is OFF. Proceeding with CI." + exit 0 + fi + + # For PRs, check if bypass-maintenance label is present + if [[ -n "$PR_NUMBER" ]]; then + HAS_BYPASS=$(gh pr view "$PR_NUMBER" --repo "$REPO" --json labels --jq '[.labels[].name] | map(select(. == "bypass-maintenance")) | length' 2>/dev/null || echo "0") + if [[ "$HAS_BYPASS" -gt 0 ]]; then + echo "✅ PR #$PR_NUMBER has 'bypass-maintenance' label. Bypassing maintenance mode." + exit 0 + fi + fi + + MSG=$(printf "%s\n" \ + "## ⚠️ CI Maintenance Mode is Active" \ + "The CI infrastructure is currently under maintenance." \ + "All PR CI runs are paused until maintenance is complete." \ + "" \ + "What should you do?" \ + "- Check back later (~12 hours)" \ + "- Follow CI Maintenance Mode issue: https://github.com/$REPO/issues/$MAINTENANCE_ISSUE for status updates") + + echo "$MSG" >> "$GITHUB_STEP_SUMMARY" + while IFS= read -r line; do + echo "::error::$line" + done <<< "$MSG" + + exit 1 diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index 4af20e1ab..458ffc783 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -61,6 +61,7 @@ env: permissions: actions: write contents: read + issues: read pull-requests: read jobs: @@ -88,6 +89,10 @@ jobs: with: ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} + - uses: ./.github/actions/check-maintenance + with: + github-token: ${{ github.token }} + - name: Show test partition assignments continue-on-error: true run: python3 test/show_partitions.py @@ -329,6 +334,11 @@ jobs: stage_a_result: ${{ steps.wait.outputs.result }} steps: - uses: actions/checkout@v4 + + - uses: ./.github/actions/check-maintenance + with: + github-token: ${{ github.token }} + - uses: ./.github/actions/wait-for-jobs id: wait with: @@ -354,6 +364,11 @@ jobs: stage_b_result: ${{ steps.wait.outputs.result }} steps: - uses: actions/checkout@v4 + + - uses: ./.github/actions/check-maintenance + with: + github-token: ${{ github.token }} + - uses: ./.github/actions/wait-for-jobs id: wait with: @@ -411,6 +426,10 @@ jobs: submodules: "recursive" ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} + - uses: ./.github/actions/check-maintenance + with: + github-token: ${{ github.token }} + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: @@ -461,6 +480,10 @@ jobs: submodules: "recursive" ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} + - uses: ./.github/actions/check-maintenance + with: + github-token: ${{ github.token }} + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: @@ -500,6 +523,10 @@ jobs: with: ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} + - uses: ./.github/actions/check-maintenance + with: + github-token: ${{ github.token }} + - name: Cleanup run: | ls -alh sgl-kernel/dist || true @@ -538,6 +565,10 @@ jobs: with: ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} + - uses: ./.github/actions/check-maintenance + with: + github-token: ${{ github.token }} + - name: Cleanup run: | ls -alh sgl-kernel/dist || true @@ -571,13 +602,15 @@ jobs: needs.check-changes.outputs.sgl_kernel == 'true' runs-on: 1-gpu-h100 timeout-minutes: 240 - env: - CI: true steps: - uses: actions/checkout@v4 with: ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} + - uses: ./.github/actions/check-maintenance + with: + github-token: ${{ github.token }} + - name: Cleanup run: | ls -alh sgl-kernel/dist || true @@ -628,6 +661,10 @@ jobs: with: ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} + - uses: ./.github/actions/check-maintenance + with: + github-token: ${{ github.token }} + - name: Cleanup run: | ls -alh sgl-kernel/dist || true @@ -699,6 +736,10 @@ jobs: with: ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} + - uses: ./.github/actions/check-maintenance + with: + github-token: ${{ github.token }} + - name: Install dependencies timeout-minutes: 20 run: | @@ -724,6 +765,10 @@ jobs: with: ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} + - uses: ./.github/actions/check-maintenance + with: + github-token: ${{ github.token }} + - name: Install dependencies timeout-minutes: 20 run: | @@ -745,13 +790,15 @@ jobs: needs.check-changes.outputs.jit_kernel == 'true' runs-on: 1-gpu-h100 timeout-minutes: 240 - env: - CI: true steps: - uses: actions/checkout@v4 with: ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} + - uses: ./.github/actions/check-maintenance + with: + github-token: ${{ github.token }} + - name: Install dependencies timeout-minutes: 20 run: | @@ -806,6 +853,10 @@ jobs: with: ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} + - uses: ./.github/actions/check-maintenance + with: + github-token: ${{ github.token }} + - name: Download artifacts if: needs.check-changes.outputs.sgl_kernel == 'true' uses: actions/download-artifact@v4 @@ -855,6 +906,10 @@ jobs: with: ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} + - uses: ./.github/actions/check-maintenance + with: + github-token: ${{ github.token }} + - name: Set up Python uses: actions/setup-python@v5 with: @@ -901,6 +956,10 @@ jobs: with: ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} + - uses: ./.github/actions/check-maintenance + with: + github-token: ${{ github.token }} + - name: Download artifacts if: needs.check-changes.outputs.sgl_kernel == 'true' uses: actions/download-artifact@v4 @@ -958,6 +1017,10 @@ jobs: with: ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} + - uses: ./.github/actions/check-maintenance + with: + github-token: ${{ github.token }} + - name: Download artifacts if: needs.check-changes.outputs.sgl_kernel == 'true' uses: actions/download-artifact@v4 @@ -1008,6 +1071,10 @@ jobs: with: ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} + - uses: ./.github/actions/check-maintenance + with: + github-token: ${{ github.token }} + - name: Download artifacts if: needs.check-changes.outputs.sgl_kernel == 'true' uses: actions/download-artifact@v4 @@ -1060,6 +1127,10 @@ jobs: with: ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} + - uses: ./.github/actions/check-maintenance + with: + github-token: ${{ github.token }} + - name: Download artifacts if: needs.check-changes.outputs.sgl_kernel == 'true' uses: actions/download-artifact@v6 @@ -1113,6 +1184,10 @@ jobs: with: ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} + - uses: ./.github/actions/check-maintenance + with: + github-token: ${{ github.token }} + - name: Download artifacts if: needs.check-changes.outputs.sgl_kernel == 'true' uses: actions/download-artifact@v4 @@ -1167,6 +1242,10 @@ jobs: with: ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} + - uses: ./.github/actions/check-maintenance + with: + github-token: ${{ github.token }} + - name: Download artifacts if: needs.check-changes.outputs.sgl_kernel == 'true' uses: actions/download-artifact@v4 @@ -1218,6 +1297,10 @@ jobs: with: ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} + - uses: ./.github/actions/check-maintenance + with: + github-token: ${{ github.token }} + - name: Download artifacts if: needs.check-changes.outputs.sgl_kernel == 'true' uses: actions/download-artifact@v4 @@ -1261,6 +1344,10 @@ jobs: with: ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} + - uses: ./.github/actions/check-maintenance + with: + github-token: ${{ github.token }} + - name: Download artifacts if: needs.check-changes.outputs.sgl_kernel == 'true' uses: actions/download-artifact@v4 @@ -1311,6 +1398,10 @@ jobs: with: ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} + - uses: ./.github/actions/check-maintenance + with: + github-token: ${{ github.token }} + - name: Download artifacts if: needs.check-changes.outputs.sgl_kernel == 'true' uses: actions/download-artifact@v4 @@ -1377,6 +1468,10 @@ jobs: with: ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} + - uses: ./.github/actions/check-maintenance + with: + github-token: ${{ github.token }} + - name: Download artifacts if: needs.check-changes.outputs.sgl_kernel == 'true' uses: actions/download-artifact@v4 @@ -1423,6 +1518,10 @@ jobs: with: ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} + - uses: ./.github/actions/check-maintenance + with: + github-token: ${{ github.token }} + - name: Download artifacts if: needs.check-changes.outputs.sgl_kernel == 'true' uses: actions/download-artifact@v4 @@ -1479,6 +1578,10 @@ jobs: with: ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} + - uses: ./.github/actions/check-maintenance + with: + github-token: ${{ github.token }} + - name: Download artifacts if: needs.check-changes.outputs.sgl_kernel == 'true' uses: actions/download-artifact@v4 @@ -1541,6 +1644,10 @@ jobs: with: ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} + - uses: ./.github/actions/check-maintenance + with: + github-token: ${{ github.token }} + - name: Download artifacts if: needs.check-changes.outputs.sgl_kernel == 'true' uses: actions/download-artifact@v6 @@ -1586,6 +1693,10 @@ jobs: # strategy: # fail-fast: false # steps: + # - uses: ./.github/actions/check-maintenance + # with: + # github-token: ${{ github.token }} + # # - name: Checkout code # uses: actions/checkout@v4 # with: