Add AMD stage support to /rerun-stage command and fix related bugs (#14463)

This commit is contained in:
Alison Shao
2025-12-04 21:13:52 -08:00
committed by GitHub
parent 41429a8c10
commit 4c5074eb78
3 changed files with 282 additions and 112 deletions

View File

@@ -343,11 +343,14 @@ jobs:
stage-a-test-1:
needs: [check-changes, call-gate, sgl-kernel-build-wheels]
if: |
(inputs.target_stage == 'stage-a-test-1') ||
always() &&
(
always() &&
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
(inputs.target_stage == 'stage-a-test-1') ||
(
!inputs.target_stage &&
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
)
)
runs-on: 1-gpu-runner
env:
@@ -380,11 +383,14 @@ jobs:
multimodal-gen-test-1-gpu:
needs: [check-changes, call-gate, sgl-kernel-build-wheels]
if: |
(inputs.target_stage == 'multimodal-gen-test-1-gpu') ||
always() &&
(
always() &&
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
needs.check-changes.outputs.multimodal_gen == 'true'
(inputs.target_stage == 'multimodal-gen-test-1-gpu') ||
(
!inputs.target_stage &&
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
needs.check-changes.outputs.multimodal_gen == 'true'
)
)
runs-on: 1-gpu-runner
strategy:
@@ -424,11 +430,14 @@ jobs:
multimodal-gen-test-2-gpu:
needs: [check-changes, call-gate, sgl-kernel-build-wheels]
if: |
(inputs.target_stage == 'multimodal-gen-test-2-gpu') ||
always() &&
(
always() &&
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
needs.check-changes.outputs.multimodal_gen == 'true'
(inputs.target_stage == 'multimodal-gen-test-2-gpu') ||
(
!inputs.target_stage &&
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
needs.check-changes.outputs.multimodal_gen == 'true'
)
)
runs-on: 2-gpu-runner
strategy:
@@ -468,11 +477,14 @@ jobs:
quantization-test:
needs: [check-changes, call-gate, stage-a-test-1]
if: |
(inputs.target_stage == 'quantization-test') ||
always() &&
(
always() &&
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
(inputs.target_stage == 'quantization-test') ||
(
!inputs.target_stage &&
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
)
)
runs-on: 1-gpu-runner
steps:
@@ -500,11 +512,14 @@ jobs:
unit-test-backend-1-gpu:
needs: [check-changes, call-gate, stage-a-test-1]
if: |
(inputs.target_stage == 'unit-test-backend-1-gpu') ||
always() &&
(
always() &&
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
(inputs.target_stage == 'unit-test-backend-1-gpu') ||
(
!inputs.target_stage &&
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
)
)
runs-on: 1-gpu-runner
env:
@@ -539,11 +554,14 @@ jobs:
unit-test-backend-2-gpu:
needs: [check-changes, call-gate, unit-test-backend-1-gpu]
if: |
(inputs.target_stage == 'unit-test-backend-2-gpu') ||
always() &&
(
always() &&
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
(inputs.target_stage == 'unit-test-backend-2-gpu') ||
(
!inputs.target_stage &&
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
)
)
runs-on: 2-gpu-runner
env:
@@ -577,11 +595,14 @@ jobs:
unit-test-backend-4-gpu:
needs: [check-changes, call-gate, unit-test-backend-2-gpu]
if: |
(inputs.target_stage == 'unit-test-backend-4-gpu') ||
always() &&
(
always() &&
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
(inputs.target_stage == 'unit-test-backend-4-gpu') ||
(
!inputs.target_stage &&
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
)
)
runs-on: 4-gpu-h100
env:
@@ -615,11 +636,14 @@ jobs:
unit-test-backend-8-gpu-h200:
needs: [check-changes, call-gate, unit-test-backend-2-gpu]
if: |
(inputs.target_stage == 'unit-test-backend-8-gpu-h200') ||
always() &&
(
always() &&
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
(inputs.target_stage == 'unit-test-backend-8-gpu-h200') ||
(
!inputs.target_stage &&
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
)
)
runs-on: 8-gpu-h200
env:
@@ -653,11 +677,14 @@ jobs:
unit-test-backend-8-gpu-h20:
needs: [check-changes, call-gate, unit-test-backend-2-gpu]
if: |
(inputs.target_stage == 'unit-test-backend-8-gpu-h20') ||
always() &&
(
always() &&
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
(inputs.target_stage == 'unit-test-backend-8-gpu-h20') ||
(
!inputs.target_stage &&
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
)
)
runs-on: 8-gpu-h20
env:
@@ -692,11 +719,14 @@ jobs:
performance-test-1-gpu-part-1:
needs: [check-changes, call-gate, stage-a-test-1]
if: |
(inputs.target_stage == 'performance-test-1-gpu-part-1') ||
always() &&
(
always() &&
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
(inputs.target_stage == 'performance-test-1-gpu-part-1') ||
(
!inputs.target_stage &&
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
)
)
runs-on: 1-gpu-runner
env:
@@ -758,11 +788,14 @@ jobs:
performance-test-1-gpu-part-2:
needs: [check-changes, call-gate, stage-a-test-1]
if: |
(inputs.target_stage == 'performance-test-1-gpu-part-2') ||
always() &&
(
always() &&
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
(inputs.target_stage == 'performance-test-1-gpu-part-2') ||
(
!inputs.target_stage &&
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
)
)
runs-on: 1-gpu-runner
env:
@@ -816,11 +849,14 @@ jobs:
performance-test-1-gpu-part-3:
needs: [check-changes, call-gate, stage-a-test-1]
if: |
(inputs.target_stage == 'performance-test-1-gpu-part-3') ||
always() &&
(
always() &&
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
(inputs.target_stage == 'performance-test-1-gpu-part-3') ||
(
!inputs.target_stage &&
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
)
)
runs-on: 1-gpu-runner
env:
@@ -868,11 +904,14 @@ jobs:
performance-test-2-gpu:
needs: [check-changes, call-gate, unit-test-backend-2-gpu]
if: |
(inputs.target_stage == 'performance-test-2-gpu') ||
always() &&
(
always() &&
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
(inputs.target_stage == 'performance-test-2-gpu') ||
(
!inputs.target_stage &&
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
)
)
runs-on: 2-gpu-runner
env:
@@ -932,11 +971,14 @@ jobs:
accuracy-test-1-gpu:
needs: [check-changes, call-gate, stage-a-test-1]
if: |
(inputs.target_stage == 'accuracy-test-1-gpu') ||
always() &&
(
always() &&
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
(inputs.target_stage == 'accuracy-test-1-gpu') ||
(
!inputs.target_stage &&
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
)
)
runs-on: 1-gpu-runner
env:
@@ -969,11 +1011,14 @@ jobs:
accuracy-test-2-gpu:
needs: [check-changes, call-gate, accuracy-test-1-gpu]
if: |
(inputs.target_stage == 'accuracy-test-2-gpu') ||
always() &&
(
always() &&
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
(inputs.target_stage == 'accuracy-test-2-gpu') ||
(
!inputs.target_stage &&
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
)
)
runs-on: 2-gpu-runner
env:
@@ -1006,11 +1051,14 @@ jobs:
unit-test-deepep-4-gpu:
needs: [check-changes, call-gate, unit-test-backend-2-gpu]
if: |
(inputs.target_stage == 'unit-test-deepep-4-gpu') ||
always() &&
(
always() &&
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
(inputs.target_stage == 'unit-test-deepep-4-gpu') ||
(
!inputs.target_stage &&
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
)
)
runs-on: 4-gpu-h100
env:
@@ -1040,11 +1088,14 @@ jobs:
unit-test-deepep-8-gpu:
needs: [check-changes, call-gate, unit-test-backend-2-gpu]
if: |
(inputs.target_stage == 'unit-test-deepep-8-gpu') ||
always() &&
(
always() &&
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
(inputs.target_stage == 'unit-test-deepep-8-gpu') ||
(
!inputs.target_stage &&
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
)
)
runs-on: 8-gpu-h200
env:
@@ -1074,11 +1125,14 @@ jobs:
unit-test-backend-4-gpu-b200:
needs: [check-changes, call-gate, unit-test-backend-2-gpu]
if: |
(inputs.target_stage == 'unit-test-backend-4-gpu-b200') ||
always() &&
(
always() &&
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
(inputs.target_stage == 'unit-test-backend-4-gpu-b200') ||
(
!inputs.target_stage &&
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
)
)
runs-on: 4-gpu-b200
env:
@@ -1113,11 +1167,14 @@ jobs:
unit-test-backend-4-gpu-gb200:
needs: [check-changes, call-gate, unit-test-backend-2-gpu, sgl-kernel-build-wheels-arm]
if: |
(inputs.target_stage == 'unit-test-backend-4-gpu-gb200') ||
always() &&
(
always() &&
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
(inputs.target_stage == 'unit-test-backend-4-gpu-gb200') ||
(
!inputs.target_stage &&
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
)
)
runs-on: 4-gpu-gb200
env: