[CI] Enable retry logic for flaky CI tests (#14983)
This commit is contained in:
Alison Shao
76
.github/workflows/pr-test.yml
vendored
76
.github/workflows/pr-test.yml
vendored
@@ -38,6 +38,7 @@ jobs:
|
||||
multimodal_gen: ${{ steps.filter.outputs.multimodal_gen || steps.scheduled.outputs.multimodal_gen }}
|
||||
max_parallel: ${{ steps.set-parallel.outputs.max_parallel }}
|
||||
b200_runner: ${{ steps.set-runner.outputs.b200_runner }}
|
||||
enable_retry: ${{ steps.set-retry.outputs.enable_retry }}
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
@@ -91,6 +92,12 @@ jobs:
|
||||
echo "b200_runner=4-gpu-b200" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
- name: Enable retry for CI
|
||||
id: set-retry
|
||||
run: |
|
||||
echo "enable_retry=true" >> $GITHUB_OUTPUT
|
||||
echo "Retry logic enabled for CI"
|
||||
|
||||
- name: Show filter results in summary (table)
|
||||
run: |
|
||||
{
|
||||
@@ -103,6 +110,7 @@ jobs:
|
||||
echo "| multimodal_gen | ${{ steps.filter.outputs.multimodal_gen || steps.scheduled.outputs.multimodal_gen }} |"
|
||||
echo "| max_parallel | ${{ steps.set-parallel.outputs.max_parallel }} |"
|
||||
echo "| b200_runner | ${{ steps.set-runner.outputs.b200_runner }} |"
|
||||
echo "| enable_retry | ${{ steps.set-retry.outputs.enable_retry }} |"
|
||||
} >> $GITHUB_STEP_SUMMARY
|
||||
|
||||
# =============================================== PR Gate ====================================================
|
||||
@@ -676,7 +684,11 @@ jobs:
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
cd test/srt
|
||||
python3 run_suite.py --suite quantization_test
|
||||
RETRY_FLAG=""
|
||||
if [[ "${{ needs.check-changes.outputs.enable_retry }}" == "true" ]]; then
|
||||
RETRY_FLAG="--enable-retry"
|
||||
fi
|
||||
python3 run_suite.py --suite quantization_test $RETRY_FLAG
|
||||
|
||||
unit-test-backend-1-gpu:
|
||||
needs: [check-changes, call-gate, stage-a-test-1]
|
||||
@@ -718,7 +730,11 @@ jobs:
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
cd test/srt
|
||||
python3 run_suite.py --suite per-commit-1-gpu --auto-partition-id ${{ matrix.part }} --auto-partition-size 15
|
||||
RETRY_FLAG=""
|
||||
if [[ "${{ needs.check-changes.outputs.enable_retry }}" == "true" ]]; then
|
||||
RETRY_FLAG="--enable-retry"
|
||||
fi
|
||||
python3 run_suite.py --suite per-commit-1-gpu --auto-partition-id ${{ matrix.part }} --auto-partition-size 15 $RETRY_FLAG
|
||||
|
||||
|
||||
stage-b-test-4-gpu-b200:
|
||||
@@ -801,7 +817,11 @@ jobs:
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
cd test/srt
|
||||
python3 run_suite.py --suite per-commit-2-gpu --auto-partition-id ${{ matrix.part }} --auto-partition-size 2
|
||||
RETRY_FLAG=""
|
||||
if [[ "${{ needs.check-changes.outputs.enable_retry }}" == "true" ]]; then
|
||||
RETRY_FLAG="--enable-retry"
|
||||
fi
|
||||
python3 run_suite.py --suite per-commit-2-gpu --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 $RETRY_FLAG
|
||||
|
||||
unit-test-backend-4-gpu:
|
||||
needs: [check-changes, call-gate, unit-test-backend-2-gpu, stage-b-test-4-gpu-b200]
|
||||
@@ -842,7 +862,11 @@ jobs:
|
||||
timeout-minutes: 20
|
||||
run: |
|
||||
cd test/srt
|
||||
python3 run_suite.py --suite per-commit-4-gpu --auto-partition-id ${{ matrix.part }} --auto-partition-size 3
|
||||
RETRY_FLAG=""
|
||||
if [[ "${{ needs.check-changes.outputs.enable_retry }}" == "true" ]]; then
|
||||
RETRY_FLAG="--enable-retry"
|
||||
fi
|
||||
python3 run_suite.py --suite per-commit-4-gpu --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 $RETRY_FLAG
|
||||
|
||||
unit-test-backend-8-gpu-h200:
|
||||
needs: [check-changes, call-gate, unit-test-backend-2-gpu, stage-b-test-4-gpu-b200]
|
||||
@@ -889,7 +913,11 @@ jobs:
|
||||
timeout-minutes: 20
|
||||
run: |
|
||||
cd test/srt
|
||||
python3 run_suite.py --suite per-commit-8-gpu-h200 --auto-partition-id ${{ matrix.part }} --auto-partition-size 3
|
||||
RETRY_FLAG=""
|
||||
if [[ "${{ needs.check-changes.outputs.enable_retry }}" == "true" ]]; then
|
||||
RETRY_FLAG="--enable-retry"
|
||||
fi
|
||||
python3 run_suite.py --suite per-commit-8-gpu-h200 --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 $RETRY_FLAG
|
||||
|
||||
unit-test-backend-8-gpu-h20:
|
||||
needs: [check-changes, call-gate, unit-test-backend-2-gpu, stage-b-test-4-gpu-b200]
|
||||
@@ -931,7 +959,11 @@ jobs:
|
||||
timeout-minutes: 20
|
||||
run: |
|
||||
cd test/srt
|
||||
python3 run_suite.py --suite per-commit-8-gpu-h20 --auto-partition-id ${{ matrix.part }} --auto-partition-size 2
|
||||
RETRY_FLAG=""
|
||||
if [[ "${{ needs.check-changes.outputs.enable_retry }}" == "true" ]]; then
|
||||
RETRY_FLAG="--enable-retry"
|
||||
fi
|
||||
python3 run_suite.py --suite per-commit-8-gpu-h20 --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 $RETRY_FLAG
|
||||
|
||||
performance-test-1-gpu-part-1:
|
||||
needs: [check-changes, call-gate, stage-a-test-1]
|
||||
@@ -1220,10 +1252,10 @@ jobs:
|
||||
pip install -e .
|
||||
|
||||
- name: Evaluate accuracy
|
||||
timeout-minutes: 20
|
||||
timeout-minutes: 25
|
||||
run: |
|
||||
cd test/srt
|
||||
python3 test_eval_accuracy_large.py
|
||||
python3 -m sglang.test.ci.run_with_retry test_eval_accuracy_large.py
|
||||
|
||||
accuracy-test-2-gpu:
|
||||
needs: [check-changes, call-gate, accuracy-test-1-gpu]
|
||||
@@ -1260,10 +1292,10 @@ jobs:
|
||||
pip install -e .
|
||||
|
||||
- name: Evaluate accuracy (TP=2)
|
||||
timeout-minutes: 20
|
||||
timeout-minutes: 25
|
||||
run: |
|
||||
cd test/srt
|
||||
python3 test_moe_eval_accuracy_large.py
|
||||
python3 -m sglang.test.ci.run_with_retry test_moe_eval_accuracy_large.py
|
||||
|
||||
unit-test-deepep-4-gpu:
|
||||
needs: [check-changes, call-gate, unit-test-backend-2-gpu, stage-b-test-4-gpu-b200]
|
||||
@@ -1300,7 +1332,11 @@ jobs:
|
||||
timeout-minutes: 20
|
||||
run: |
|
||||
cd test/srt
|
||||
python3 run_suite.py --suite per-commit-4-gpu-deepep
|
||||
RETRY_FLAG=""
|
||||
if [[ "${{ needs.check-changes.outputs.enable_retry }}" == "true" ]]; then
|
||||
RETRY_FLAG="--enable-retry"
|
||||
fi
|
||||
python3 run_suite.py --suite per-commit-4-gpu-deepep $RETRY_FLAG
|
||||
|
||||
unit-test-deepep-8-gpu:
|
||||
needs: [check-changes, call-gate, unit-test-backend-2-gpu, stage-b-test-4-gpu-b200]
|
||||
@@ -1337,7 +1373,11 @@ jobs:
|
||||
timeout-minutes: 20
|
||||
run: |
|
||||
cd test/srt
|
||||
python3 run_suite.py --suite per-commit-8-gpu-h200-deepep
|
||||
RETRY_FLAG=""
|
||||
if [[ "${{ needs.check-changes.outputs.enable_retry }}" == "true" ]]; then
|
||||
RETRY_FLAG="--enable-retry"
|
||||
fi
|
||||
python3 run_suite.py --suite per-commit-8-gpu-h200-deepep $RETRY_FLAG
|
||||
|
||||
unit-test-backend-4-gpu-b200:
|
||||
needs: [check-changes, call-gate, unit-test-backend-2-gpu, stage-b-test-4-gpu-b200]
|
||||
@@ -1379,7 +1419,11 @@ jobs:
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
cd test/srt
|
||||
IS_BLACKWELL=1 python3 run_suite.py --suite per-commit-4-gpu-b200 --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --timeout-per-file 1800
|
||||
RETRY_FLAG=""
|
||||
if [[ "${{ needs.check-changes.outputs.enable_retry }}" == "true" ]]; then
|
||||
RETRY_FLAG="--enable-retry"
|
||||
fi
|
||||
IS_BLACKWELL=1 python3 run_suite.py --suite per-commit-4-gpu-b200 --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --timeout-per-file 1800 $RETRY_FLAG
|
||||
|
||||
unit-test-backend-4-gpu-gb200:
|
||||
needs: [check-changes, call-gate, unit-test-backend-2-gpu, stage-b-test-4-gpu-b200, sgl-kernel-build-wheels-arm]
|
||||
@@ -1418,7 +1462,11 @@ jobs:
|
||||
timeout-minutes: 45
|
||||
run: |
|
||||
cd test/srt
|
||||
python3 run_suite.py --suite per-commit-4-gpu-gb200 --auto-partition-id 0 --auto-partition-size 1 --timeout-per-file 3600
|
||||
RETRY_FLAG=""
|
||||
if [[ "${{ needs.check-changes.outputs.enable_retry }}" == "true" ]]; then
|
||||
RETRY_FLAG="--enable-retry"
|
||||
fi
|
||||
python3 run_suite.py --suite per-commit-4-gpu-gb200 --auto-partition-id 0 --auto-partition-size 1 --timeout-per-file 3600 $RETRY_FLAG
|
||||
|
||||
pr-test-finish:
|
||||
needs:
|
||||
|
||||
Reference in New Issue
Block a user