chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 11 Packages Projects Releases Wiki Activity

Add retry logic for scheduled CI tests (#14771)

Browse Source
This commit is contained in:
Alison Shao
2025-12-11 16:59:58 -08:00
committed by GitHub
parent d6bd2d1126
commit e59435c34b
4 changed files with 281 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files

49
.github/workflows/pr-test.yml vendored
View File

@@ -27,6 +27,7 @@ concurrency:
env:
SGLANG_IS_IN_CI: true
SGLANG_CI_ENABLE_RETRY: true
jobs:
# =============================================== check changes ====================================================
@@ -550,10 +551,10 @@ jobs:
CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/ci_install_dependency.sh
- name: Run test
timeout-minutes: 30
timeout-minutes: 40
run: |
cd test/srt
python3 run_suite.py --suite quantization_test
python3 run_suite.py --suite quantization_test --enable-retry
unit-test-backend-1-gpu:
needs: [check-changes, call-gate, stage-a-test-1]
@@ -592,10 +593,10 @@ jobs:
CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/ci_install_dependency.sh
- name: Run test
timeout-minutes: 30
timeout-minutes: 40
run: |
cd test/srt
python3 run_suite.py --suite per-commit-1-gpu --auto-partition-id ${{ matrix.part }} --auto-partition-size 15
python3 run_suite.py --suite per-commit-1-gpu --auto-partition-id ${{ matrix.part }} --auto-partition-size 15 --enable-retry
unit-test-backend-2-gpu:
needs: [check-changes, call-gate, unit-test-backend-1-gpu]
@@ -633,10 +634,10 @@ jobs:
CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/ci_install_dependency.sh
- name: Run test
timeout-minutes: 30
timeout-minutes: 40
run: |
cd test/srt
python3 run_suite.py --suite per-commit-2-gpu --auto-partition-id ${{ matrix.part }} --auto-partition-size 2
python3 run_suite.py --suite per-commit-2-gpu --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 --enable-retry
unit-test-backend-4-gpu:
needs: [check-changes, call-gate, unit-test-backend-2-gpu]
@@ -674,10 +675,10 @@ jobs:
CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/ci_install_dependency.sh
- name: Run test
timeout-minutes: 20
timeout-minutes: 30
run: |
cd test/srt
python3 run_suite.py --suite per-commit-4-gpu --auto-partition-id ${{ matrix.part }} --auto-partition-size 3
python3 run_suite.py --suite per-commit-4-gpu --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --enable-retry
unit-test-backend-8-gpu-h200:
needs: [check-changes, call-gate, unit-test-backend-2-gpu]
@@ -721,10 +722,10 @@ jobs:
python3 -m sglang.compile_deep_gemm --model deepseek-ai/DeepSeek-V3-0324 --tp 8 --trust-remote-code
- name: Run test
timeout-minutes: 20
timeout-minutes: 30
run: |
cd test/srt
python3 run_suite.py --suite per-commit-8-gpu-h200 --auto-partition-id ${{ matrix.part }} --auto-partition-size 3
python3 run_suite.py --suite per-commit-8-gpu-h200 --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --enable-retry
unit-test-backend-8-gpu-h20:
needs: [check-changes, call-gate, unit-test-backend-2-gpu]
@@ -763,10 +764,10 @@ jobs:
CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/ci_install_deepep.sh
- name: Run test
timeout-minutes: 20
timeout-minutes: 30
run: |
cd test/srt
python3 run_suite.py --suite per-commit-8-gpu-h20 --auto-partition-id ${{ matrix.part }} --auto-partition-size 2
python3 run_suite.py --suite per-commit-8-gpu-h20 --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 --enable-retry
performance-test-1-gpu-part-1:
needs: [check-changes, call-gate, stage-a-test-1]
@@ -1055,10 +1056,12 @@ jobs:
pip install -e .
- name: Evaluate accuracy
timeout-minutes: 20
timeout-minutes: 25
run: |
cd test/srt
python3 test_eval_accuracy_large.py
env:
SGLANG_CI_ENABLE_RETRY: ${{ env.SGLANG_CI_ENABLE_RETRY }}
accuracy-test-2-gpu:
needs: [check-changes, call-gate, accuracy-test-1-gpu]
@@ -1095,10 +1098,12 @@ jobs:
pip install -e .
- name: Evaluate accuracy (TP=2)
timeout-minutes: 20
timeout-minutes: 25
run: |
cd test/srt
python3 test_moe_eval_accuracy_large.py
env:
SGLANG_CI_ENABLE_RETRY: ${{ env.SGLANG_CI_ENABLE_RETRY }}
unit-test-deepep-4-gpu:
needs: [check-changes, call-gate, unit-test-backend-2-gpu]
@@ -1132,10 +1137,10 @@ jobs:
CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/ci_install_deepep.sh
- name: Run test
timeout-minutes: 20
timeout-minutes: 30
run: |
cd test/srt
python3 run_suite.py --suite per-commit-4-gpu-deepep
python3 run_suite.py --suite per-commit-4-gpu-deepep --enable-retry
unit-test-deepep-8-gpu:
needs: [check-changes, call-gate, unit-test-backend-2-gpu]
@@ -1169,10 +1174,10 @@ jobs:
CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/ci_install_deepep.sh
- name: Run test
timeout-minutes: 20
timeout-minutes: 30
run: |
cd test/srt
python3 run_suite.py --suite per-commit-8-gpu-h200-deepep
python3 run_suite.py --suite per-commit-8-gpu-h200-deepep --enable-retry
unit-test-backend-4-gpu-b200:
needs: [check-changes, call-gate, unit-test-backend-2-gpu]
@@ -1211,10 +1216,10 @@ jobs:
CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} IS_BLACKWELL=1 bash scripts/ci/ci_install_dependency.sh
- name: Run test
timeout-minutes: 30
timeout-minutes: 40
run: |
cd test/srt
IS_BLACKWELL=1 python3 run_suite.py --suite per-commit-4-gpu-b200 --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --timeout-per-file 1800
IS_BLACKWELL=1 python3 run_suite.py --suite per-commit-4-gpu-b200 --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --timeout-per-file 1800 --enable-retry
# TODO: Add gb200 tests back after the ci runner is fixed
# unit-test-backend-4-gpu-gb200:
@@ -1251,10 +1256,10 @@ jobs:
# CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} IS_BLACKWELL=1 GRACE_BLACKWELL=1 bash scripts/ci/ci_install_deepep.sh
# - name: Run test
# timeout-minutes: 45
# timeout-minutes: 55
# run: |
# cd test/srt
# python3 run_suite.py --suite per-commit-4-gpu-gb200 --auto-partition-id 0 --auto-partition-size 1 --timeout-per-file 3600
# python3 run_suite.py --suite per-commit-4-gpu-gb200 --auto-partition-id 0 --auto-partition-size 1 --timeout-per-file 3600 --enable-retry
pr-test-finish:
needs: