chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

ci: add continue-on-error for scheduled PR tests (#15701)

Browse Source
This commit is contained in:
Alison Shao
2025-12-25 13:26:14 -08:00
committed by GitHub
parent f2ccc44240
commit 4805118122
2 changed files with 125 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

146
.github/workflows/pr-test.yml vendored
View File

@@ -20,6 +20,11 @@ on:
required: false
type: string
default: ""
force_continue_on_error:
description: "Force continue-on-error (test scheduled CI behavior)"
required: false
type: boolean
default: false
concurrency:
group: pr-test-${{ github.ref }}
@@ -39,6 +44,7 @@ jobs:
max_parallel: ${{ steps.set-parallel.outputs.max_parallel }}
b200_runner: ${{ steps.set-runner.outputs.b200_runner }}
enable_retry: ${{ steps.set-retry.outputs.enable_retry }}
continue_on_error: ${{ steps.set-continue-on-error.outputs.continue_on_error }}
steps:
- name: Checkout code
uses: actions/checkout@v4
@@ -98,19 +104,31 @@ jobs:
echo "enable_retry=true" >> $GITHUB_OUTPUT
echo "Retry logic enabled for CI"
- name: Set continue-on-error for scheduled runs
id: set-continue-on-error
run: |
if [[ "${{ github.event_name }}" == "schedule" || "${{ inputs.force_continue_on_error }}" == "true" ]]; then
echo "continue_on_error=true" >> $GITHUB_OUTPUT
echo "Scheduled run or force flag detected, enabling continue-on-error to run all tests"
else
echo "continue_on_error=false" >> $GITHUB_OUTPUT
echo "Non-scheduled run, continue-on-error disabled"
fi
- name: Show filter results in summary (table)
run: |
{
echo "## Change Detection"
echo ""
echo "| Component | Changed |"
echo "|----------------|---------|"
echo "| main_package | ${{ steps.filter.outputs.main_package || steps.scheduled.outputs.main_package }} |"
echo "| sgl_kernel | ${{ steps.filter.outputs.sgl_kernel || steps.scheduled.outputs.sgl_kernel }} |"
echo "| multimodal_gen | ${{ steps.filter.outputs.multimodal_gen || steps.scheduled.outputs.multimodal_gen }} |"
echo "| max_parallel | ${{ steps.set-parallel.outputs.max_parallel }} |"
echo "| b200_runner | ${{ steps.set-runner.outputs.b200_runner }} |"
echo "| enable_retry | ${{ steps.set-retry.outputs.enable_retry }} |"
echo "| Component | Changed |"
echo "|-------------------|---------|"
echo "| main_package | ${{ steps.filter.outputs.main_package || steps.scheduled.outputs.main_package }} |"
echo "| sgl_kernel | ${{ steps.filter.outputs.sgl_kernel || steps.scheduled.outputs.sgl_kernel }} |"
echo "| multimodal_gen | ${{ steps.filter.outputs.multimodal_gen || steps.scheduled.outputs.multimodal_gen }} |"
echo "| max_parallel | ${{ steps.set-parallel.outputs.max_parallel }} |"
echo "| b200_runner | ${{ steps.set-runner.outputs.b200_runner }} |"
echo "| enable_retry | ${{ steps.set-retry.outputs.enable_retry }} |"
echo "| continue_on_error | ${{ steps.set-continue-on-error.outputs.continue_on_error }} |"
} >> $GITHUB_STEP_SUMMARY
# =============================================== PR Gate ====================================================
@@ -411,9 +429,13 @@ jobs:
timeout-minutes: 10
run: |
cd test/
python3 run_suite.py --hw cuda --suite stage-a-test-1
CONTINUE_ON_ERROR_FLAG=""
if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then
CONTINUE_ON_ERROR_FLAG="--continue-on-error"
fi
python3 run_suite.py --hw cuda --suite stage-a-test-1 $CONTINUE_ON_ERROR_FLAG
# temporarily put backend-independent cpu tests here
python3 run_suite.py --hw cpu --suite default
python3 run_suite.py --hw cpu --suite default $CONTINUE_ON_ERROR_FLAG
stage-b-test-small-1-gpu:
needs: [check-changes, call-gate, stage-a-test-1, sgl-kernel-build-wheels]
@@ -454,7 +476,11 @@ jobs:
timeout-minutes: 30
run: |
cd test/
python3 run_suite.py --hw cuda --suite stage-b-test-small-1-gpu --auto-partition-id ${{ matrix.partition }} --auto-partition-size 4
CONTINUE_ON_ERROR_FLAG=""
if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then
CONTINUE_ON_ERROR_FLAG="--continue-on-error"
fi
python3 run_suite.py --hw cuda --suite stage-b-test-small-1-gpu --auto-partition-id ${{ matrix.partition }} --auto-partition-size 4 $CONTINUE_ON_ERROR_FLAG
stage-b-test-large-1-gpu:
needs: [check-changes, call-gate, stage-a-test-1, sgl-kernel-build-wheels]
@@ -491,7 +517,11 @@ jobs:
timeout-minutes: 30
run: |
cd test/
python3 run_suite.py --hw cuda --suite stage-b-test-large-1-gpu
CONTINUE_ON_ERROR_FLAG=""
if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then
CONTINUE_ON_ERROR_FLAG="--continue-on-error"
fi
python3 run_suite.py --hw cuda --suite stage-b-test-large-1-gpu $CONTINUE_ON_ERROR_FLAG
stage-b-test-large-2-gpu:
needs: [check-changes, call-gate, stage-a-test-1, sgl-kernel-build-wheels]
@@ -528,7 +558,11 @@ jobs:
timeout-minutes: 30
run: |
cd test/
python3 run_suite.py --hw cuda --suite stage-b-test-large-2-gpu
CONTINUE_ON_ERROR_FLAG=""
if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then
CONTINUE_ON_ERROR_FLAG="--continue-on-error"
fi
python3 run_suite.py --hw cuda --suite stage-b-test-large-2-gpu $CONTINUE_ON_ERROR_FLAG
stage-c-test-large-4-gpu:
needs: [check-changes, call-gate, stage-b-test-small-1-gpu, stage-b-test-large-1-gpu, stage-b-test-large-2-gpu, sgl-kernel-build-wheels]
@@ -565,7 +599,11 @@ jobs:
timeout-minutes: 30
run: |
cd test/
python3 run_suite.py --hw cuda --suite stage-c-test-large-4-gpu
CONTINUE_ON_ERROR_FLAG=""
if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then
CONTINUE_ON_ERROR_FLAG="--continue-on-error"
fi
python3 run_suite.py --hw cuda --suite stage-c-test-large-4-gpu $CONTINUE_ON_ERROR_FLAG
multimodal-gen-test-1-gpu:
needs: [check-changes, call-gate, sgl-kernel-build-wheels]
@@ -603,10 +641,15 @@ jobs:
timeout-minutes: 60
run: |
cd python
CONTINUE_ON_ERROR_FLAG=""
if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then
CONTINUE_ON_ERROR_FLAG="--continue-on-error"
fi
python3 sglang/multimodal_gen/test/run_suite.py \
--suite 1-gpu \
--partition-id ${{ matrix.part }} \
--total-partitions 2
--total-partitions 2 \
$CONTINUE_ON_ERROR_FLAG
multimodal-gen-test-2-gpu:
@@ -646,10 +689,15 @@ jobs:
timeout-minutes: 60
run: |
cd python
CONTINUE_ON_ERROR_FLAG=""
if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then
CONTINUE_ON_ERROR_FLAG="--continue-on-error"
fi
python3 sglang/multimodal_gen/test/run_suite.py \
--suite 2-gpu \
--partition-id ${{ matrix.part }} \
--total-partitions 2
--total-partitions 2 \
$CONTINUE_ON_ERROR_FLAG
quantization-test:
needs: [check-changes, call-gate, stage-a-test-1]
@@ -688,7 +736,11 @@ jobs:
if [[ "${{ needs.check-changes.outputs.enable_retry }}" == "true" ]]; then
RETRY_FLAG="--enable-retry"
fi
python3 run_suite.py --suite quantization_test $RETRY_FLAG
CONTINUE_ON_ERROR_FLAG=""
if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then
CONTINUE_ON_ERROR_FLAG="--continue-on-error"
fi
python3 run_suite.py --suite quantization_test $RETRY_FLAG $CONTINUE_ON_ERROR_FLAG
unit-test-backend-1-gpu:
needs: [check-changes, call-gate, stage-a-test-1]
@@ -734,7 +786,11 @@ jobs:
if [[ "${{ needs.check-changes.outputs.enable_retry }}" == "true" ]]; then
RETRY_FLAG="--enable-retry"
fi
python3 run_suite.py --suite per-commit-1-gpu --auto-partition-id ${{ matrix.part }} --auto-partition-size 13 $RETRY_FLAG
CONTINUE_ON_ERROR_FLAG=""
if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then
CONTINUE_ON_ERROR_FLAG="--continue-on-error"
fi
python3 run_suite.py --suite per-commit-1-gpu --auto-partition-id ${{ matrix.part }} --auto-partition-size 13 $RETRY_FLAG $CONTINUE_ON_ERROR_FLAG
stage-b-test-4-gpu-b200:
@@ -775,7 +831,11 @@ jobs:
timeout-minutes: 30
run: |
cd test
IS_BLACKWELL=1 python3 run_suite.py --hw cuda --suite stage-b-test-4-gpu-b200
CONTINUE_ON_ERROR_FLAG=""
if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then
CONTINUE_ON_ERROR_FLAG="--continue-on-error"
fi
IS_BLACKWELL=1 python3 run_suite.py --hw cuda --suite stage-b-test-4-gpu-b200 $CONTINUE_ON_ERROR_FLAG
unit-test-backend-2-gpu:
@@ -821,7 +881,11 @@ jobs:
if [[ "${{ needs.check-changes.outputs.enable_retry }}" == "true" ]]; then
RETRY_FLAG="--enable-retry"
fi
python3 run_suite.py --suite per-commit-2-gpu --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 $RETRY_FLAG
CONTINUE_ON_ERROR_FLAG=""
if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then
CONTINUE_ON_ERROR_FLAG="--continue-on-error"
fi
python3 run_suite.py --suite per-commit-2-gpu --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 $RETRY_FLAG $CONTINUE_ON_ERROR_FLAG
unit-test-backend-4-gpu:
needs: [check-changes, call-gate, unit-test-backend-2-gpu, stage-b-test-4-gpu-b200]
@@ -866,7 +930,11 @@ jobs:
if [[ "${{ needs.check-changes.outputs.enable_retry }}" == "true" ]]; then
RETRY_FLAG="--enable-retry"
fi
python3 run_suite.py --suite per-commit-4-gpu --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 $RETRY_FLAG
CONTINUE_ON_ERROR_FLAG=""
if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then
CONTINUE_ON_ERROR_FLAG="--continue-on-error"
fi
python3 run_suite.py --suite per-commit-4-gpu --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 $RETRY_FLAG $CONTINUE_ON_ERROR_FLAG
unit-test-backend-8-gpu-h200:
needs: [check-changes, call-gate, unit-test-backend-2-gpu, stage-b-test-4-gpu-b200]
@@ -917,7 +985,11 @@ jobs:
if [[ "${{ needs.check-changes.outputs.enable_retry }}" == "true" ]]; then
RETRY_FLAG="--enable-retry"
fi
python3 run_suite.py --suite per-commit-8-gpu-h200 --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 $RETRY_FLAG
CONTINUE_ON_ERROR_FLAG=""
if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then
CONTINUE_ON_ERROR_FLAG="--continue-on-error"
fi
python3 run_suite.py --suite per-commit-8-gpu-h200 --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 $RETRY_FLAG $CONTINUE_ON_ERROR_FLAG
unit-test-backend-8-gpu-h20:
needs: [check-changes, call-gate, unit-test-backend-2-gpu, stage-b-test-4-gpu-b200]
@@ -963,7 +1035,11 @@ jobs:
if [[ "${{ needs.check-changes.outputs.enable_retry }}" == "true" ]]; then
RETRY_FLAG="--enable-retry"
fi
python3 run_suite.py --suite per-commit-8-gpu-h20 --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 $RETRY_FLAG
CONTINUE_ON_ERROR_FLAG=""
if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then
CONTINUE_ON_ERROR_FLAG="--continue-on-error"
fi
python3 run_suite.py --suite per-commit-8-gpu-h20 --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 $RETRY_FLAG $CONTINUE_ON_ERROR_FLAG
performance-test-1-gpu-part-1:
needs: [check-changes, call-gate, stage-a-test-1]
@@ -1336,7 +1412,11 @@ jobs:
if [[ "${{ needs.check-changes.outputs.enable_retry }}" == "true" ]]; then
RETRY_FLAG="--enable-retry"
fi
python3 run_suite.py --suite per-commit-4-gpu-deepep $RETRY_FLAG
CONTINUE_ON_ERROR_FLAG=""
if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then
CONTINUE_ON_ERROR_FLAG="--continue-on-error"
fi
python3 run_suite.py --suite per-commit-4-gpu-deepep $RETRY_FLAG $CONTINUE_ON_ERROR_FLAG
unit-test-deepep-8-gpu:
needs: [check-changes, call-gate, unit-test-backend-2-gpu, stage-b-test-4-gpu-b200]
@@ -1377,7 +1457,11 @@ jobs:
if [[ "${{ needs.check-changes.outputs.enable_retry }}" == "true" ]]; then
RETRY_FLAG="--enable-retry"
fi
python3 run_suite.py --suite per-commit-8-gpu-h200-deepep $RETRY_FLAG
CONTINUE_ON_ERROR_FLAG=""
if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then
CONTINUE_ON_ERROR_FLAG="--continue-on-error"
fi
python3 run_suite.py --suite per-commit-8-gpu-h200-deepep $RETRY_FLAG $CONTINUE_ON_ERROR_FLAG
unit-test-backend-4-gpu-b200:
needs: [check-changes, call-gate, unit-test-backend-2-gpu, stage-b-test-4-gpu-b200]
@@ -1423,7 +1507,11 @@ jobs:
if [[ "${{ needs.check-changes.outputs.enable_retry }}" == "true" ]]; then
RETRY_FLAG="--enable-retry"
fi
IS_BLACKWELL=1 python3 run_suite.py --suite per-commit-4-gpu-b200 --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --timeout-per-file 1800 $RETRY_FLAG
CONTINUE_ON_ERROR_FLAG=""
if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then
CONTINUE_ON_ERROR_FLAG="--continue-on-error"
fi
IS_BLACKWELL=1 python3 run_suite.py --suite per-commit-4-gpu-b200 --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --timeout-per-file 1800 $RETRY_FLAG $CONTINUE_ON_ERROR_FLAG
unit-test-backend-4-gpu-gb200:
needs: [check-changes, call-gate, unit-test-backend-2-gpu, stage-b-test-4-gpu-b200, sgl-kernel-build-wheels-arm]
@@ -1466,7 +1554,11 @@ jobs:
if [[ "${{ needs.check-changes.outputs.enable_retry }}" == "true" ]]; then
RETRY_FLAG="--enable-retry"
fi
python3 run_suite.py --suite per-commit-4-gpu-gb200 --auto-partition-id 0 --auto-partition-size 1 --timeout-per-file 3600 $RETRY_FLAG
CONTINUE_ON_ERROR_FLAG=""
if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then
CONTINUE_ON_ERROR_FLAG="--continue-on-error"
fi
python3 run_suite.py --suite per-commit-4-gpu-gb200 --auto-partition-id 0 --auto-partition-size 1 --timeout-per-file 3600 $RETRY_FLAG $CONTINUE_ON_ERROR_FLAG
pr-test-finish:
needs: