diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index d1e160017..f25b27402 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -20,6 +20,11 @@ on: required: false type: string default: "" + force_continue_on_error: + description: "Force continue-on-error (test scheduled CI behavior)" + required: false + type: boolean + default: false concurrency: group: pr-test-${{ github.ref }} @@ -39,6 +44,7 @@ jobs: max_parallel: ${{ steps.set-parallel.outputs.max_parallel }} b200_runner: ${{ steps.set-runner.outputs.b200_runner }} enable_retry: ${{ steps.set-retry.outputs.enable_retry }} + continue_on_error: ${{ steps.set-continue-on-error.outputs.continue_on_error }} steps: - name: Checkout code uses: actions/checkout@v4 @@ -98,19 +104,31 @@ jobs: echo "enable_retry=true" >> $GITHUB_OUTPUT echo "Retry logic enabled for CI" + - name: Set continue-on-error for scheduled runs + id: set-continue-on-error + run: | + if [[ "${{ github.event_name }}" == "schedule" || "${{ inputs.force_continue_on_error }}" == "true" ]]; then + echo "continue_on_error=true" >> $GITHUB_OUTPUT + echo "Scheduled run or force flag detected, enabling continue-on-error to run all tests" + else + echo "continue_on_error=false" >> $GITHUB_OUTPUT + echo "Non-scheduled run, continue-on-error disabled" + fi + - name: Show filter results in summary (table) run: | { echo "## Change Detection" echo "" - echo "| Component | Changed |" - echo "|----------------|---------|" - echo "| main_package | ${{ steps.filter.outputs.main_package || steps.scheduled.outputs.main_package }} |" - echo "| sgl_kernel | ${{ steps.filter.outputs.sgl_kernel || steps.scheduled.outputs.sgl_kernel }} |" - echo "| multimodal_gen | ${{ steps.filter.outputs.multimodal_gen || steps.scheduled.outputs.multimodal_gen }} |" - echo "| max_parallel | ${{ steps.set-parallel.outputs.max_parallel }} |" - echo "| b200_runner | ${{ steps.set-runner.outputs.b200_runner }} |" - echo "| enable_retry | ${{ steps.set-retry.outputs.enable_retry }} |" + echo "| Component | Changed |" + echo "|-------------------|---------|" + echo "| main_package | ${{ steps.filter.outputs.main_package || steps.scheduled.outputs.main_package }} |" + echo "| sgl_kernel | ${{ steps.filter.outputs.sgl_kernel || steps.scheduled.outputs.sgl_kernel }} |" + echo "| multimodal_gen | ${{ steps.filter.outputs.multimodal_gen || steps.scheduled.outputs.multimodal_gen }} |" + echo "| max_parallel | ${{ steps.set-parallel.outputs.max_parallel }} |" + echo "| b200_runner | ${{ steps.set-runner.outputs.b200_runner }} |" + echo "| enable_retry | ${{ steps.set-retry.outputs.enable_retry }} |" + echo "| continue_on_error | ${{ steps.set-continue-on-error.outputs.continue_on_error }} |" } >> $GITHUB_STEP_SUMMARY # =============================================== PR Gate ==================================================== @@ -411,9 +429,13 @@ jobs: timeout-minutes: 10 run: | cd test/ - python3 run_suite.py --hw cuda --suite stage-a-test-1 + CONTINUE_ON_ERROR_FLAG="" + if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then + CONTINUE_ON_ERROR_FLAG="--continue-on-error" + fi + python3 run_suite.py --hw cuda --suite stage-a-test-1 $CONTINUE_ON_ERROR_FLAG # temporarily put backend-independent cpu tests here - python3 run_suite.py --hw cpu --suite default + python3 run_suite.py --hw cpu --suite default $CONTINUE_ON_ERROR_FLAG stage-b-test-small-1-gpu: needs: [check-changes, call-gate, stage-a-test-1, sgl-kernel-build-wheels] @@ -454,7 +476,11 @@ jobs: timeout-minutes: 30 run: | cd test/ - python3 run_suite.py --hw cuda --suite stage-b-test-small-1-gpu --auto-partition-id ${{ matrix.partition }} --auto-partition-size 4 + CONTINUE_ON_ERROR_FLAG="" + if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then + CONTINUE_ON_ERROR_FLAG="--continue-on-error" + fi + python3 run_suite.py --hw cuda --suite stage-b-test-small-1-gpu --auto-partition-id ${{ matrix.partition }} --auto-partition-size 4 $CONTINUE_ON_ERROR_FLAG stage-b-test-large-1-gpu: needs: [check-changes, call-gate, stage-a-test-1, sgl-kernel-build-wheels] @@ -491,7 +517,11 @@ jobs: timeout-minutes: 30 run: | cd test/ - python3 run_suite.py --hw cuda --suite stage-b-test-large-1-gpu + CONTINUE_ON_ERROR_FLAG="" + if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then + CONTINUE_ON_ERROR_FLAG="--continue-on-error" + fi + python3 run_suite.py --hw cuda --suite stage-b-test-large-1-gpu $CONTINUE_ON_ERROR_FLAG stage-b-test-large-2-gpu: needs: [check-changes, call-gate, stage-a-test-1, sgl-kernel-build-wheels] @@ -528,7 +558,11 @@ jobs: timeout-minutes: 30 run: | cd test/ - python3 run_suite.py --hw cuda --suite stage-b-test-large-2-gpu + CONTINUE_ON_ERROR_FLAG="" + if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then + CONTINUE_ON_ERROR_FLAG="--continue-on-error" + fi + python3 run_suite.py --hw cuda --suite stage-b-test-large-2-gpu $CONTINUE_ON_ERROR_FLAG stage-c-test-large-4-gpu: needs: [check-changes, call-gate, stage-b-test-small-1-gpu, stage-b-test-large-1-gpu, stage-b-test-large-2-gpu, sgl-kernel-build-wheels] @@ -565,7 +599,11 @@ jobs: timeout-minutes: 30 run: | cd test/ - python3 run_suite.py --hw cuda --suite stage-c-test-large-4-gpu + CONTINUE_ON_ERROR_FLAG="" + if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then + CONTINUE_ON_ERROR_FLAG="--continue-on-error" + fi + python3 run_suite.py --hw cuda --suite stage-c-test-large-4-gpu $CONTINUE_ON_ERROR_FLAG multimodal-gen-test-1-gpu: needs: [check-changes, call-gate, sgl-kernel-build-wheels] @@ -603,10 +641,15 @@ jobs: timeout-minutes: 60 run: | cd python + CONTINUE_ON_ERROR_FLAG="" + if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then + CONTINUE_ON_ERROR_FLAG="--continue-on-error" + fi python3 sglang/multimodal_gen/test/run_suite.py \ --suite 1-gpu \ --partition-id ${{ matrix.part }} \ - --total-partitions 2 + --total-partitions 2 \ + $CONTINUE_ON_ERROR_FLAG multimodal-gen-test-2-gpu: @@ -646,10 +689,15 @@ jobs: timeout-minutes: 60 run: | cd python + CONTINUE_ON_ERROR_FLAG="" + if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then + CONTINUE_ON_ERROR_FLAG="--continue-on-error" + fi python3 sglang/multimodal_gen/test/run_suite.py \ --suite 2-gpu \ --partition-id ${{ matrix.part }} \ - --total-partitions 2 + --total-partitions 2 \ + $CONTINUE_ON_ERROR_FLAG quantization-test: needs: [check-changes, call-gate, stage-a-test-1] @@ -688,7 +736,11 @@ jobs: if [[ "${{ needs.check-changes.outputs.enable_retry }}" == "true" ]]; then RETRY_FLAG="--enable-retry" fi - python3 run_suite.py --suite quantization_test $RETRY_FLAG + CONTINUE_ON_ERROR_FLAG="" + if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then + CONTINUE_ON_ERROR_FLAG="--continue-on-error" + fi + python3 run_suite.py --suite quantization_test $RETRY_FLAG $CONTINUE_ON_ERROR_FLAG unit-test-backend-1-gpu: needs: [check-changes, call-gate, stage-a-test-1] @@ -734,7 +786,11 @@ jobs: if [[ "${{ needs.check-changes.outputs.enable_retry }}" == "true" ]]; then RETRY_FLAG="--enable-retry" fi - python3 run_suite.py --suite per-commit-1-gpu --auto-partition-id ${{ matrix.part }} --auto-partition-size 13 $RETRY_FLAG + CONTINUE_ON_ERROR_FLAG="" + if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then + CONTINUE_ON_ERROR_FLAG="--continue-on-error" + fi + python3 run_suite.py --suite per-commit-1-gpu --auto-partition-id ${{ matrix.part }} --auto-partition-size 13 $RETRY_FLAG $CONTINUE_ON_ERROR_FLAG stage-b-test-4-gpu-b200: @@ -775,7 +831,11 @@ jobs: timeout-minutes: 30 run: | cd test - IS_BLACKWELL=1 python3 run_suite.py --hw cuda --suite stage-b-test-4-gpu-b200 + CONTINUE_ON_ERROR_FLAG="" + if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then + CONTINUE_ON_ERROR_FLAG="--continue-on-error" + fi + IS_BLACKWELL=1 python3 run_suite.py --hw cuda --suite stage-b-test-4-gpu-b200 $CONTINUE_ON_ERROR_FLAG unit-test-backend-2-gpu: @@ -821,7 +881,11 @@ jobs: if [[ "${{ needs.check-changes.outputs.enable_retry }}" == "true" ]]; then RETRY_FLAG="--enable-retry" fi - python3 run_suite.py --suite per-commit-2-gpu --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 $RETRY_FLAG + CONTINUE_ON_ERROR_FLAG="" + if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then + CONTINUE_ON_ERROR_FLAG="--continue-on-error" + fi + python3 run_suite.py --suite per-commit-2-gpu --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 $RETRY_FLAG $CONTINUE_ON_ERROR_FLAG unit-test-backend-4-gpu: needs: [check-changes, call-gate, unit-test-backend-2-gpu, stage-b-test-4-gpu-b200] @@ -866,7 +930,11 @@ jobs: if [[ "${{ needs.check-changes.outputs.enable_retry }}" == "true" ]]; then RETRY_FLAG="--enable-retry" fi - python3 run_suite.py --suite per-commit-4-gpu --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 $RETRY_FLAG + CONTINUE_ON_ERROR_FLAG="" + if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then + CONTINUE_ON_ERROR_FLAG="--continue-on-error" + fi + python3 run_suite.py --suite per-commit-4-gpu --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 $RETRY_FLAG $CONTINUE_ON_ERROR_FLAG unit-test-backend-8-gpu-h200: needs: [check-changes, call-gate, unit-test-backend-2-gpu, stage-b-test-4-gpu-b200] @@ -917,7 +985,11 @@ jobs: if [[ "${{ needs.check-changes.outputs.enable_retry }}" == "true" ]]; then RETRY_FLAG="--enable-retry" fi - python3 run_suite.py --suite per-commit-8-gpu-h200 --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 $RETRY_FLAG + CONTINUE_ON_ERROR_FLAG="" + if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then + CONTINUE_ON_ERROR_FLAG="--continue-on-error" + fi + python3 run_suite.py --suite per-commit-8-gpu-h200 --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 $RETRY_FLAG $CONTINUE_ON_ERROR_FLAG unit-test-backend-8-gpu-h20: needs: [check-changes, call-gate, unit-test-backend-2-gpu, stage-b-test-4-gpu-b200] @@ -963,7 +1035,11 @@ jobs: if [[ "${{ needs.check-changes.outputs.enable_retry }}" == "true" ]]; then RETRY_FLAG="--enable-retry" fi - python3 run_suite.py --suite per-commit-8-gpu-h20 --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 $RETRY_FLAG + CONTINUE_ON_ERROR_FLAG="" + if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then + CONTINUE_ON_ERROR_FLAG="--continue-on-error" + fi + python3 run_suite.py --suite per-commit-8-gpu-h20 --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 $RETRY_FLAG $CONTINUE_ON_ERROR_FLAG performance-test-1-gpu-part-1: needs: [check-changes, call-gate, stage-a-test-1] @@ -1336,7 +1412,11 @@ jobs: if [[ "${{ needs.check-changes.outputs.enable_retry }}" == "true" ]]; then RETRY_FLAG="--enable-retry" fi - python3 run_suite.py --suite per-commit-4-gpu-deepep $RETRY_FLAG + CONTINUE_ON_ERROR_FLAG="" + if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then + CONTINUE_ON_ERROR_FLAG="--continue-on-error" + fi + python3 run_suite.py --suite per-commit-4-gpu-deepep $RETRY_FLAG $CONTINUE_ON_ERROR_FLAG unit-test-deepep-8-gpu: needs: [check-changes, call-gate, unit-test-backend-2-gpu, stage-b-test-4-gpu-b200] @@ -1377,7 +1457,11 @@ jobs: if [[ "${{ needs.check-changes.outputs.enable_retry }}" == "true" ]]; then RETRY_FLAG="--enable-retry" fi - python3 run_suite.py --suite per-commit-8-gpu-h200-deepep $RETRY_FLAG + CONTINUE_ON_ERROR_FLAG="" + if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then + CONTINUE_ON_ERROR_FLAG="--continue-on-error" + fi + python3 run_suite.py --suite per-commit-8-gpu-h200-deepep $RETRY_FLAG $CONTINUE_ON_ERROR_FLAG unit-test-backend-4-gpu-b200: needs: [check-changes, call-gate, unit-test-backend-2-gpu, stage-b-test-4-gpu-b200] @@ -1423,7 +1507,11 @@ jobs: if [[ "${{ needs.check-changes.outputs.enable_retry }}" == "true" ]]; then RETRY_FLAG="--enable-retry" fi - IS_BLACKWELL=1 python3 run_suite.py --suite per-commit-4-gpu-b200 --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --timeout-per-file 1800 $RETRY_FLAG + CONTINUE_ON_ERROR_FLAG="" + if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then + CONTINUE_ON_ERROR_FLAG="--continue-on-error" + fi + IS_BLACKWELL=1 python3 run_suite.py --suite per-commit-4-gpu-b200 --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --timeout-per-file 1800 $RETRY_FLAG $CONTINUE_ON_ERROR_FLAG unit-test-backend-4-gpu-gb200: needs: [check-changes, call-gate, unit-test-backend-2-gpu, stage-b-test-4-gpu-b200, sgl-kernel-build-wheels-arm] @@ -1466,7 +1554,11 @@ jobs: if [[ "${{ needs.check-changes.outputs.enable_retry }}" == "true" ]]; then RETRY_FLAG="--enable-retry" fi - python3 run_suite.py --suite per-commit-4-gpu-gb200 --auto-partition-id 0 --auto-partition-size 1 --timeout-per-file 3600 $RETRY_FLAG + CONTINUE_ON_ERROR_FLAG="" + if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then + CONTINUE_ON_ERROR_FLAG="--continue-on-error" + fi + python3 run_suite.py --suite per-commit-4-gpu-gb200 --auto-partition-id 0 --auto-partition-size 1 --timeout-per-file 3600 $RETRY_FLAG $CONTINUE_ON_ERROR_FLAG pr-test-finish: needs: diff --git a/python/sglang/multimodal_gen/test/run_suite.py b/python/sglang/multimodal_gen/test/run_suite.py index eaa103a6b..35fea064f 100644 --- a/python/sglang/multimodal_gen/test/run_suite.py +++ b/python/sglang/multimodal_gen/test/run_suite.py @@ -67,6 +67,12 @@ def parse_args(): default=None, help="Pytest filter expression (passed to pytest -k)", ) + parser.add_argument( + "--continue-on-error", + action="store_true", + default=False, + help="Continue running remaining tests even if one fails (for CI consistency; pytest already continues by default)", + ) return parser.parse_args()