ci: run Stage A CUDA tests as stage-a-test-small-1-gpu on 5090 (#20988)
This commit is contained in:
@@ -333,7 +333,7 @@ jobs:
|
||||
id: wait
|
||||
with:
|
||||
stage-name: stage-a
|
||||
jobs: '["stage-a-test-1", "stage-a-cpu-only"]'
|
||||
jobs: '["stage-a-test-small-1-gpu", "stage-a-cpu-only"]'
|
||||
max-wait-minutes: '240'
|
||||
|
||||
wait-for-stage-b:
|
||||
@@ -783,20 +783,23 @@ jobs:
|
||||
|
||||
# =============================================== primary ====================================================
|
||||
|
||||
stage-a-test-1:
|
||||
# Runs on 5090 (32GB, SM120)
|
||||
stage-a-test-small-1-gpu:
|
||||
needs: [check-changes, call-gate, sgl-kernel-build-wheels]
|
||||
if: |
|
||||
always() &&
|
||||
(
|
||||
(inputs.target_stage == 'stage-a-test-1') ||
|
||||
(inputs.target_stage == 'stage-a-test-small-1-gpu') ||
|
||||
(
|
||||
!inputs.target_stage &&
|
||||
((github.event_name == 'schedule' || inputs.test_parallel_dispatch == true) || (!failure() && !cancelled())) &&
|
||||
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
||||
)
|
||||
)
|
||||
runs-on: 1-gpu-runner
|
||||
runs-on: 1-gpu-5090
|
||||
timeout-minutes: 240
|
||||
env:
|
||||
IS_BLACKWELL: "1"
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
@@ -818,13 +821,11 @@ jobs:
|
||||
|
||||
- name: Run test
|
||||
timeout-minutes: 10
|
||||
env:
|
||||
CONTINUE_ON_ERROR_FLAG: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
||||
run: |
|
||||
cd test/
|
||||
CONTINUE_ON_ERROR_FLAG=""
|
||||
if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then
|
||||
CONTINUE_ON_ERROR_FLAG="--continue-on-error"
|
||||
fi
|
||||
python3 run_suite.py --hw cuda --suite stage-a-test-1 $CONTINUE_ON_ERROR_FLAG
|
||||
python3 run_suite.py --hw cuda --suite stage-a-test-small-1-gpu $CONTINUE_ON_ERROR_FLAG
|
||||
|
||||
- uses: ./.github/actions/upload-cuda-coredumps
|
||||
if: always()
|
||||
@@ -866,12 +867,10 @@ jobs:
|
||||
|
||||
- name: Run test
|
||||
timeout-minutes: 10
|
||||
env:
|
||||
CONTINUE_ON_ERROR_FLAG: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
||||
run: |
|
||||
cd test/
|
||||
CONTINUE_ON_ERROR_FLAG=""
|
||||
if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then
|
||||
CONTINUE_ON_ERROR_FLAG="--continue-on-error"
|
||||
fi
|
||||
python3 run_suite.py --hw cpu --suite stage-a-cpu-only $CONTINUE_ON_ERROR_FLAG
|
||||
|
||||
# Runs on 5090 (32GB, SM120)
|
||||
@@ -921,13 +920,11 @@ jobs:
|
||||
|
||||
- name: Run test
|
||||
timeout-minutes: 30
|
||||
env:
|
||||
CONTINUE_ON_ERROR_FLAG: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
||||
run: |
|
||||
source /etc/profile.d/sglang-ci.sh
|
||||
cd test/
|
||||
CONTINUE_ON_ERROR_FLAG=""
|
||||
if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then
|
||||
CONTINUE_ON_ERROR_FLAG="--continue-on-error"
|
||||
fi
|
||||
python3 run_suite.py --hw cuda --suite stage-b-test-small-1-gpu --auto-partition-id ${{ matrix.partition }} --auto-partition-size 8 $CONTINUE_ON_ERROR_FLAG
|
||||
|
||||
- uses: ./.github/actions/upload-cuda-coredumps
|
||||
@@ -976,12 +973,10 @@ jobs:
|
||||
|
||||
- name: Run test
|
||||
timeout-minutes: 30
|
||||
env:
|
||||
CONTINUE_ON_ERROR_FLAG: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
||||
run: |
|
||||
cd test/
|
||||
CONTINUE_ON_ERROR_FLAG=""
|
||||
if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then
|
||||
CONTINUE_ON_ERROR_FLAG="--continue-on-error"
|
||||
fi
|
||||
python3 run_suite.py --hw cuda --suite stage-b-test-large-1-gpu --auto-partition-id ${{ matrix.partition }} --auto-partition-size 14 --timeout-per-file 1800 $CONTINUE_ON_ERROR_FLAG
|
||||
|
||||
- uses: ./.github/actions/upload-cuda-coredumps
|
||||
@@ -1031,12 +1026,10 @@ jobs:
|
||||
|
||||
- name: Run test
|
||||
timeout-minutes: 30
|
||||
env:
|
||||
CONTINUE_ON_ERROR_FLAG: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
||||
run: |
|
||||
cd test/
|
||||
CONTINUE_ON_ERROR_FLAG=""
|
||||
if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then
|
||||
CONTINUE_ON_ERROR_FLAG="--continue-on-error"
|
||||
fi
|
||||
python3 run_suite.py --hw cuda --suite stage-b-test-large-2-gpu --auto-partition-id ${{ matrix.partition }} --auto-partition-size 4 $CONTINUE_ON_ERROR_FLAG
|
||||
|
||||
- uses: ./.github/actions/upload-cuda-coredumps
|
||||
@@ -1082,12 +1075,10 @@ jobs:
|
||||
|
||||
- name: Run test
|
||||
timeout-minutes: 30
|
||||
env:
|
||||
CONTINUE_ON_ERROR_FLAG: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
||||
run: |
|
||||
cd test
|
||||
CONTINUE_ON_ERROR_FLAG=""
|
||||
if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then
|
||||
CONTINUE_ON_ERROR_FLAG="--continue-on-error"
|
||||
fi
|
||||
IS_BLACKWELL=1 python3 run_suite.py --hw cuda --suite stage-b-test-4-gpu-b200 $CONTINUE_ON_ERROR_FLAG
|
||||
|
||||
- name: Run FA4 jit_kernel tests (SM100+)
|
||||
@@ -1138,12 +1129,9 @@ jobs:
|
||||
timeout-minutes: 240
|
||||
env:
|
||||
RUNAI_STREAMER_MEMORY_LIMIT: 0
|
||||
CONTINUE_ON_ERROR_FLAG: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
||||
run: |
|
||||
cd python
|
||||
CONTINUE_ON_ERROR_FLAG=""
|
||||
if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then
|
||||
CONTINUE_ON_ERROR_FLAG="--continue-on-error"
|
||||
fi
|
||||
python3 sglang/multimodal_gen/test/run_suite.py \
|
||||
--suite 1-gpu \
|
||||
--partition-id ${{ matrix.part }} \
|
||||
@@ -1196,12 +1184,9 @@ jobs:
|
||||
timeout-minutes: 240
|
||||
env:
|
||||
RUNAI_STREAMER_MEMORY_LIMIT: 0
|
||||
CONTINUE_ON_ERROR_FLAG: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
||||
run: |
|
||||
cd python
|
||||
CONTINUE_ON_ERROR_FLAG=""
|
||||
if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then
|
||||
CONTINUE_ON_ERROR_FLAG="--continue-on-error"
|
||||
fi
|
||||
python3 sglang/multimodal_gen/test/run_suite.py \
|
||||
--suite 2-gpu \
|
||||
--partition-id ${{ matrix.part }} \
|
||||
@@ -1291,12 +1276,10 @@ jobs:
|
||||
|
||||
- name: Run test
|
||||
timeout-minutes: 30
|
||||
env:
|
||||
CONTINUE_ON_ERROR_FLAG: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
||||
run: |
|
||||
cd test
|
||||
CONTINUE_ON_ERROR_FLAG=""
|
||||
if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then
|
||||
CONTINUE_ON_ERROR_FLAG="--continue-on-error"
|
||||
fi
|
||||
python3 run_suite.py --hw cuda --suite stage-c-test-4-gpu-h100 --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 $CONTINUE_ON_ERROR_FLAG
|
||||
|
||||
- uses: ./.github/actions/upload-cuda-coredumps
|
||||
@@ -1357,12 +1340,10 @@ jobs:
|
||||
|
||||
- name: Run test
|
||||
timeout-minutes: 30
|
||||
env:
|
||||
CONTINUE_ON_ERROR_FLAG: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
||||
run: |
|
||||
cd test
|
||||
CONTINUE_ON_ERROR_FLAG=""
|
||||
if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then
|
||||
CONTINUE_ON_ERROR_FLAG="--continue-on-error"
|
||||
fi
|
||||
python3 run_suite.py --hw cuda --suite stage-c-test-8-gpu-h200 --auto-partition-id ${{ matrix.part }} --auto-partition-size 4 $CONTINUE_ON_ERROR_FLAG
|
||||
|
||||
- uses: ./.github/actions/upload-cuda-coredumps
|
||||
@@ -1411,12 +1392,10 @@ jobs:
|
||||
|
||||
- name: Run test
|
||||
timeout-minutes: 30
|
||||
env:
|
||||
CONTINUE_ON_ERROR_FLAG: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
||||
run: |
|
||||
cd test
|
||||
CONTINUE_ON_ERROR_FLAG=""
|
||||
if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then
|
||||
CONTINUE_ON_ERROR_FLAG="--continue-on-error"
|
||||
fi
|
||||
python3 run_suite.py --hw cuda --suite stage-c-test-8-gpu-h20 --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 $CONTINUE_ON_ERROR_FLAG
|
||||
|
||||
- uses: ./.github/actions/upload-cuda-coredumps
|
||||
@@ -1471,12 +1450,10 @@ jobs:
|
||||
|
||||
- name: Run test
|
||||
timeout-minutes: 30
|
||||
env:
|
||||
CONTINUE_ON_ERROR_FLAG: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
||||
run: |
|
||||
cd test
|
||||
CONTINUE_ON_ERROR_FLAG=""
|
||||
if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then
|
||||
CONTINUE_ON_ERROR_FLAG="--continue-on-error"
|
||||
fi
|
||||
python3 run_suite.py --hw cuda --suite stage-c-test-deepep-4-gpu $CONTINUE_ON_ERROR_FLAG
|
||||
|
||||
- uses: ./.github/actions/upload-cuda-coredumps
|
||||
@@ -1530,12 +1507,10 @@ jobs:
|
||||
|
||||
- name: Run test
|
||||
timeout-minutes: 45
|
||||
env:
|
||||
CONTINUE_ON_ERROR_FLAG: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
||||
run: |
|
||||
cd test
|
||||
CONTINUE_ON_ERROR_FLAG=""
|
||||
if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then
|
||||
CONTINUE_ON_ERROR_FLAG="--continue-on-error"
|
||||
fi
|
||||
python3 run_suite.py --hw cuda --suite stage-c-test-deepep-8-gpu-h200 $CONTINUE_ON_ERROR_FLAG
|
||||
|
||||
- uses: ./.github/actions/upload-cuda-coredumps
|
||||
@@ -1581,12 +1556,10 @@ jobs:
|
||||
|
||||
- name: Run test
|
||||
timeout-minutes: 30
|
||||
env:
|
||||
CONTINUE_ON_ERROR_FLAG: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
||||
run: |
|
||||
cd test
|
||||
CONTINUE_ON_ERROR_FLAG=""
|
||||
if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then
|
||||
CONTINUE_ON_ERROR_FLAG="--continue-on-error"
|
||||
fi
|
||||
IS_BLACKWELL=1 python3 run_suite.py --hw cuda --suite stage-c-test-4-gpu-b200 --auto-partition-id ${{ matrix.part }} --auto-partition-size 4 --timeout-per-file 1800 $CONTINUE_ON_ERROR_FLAG
|
||||
|
||||
- uses: ./.github/actions/upload-cuda-coredumps
|
||||
@@ -1633,12 +1606,10 @@ jobs:
|
||||
#
|
||||
# - name: Run test
|
||||
# timeout-minutes: 45
|
||||
# env:
|
||||
# CONTINUE_ON_ERROR_FLAG: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
||||
# run: |
|
||||
# cd test
|
||||
# CONTINUE_ON_ERROR_FLAG=""
|
||||
# if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then
|
||||
# CONTINUE_ON_ERROR_FLAG="--continue-on-error"
|
||||
# fi
|
||||
# python3 run_suite.py --hw cuda --suite stage-c-test-4-gpu-gb200 --timeout-per-file 3600 $CONTINUE_ON_ERROR_FLAG
|
||||
#
|
||||
# - uses: ./.github/actions/upload-cuda-coredumps
|
||||
@@ -1667,7 +1638,7 @@ jobs:
|
||||
multimodal-gen-test-1-gpu,
|
||||
multimodal-gen-test-2-gpu,
|
||||
|
||||
stage-a-test-1,
|
||||
stage-a-test-small-1-gpu,
|
||||
stage-a-cpu-only,
|
||||
stage-b-test-small-1-gpu,
|
||||
stage-b-test-large-1-gpu,
|
||||
|
||||
Reference in New Issue
Block a user