feat: add cuda core dump CI warpper (#18909)
This commit is contained in:
@@ -63,6 +63,7 @@ concurrency:
|
||||
|
||||
env:
|
||||
SGLANG_IS_IN_CI: true
|
||||
SGLANG_CUDA_COREDUMP: "1"
|
||||
SGLANG_JIT_DEEPGEMM_FAST_WARMUP: true
|
||||
|
||||
permissions:
|
||||
@@ -874,6 +875,9 @@ jobs:
|
||||
# temporarily put backend-independent cpu tests here
|
||||
python3 run_suite.py --hw cpu --suite default $CONTINUE_ON_ERROR_FLAG
|
||||
|
||||
- uses: ./.github/actions/upload-cuda-coredumps
|
||||
if: always()
|
||||
|
||||
stage-a-cpu-only:
|
||||
needs: [check-changes, call-gate]
|
||||
if: |
|
||||
@@ -976,6 +980,11 @@ jobs:
|
||||
fi
|
||||
python3 run_suite.py --hw cuda --suite stage-b-test-small-1-gpu --auto-partition-id ${{ matrix.partition }} --auto-partition-size 8 $CONTINUE_ON_ERROR_FLAG
|
||||
|
||||
- uses: ./.github/actions/upload-cuda-coredumps
|
||||
if: always()
|
||||
with:
|
||||
artifact-suffix: ${{ matrix.partition }}
|
||||
|
||||
# Runs on H100 (80GB, SM90) - tests that don't pass on 5090 (FA3, FP8, high VRAM, etc.)
|
||||
stage-b-test-large-1-gpu:
|
||||
needs: [check-changes, call-gate, wait-for-stage-a, sgl-kernel-build-wheels]
|
||||
@@ -1027,6 +1036,11 @@ jobs:
|
||||
fi
|
||||
python3 run_suite.py --hw cuda --suite stage-b-test-large-1-gpu --auto-partition-id ${{ matrix.partition }} --auto-partition-size 14 --timeout-per-file 1800 $CONTINUE_ON_ERROR_FLAG
|
||||
|
||||
- uses: ./.github/actions/upload-cuda-coredumps
|
||||
if: always()
|
||||
with:
|
||||
artifact-suffix: ${{ matrix.partition }}
|
||||
|
||||
stage-b-test-large-2-gpu:
|
||||
needs: [check-changes, call-gate, wait-for-stage-a, sgl-kernel-build-wheels]
|
||||
if: |
|
||||
@@ -1079,6 +1093,11 @@ jobs:
|
||||
fi
|
||||
python3 run_suite.py --hw cuda --suite stage-b-test-large-2-gpu --auto-partition-id ${{ matrix.partition }} --auto-partition-size 4 $CONTINUE_ON_ERROR_FLAG
|
||||
|
||||
- uses: ./.github/actions/upload-cuda-coredumps
|
||||
if: always()
|
||||
with:
|
||||
artifact-suffix: ${{ matrix.partition }}
|
||||
|
||||
stage-b-test-4-gpu-b200:
|
||||
needs: [check-changes, call-gate, wait-for-stage-a, sgl-kernel-build-wheels]
|
||||
if: |
|
||||
@@ -1132,6 +1151,9 @@ jobs:
|
||||
run: |
|
||||
IS_BLACKWELL=1 python3 -m pytest -q python/sglang/jit_kernel/tests/test_flash_attention_4.py
|
||||
|
||||
- uses: ./.github/actions/upload-cuda-coredumps
|
||||
if: always()
|
||||
|
||||
multimodal-gen-test-1-gpu:
|
||||
needs: [check-changes, call-gate, sgl-kernel-build-wheels]
|
||||
if: |
|
||||
@@ -1184,6 +1206,10 @@ jobs:
|
||||
--total-partitions 2 \
|
||||
$CONTINUE_ON_ERROR_FLAG
|
||||
|
||||
- uses: ./.github/actions/upload-cuda-coredumps
|
||||
if: always()
|
||||
with:
|
||||
artifact-suffix: ${{ matrix.part }}
|
||||
|
||||
multimodal-gen-test-2-gpu:
|
||||
needs: [check-changes, call-gate, sgl-kernel-build-wheels]
|
||||
@@ -1238,6 +1264,11 @@ jobs:
|
||||
--total-partitions 2 \
|
||||
$CONTINUE_ON_ERROR_FLAG
|
||||
|
||||
- uses: ./.github/actions/upload-cuda-coredumps
|
||||
if: always()
|
||||
with:
|
||||
artifact-suffix: ${{ matrix.part }}
|
||||
|
||||
stage-c-test-4-gpu-h100:
|
||||
needs: [check-changes, call-gate, wait-for-stage-b]
|
||||
if: |
|
||||
@@ -1287,6 +1318,11 @@ jobs:
|
||||
fi
|
||||
python3 run_suite.py --hw cuda --suite stage-c-test-4-gpu-h100 --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 $CONTINUE_ON_ERROR_FLAG
|
||||
|
||||
- uses: ./.github/actions/upload-cuda-coredumps
|
||||
if: always()
|
||||
with:
|
||||
artifact-suffix: ${{ matrix.part }}
|
||||
|
||||
stage-c-test-8-gpu-h200:
|
||||
needs: [check-changes, call-gate, wait-for-stage-b]
|
||||
if: |
|
||||
@@ -1342,6 +1378,11 @@ jobs:
|
||||
fi
|
||||
python3 run_suite.py --hw cuda --suite stage-c-test-8-gpu-h200 --auto-partition-id ${{ matrix.part }} --auto-partition-size 4 $CONTINUE_ON_ERROR_FLAG
|
||||
|
||||
- uses: ./.github/actions/upload-cuda-coredumps
|
||||
if: always()
|
||||
with:
|
||||
artifact-suffix: ${{ matrix.part }}
|
||||
|
||||
stage-c-test-8-gpu-h20:
|
||||
needs: [check-changes, call-gate, wait-for-stage-b]
|
||||
if: |
|
||||
@@ -1392,6 +1433,11 @@ jobs:
|
||||
fi
|
||||
python3 run_suite.py --hw cuda --suite stage-c-test-8-gpu-h20 --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 $CONTINUE_ON_ERROR_FLAG
|
||||
|
||||
- uses: ./.github/actions/upload-cuda-coredumps
|
||||
if: always()
|
||||
with:
|
||||
artifact-suffix: ${{ matrix.part }}
|
||||
|
||||
stage-c-test-deepep-4-gpu:
|
||||
needs: [check-changes, call-gate, wait-for-stage-b]
|
||||
if: |
|
||||
@@ -1437,6 +1483,9 @@ jobs:
|
||||
fi
|
||||
python3 run_suite.py --hw cuda --suite stage-c-test-deepep-4-gpu $CONTINUE_ON_ERROR_FLAG
|
||||
|
||||
- uses: ./.github/actions/upload-cuda-coredumps
|
||||
if: always()
|
||||
|
||||
stage-c-test-deepep-8-gpu-h200:
|
||||
needs: [check-changes, call-gate, wait-for-stage-b]
|
||||
if: |
|
||||
@@ -1482,6 +1531,9 @@ jobs:
|
||||
fi
|
||||
python3 run_suite.py --hw cuda --suite stage-c-test-deepep-8-gpu-h200 $CONTINUE_ON_ERROR_FLAG
|
||||
|
||||
- uses: ./.github/actions/upload-cuda-coredumps
|
||||
if: always()
|
||||
|
||||
stage-c-test-4-gpu-b200:
|
||||
needs: [check-changes, call-gate, wait-for-stage-b]
|
||||
if: |
|
||||
@@ -1532,6 +1584,11 @@ jobs:
|
||||
fi
|
||||
IS_BLACKWELL=1 python3 run_suite.py --hw cuda --suite stage-c-test-4-gpu-b200 --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --timeout-per-file 1800 $CONTINUE_ON_ERROR_FLAG
|
||||
|
||||
- uses: ./.github/actions/upload-cuda-coredumps
|
||||
if: always()
|
||||
with:
|
||||
artifact-suffix: ${{ matrix.part }}
|
||||
|
||||
stage-c-test-4-gpu-gb200:
|
||||
needs: [check-changes, call-gate, wait-for-stage-b, sgl-kernel-build-wheels-arm]
|
||||
if: |
|
||||
@@ -1579,6 +1636,9 @@ jobs:
|
||||
fi
|
||||
python3 run_suite.py --hw cuda --suite stage-c-test-4-gpu-gb200 --timeout-per-file 3600 $CONTINUE_ON_ERROR_FLAG
|
||||
|
||||
- uses: ./.github/actions/upload-cuda-coredumps
|
||||
if: always()
|
||||
|
||||
pr-test-finish:
|
||||
needs:
|
||||
[
|
||||
|
||||
Reference in New Issue
Block a user