feat: add cuda core dump CI warpper (#18909)
This commit is contained in:
Liangsheng Yin
47
.github/workflows/nightly-test-nvidia.yml
vendored
47
.github/workflows/nightly-test-nvidia.yml
vendored
@@ -44,6 +44,7 @@ concurrency:
|
||||
|
||||
env:
|
||||
SGLANG_IS_IN_CI: true
|
||||
SGLANG_CUDA_COREDUMP: "1"
|
||||
HF_HUB_DOWNLOAD_TIMEOUT: 300
|
||||
HF_HUB_ETAG_TIMEOUT: 300
|
||||
|
||||
@@ -68,6 +69,9 @@ jobs:
|
||||
cd test
|
||||
python3 run_suite.py --hw cuda --suite nightly-1-gpu --nightly --continue-on-error
|
||||
|
||||
- uses: ./.github/actions/upload-cuda-coredumps
|
||||
if: always()
|
||||
|
||||
# General tests - 4 GPU H100
|
||||
nightly-test-general-4-gpu-h100:
|
||||
if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-general-4-gpu-h100')
|
||||
@@ -88,6 +92,9 @@ jobs:
|
||||
cd test
|
||||
python3 run_suite.py --hw cuda --suite nightly-4-gpu --nightly --continue-on-error
|
||||
|
||||
- uses: ./.github/actions/upload-cuda-coredumps
|
||||
if: always()
|
||||
|
||||
# General tests - 8 GPU H200
|
||||
nightly-test-general-8-gpu-h200:
|
||||
if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-general-8-gpu-h200')
|
||||
@@ -167,6 +174,11 @@ jobs:
|
||||
retention-days: 5
|
||||
if-no-files-found: ignore
|
||||
|
||||
- uses: ./.github/actions/upload-cuda-coredumps
|
||||
if: always()
|
||||
with:
|
||||
artifact-suffix: ${{ matrix.partition }}
|
||||
|
||||
# General tests - 8 GPU H20
|
||||
nightly-test-general-8-gpu-h20:
|
||||
if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-general-8-gpu-h20')
|
||||
@@ -191,6 +203,9 @@ jobs:
|
||||
cd test
|
||||
python3 run_suite.py --hw cuda --suite nightly-8-gpu-h20 --nightly --continue-on-error
|
||||
|
||||
- uses: ./.github/actions/upload-cuda-coredumps
|
||||
if: always()
|
||||
|
||||
# General tests - 8 GPU B200
|
||||
nightly-test-general-8-gpu-b200:
|
||||
if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-general-8-gpu-b200')
|
||||
@@ -259,6 +274,11 @@ jobs:
|
||||
retention-days: 5
|
||||
if-no-files-found: ignore
|
||||
|
||||
- uses: ./.github/actions/upload-cuda-coredumps
|
||||
if: always()
|
||||
with:
|
||||
artifact-suffix: ${{ matrix.partition }}
|
||||
|
||||
# Text model accuracy tests
|
||||
nightly-test-text-accuracy-2-gpu-runner:
|
||||
if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-text-accuracy-2-gpu-runner')
|
||||
@@ -279,6 +299,9 @@ jobs:
|
||||
cd test
|
||||
python3 run_suite.py --hw cuda --suite nightly-eval-text-2-gpu --nightly --continue-on-error --timeout-per-file 4500
|
||||
|
||||
- uses: ./.github/actions/upload-cuda-coredumps
|
||||
if: always()
|
||||
|
||||
# Text model performance tests
|
||||
nightly-test-text-perf-2-gpu-runner:
|
||||
if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-text-perf-2-gpu-runner')
|
||||
@@ -312,6 +335,9 @@ jobs:
|
||||
run: |
|
||||
python3 scripts/ci/utils/publish_traces.py --traces-dir test/performance_profiles_text_models
|
||||
|
||||
- uses: ./.github/actions/upload-cuda-coredumps
|
||||
if: always()
|
||||
|
||||
# VLM accuracy tests
|
||||
nightly-test-vlm-accuracy-2-gpu-runner:
|
||||
if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-vlm-accuracy-2-gpu-runner')
|
||||
@@ -332,6 +358,9 @@ jobs:
|
||||
cd test
|
||||
python3 run_suite.py --hw cuda --suite nightly-eval-vlm-2-gpu --nightly --continue-on-error --timeout-per-file 9000
|
||||
|
||||
- uses: ./.github/actions/upload-cuda-coredumps
|
||||
if: always()
|
||||
|
||||
# VLM performance tests
|
||||
nightly-test-vlm-perf-2-gpu-runner:
|
||||
if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-vlm-perf-2-gpu-runner')
|
||||
@@ -365,6 +394,9 @@ jobs:
|
||||
run: |
|
||||
python3 scripts/ci/utils/publish_traces.py --traces-dir test/performance_profiles_vlms
|
||||
|
||||
- uses: ./.github/actions/upload-cuda-coredumps
|
||||
if: always()
|
||||
|
||||
# diffusion performance tests
|
||||
nightly-test-multimodal-server-1-gpu:
|
||||
if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-multimodal-server-1-gpu')
|
||||
@@ -417,6 +449,10 @@ jobs:
|
||||
retention-days: 90
|
||||
if-no-files-found: ignore
|
||||
|
||||
- uses: ./.github/actions/upload-cuda-coredumps
|
||||
if: always()
|
||||
with:
|
||||
artifact-suffix: ${{ matrix.part }}
|
||||
|
||||
nightly-test-multimodal-server-2-gpu:
|
||||
if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-multimodal-server-2-gpu')
|
||||
@@ -469,6 +505,11 @@ jobs:
|
||||
retention-days: 90
|
||||
if-no-files-found: ignore
|
||||
|
||||
- uses: ./.github/actions/upload-cuda-coredumps
|
||||
if: always()
|
||||
with:
|
||||
artifact-suffix: ${{ matrix.part }}
|
||||
|
||||
# B200 Performance tests - 4 GPU
|
||||
nightly-test-perf-4-gpu-b200:
|
||||
if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-perf-4-gpu-b200')
|
||||
@@ -489,6 +530,9 @@ jobs:
|
||||
cd test
|
||||
python3 run_suite.py --hw cuda --suite nightly-4-gpu-b200 --nightly --continue-on-error --timeout-per-file 12000
|
||||
|
||||
- uses: ./.github/actions/upload-cuda-coredumps
|
||||
if: always()
|
||||
|
||||
# Specialized B200 tests - 8 GPU, for specific backends and configs
|
||||
nightly-test-specialized-8-gpu-b200:
|
||||
if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-perf-8-gpu-b200')
|
||||
@@ -513,6 +557,9 @@ jobs:
|
||||
cd test
|
||||
python3 run_suite.py --hw cuda --suite nightly-8-gpu-b200 --nightly --continue-on-error --timeout-per-file 2400
|
||||
|
||||
- uses: ./.github/actions/upload-cuda-coredumps
|
||||
if: always()
|
||||
|
||||
# Consolidate performance metrics from all jobs
|
||||
consolidate-metrics:
|
||||
if: github.repository == 'sgl-project/sglang' && always()
|
||||
|
||||
60
.github/workflows/pr-test.yml
vendored
60
.github/workflows/pr-test.yml
vendored
@@ -63,6 +63,7 @@ concurrency:
|
||||
|
||||
env:
|
||||
SGLANG_IS_IN_CI: true
|
||||
SGLANG_CUDA_COREDUMP: "1"
|
||||
SGLANG_JIT_DEEPGEMM_FAST_WARMUP: true
|
||||
|
||||
permissions:
|
||||
@@ -874,6 +875,9 @@ jobs:
|
||||
# temporarily put backend-independent cpu tests here
|
||||
python3 run_suite.py --hw cpu --suite default $CONTINUE_ON_ERROR_FLAG
|
||||
|
||||
- uses: ./.github/actions/upload-cuda-coredumps
|
||||
if: always()
|
||||
|
||||
stage-a-cpu-only:
|
||||
needs: [check-changes, call-gate]
|
||||
if: |
|
||||
@@ -976,6 +980,11 @@ jobs:
|
||||
fi
|
||||
python3 run_suite.py --hw cuda --suite stage-b-test-small-1-gpu --auto-partition-id ${{ matrix.partition }} --auto-partition-size 8 $CONTINUE_ON_ERROR_FLAG
|
||||
|
||||
- uses: ./.github/actions/upload-cuda-coredumps
|
||||
if: always()
|
||||
with:
|
||||
artifact-suffix: ${{ matrix.partition }}
|
||||
|
||||
# Runs on H100 (80GB, SM90) - tests that don't pass on 5090 (FA3, FP8, high VRAM, etc.)
|
||||
stage-b-test-large-1-gpu:
|
||||
needs: [check-changes, call-gate, wait-for-stage-a, sgl-kernel-build-wheels]
|
||||
@@ -1027,6 +1036,11 @@ jobs:
|
||||
fi
|
||||
python3 run_suite.py --hw cuda --suite stage-b-test-large-1-gpu --auto-partition-id ${{ matrix.partition }} --auto-partition-size 14 --timeout-per-file 1800 $CONTINUE_ON_ERROR_FLAG
|
||||
|
||||
- uses: ./.github/actions/upload-cuda-coredumps
|
||||
if: always()
|
||||
with:
|
||||
artifact-suffix: ${{ matrix.partition }}
|
||||
|
||||
stage-b-test-large-2-gpu:
|
||||
needs: [check-changes, call-gate, wait-for-stage-a, sgl-kernel-build-wheels]
|
||||
if: |
|
||||
@@ -1079,6 +1093,11 @@ jobs:
|
||||
fi
|
||||
python3 run_suite.py --hw cuda --suite stage-b-test-large-2-gpu --auto-partition-id ${{ matrix.partition }} --auto-partition-size 4 $CONTINUE_ON_ERROR_FLAG
|
||||
|
||||
- uses: ./.github/actions/upload-cuda-coredumps
|
||||
if: always()
|
||||
with:
|
||||
artifact-suffix: ${{ matrix.partition }}
|
||||
|
||||
stage-b-test-4-gpu-b200:
|
||||
needs: [check-changes, call-gate, wait-for-stage-a, sgl-kernel-build-wheels]
|
||||
if: |
|
||||
@@ -1132,6 +1151,9 @@ jobs:
|
||||
run: |
|
||||
IS_BLACKWELL=1 python3 -m pytest -q python/sglang/jit_kernel/tests/test_flash_attention_4.py
|
||||
|
||||
- uses: ./.github/actions/upload-cuda-coredumps
|
||||
if: always()
|
||||
|
||||
multimodal-gen-test-1-gpu:
|
||||
needs: [check-changes, call-gate, sgl-kernel-build-wheels]
|
||||
if: |
|
||||
@@ -1184,6 +1206,10 @@ jobs:
|
||||
--total-partitions 2 \
|
||||
$CONTINUE_ON_ERROR_FLAG
|
||||
|
||||
- uses: ./.github/actions/upload-cuda-coredumps
|
||||
if: always()
|
||||
with:
|
||||
artifact-suffix: ${{ matrix.part }}
|
||||
|
||||
multimodal-gen-test-2-gpu:
|
||||
needs: [check-changes, call-gate, sgl-kernel-build-wheels]
|
||||
@@ -1238,6 +1264,11 @@ jobs:
|
||||
--total-partitions 2 \
|
||||
$CONTINUE_ON_ERROR_FLAG
|
||||
|
||||
- uses: ./.github/actions/upload-cuda-coredumps
|
||||
if: always()
|
||||
with:
|
||||
artifact-suffix: ${{ matrix.part }}
|
||||
|
||||
stage-c-test-4-gpu-h100:
|
||||
needs: [check-changes, call-gate, wait-for-stage-b]
|
||||
if: |
|
||||
@@ -1287,6 +1318,11 @@ jobs:
|
||||
fi
|
||||
python3 run_suite.py --hw cuda --suite stage-c-test-4-gpu-h100 --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 $CONTINUE_ON_ERROR_FLAG
|
||||
|
||||
- uses: ./.github/actions/upload-cuda-coredumps
|
||||
if: always()
|
||||
with:
|
||||
artifact-suffix: ${{ matrix.part }}
|
||||
|
||||
stage-c-test-8-gpu-h200:
|
||||
needs: [check-changes, call-gate, wait-for-stage-b]
|
||||
if: |
|
||||
@@ -1342,6 +1378,11 @@ jobs:
|
||||
fi
|
||||
python3 run_suite.py --hw cuda --suite stage-c-test-8-gpu-h200 --auto-partition-id ${{ matrix.part }} --auto-partition-size 4 $CONTINUE_ON_ERROR_FLAG
|
||||
|
||||
- uses: ./.github/actions/upload-cuda-coredumps
|
||||
if: always()
|
||||
with:
|
||||
artifact-suffix: ${{ matrix.part }}
|
||||
|
||||
stage-c-test-8-gpu-h20:
|
||||
needs: [check-changes, call-gate, wait-for-stage-b]
|
||||
if: |
|
||||
@@ -1392,6 +1433,11 @@ jobs:
|
||||
fi
|
||||
python3 run_suite.py --hw cuda --suite stage-c-test-8-gpu-h20 --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 $CONTINUE_ON_ERROR_FLAG
|
||||
|
||||
- uses: ./.github/actions/upload-cuda-coredumps
|
||||
if: always()
|
||||
with:
|
||||
artifact-suffix: ${{ matrix.part }}
|
||||
|
||||
stage-c-test-deepep-4-gpu:
|
||||
needs: [check-changes, call-gate, wait-for-stage-b]
|
||||
if: |
|
||||
@@ -1437,6 +1483,9 @@ jobs:
|
||||
fi
|
||||
python3 run_suite.py --hw cuda --suite stage-c-test-deepep-4-gpu $CONTINUE_ON_ERROR_FLAG
|
||||
|
||||
- uses: ./.github/actions/upload-cuda-coredumps
|
||||
if: always()
|
||||
|
||||
stage-c-test-deepep-8-gpu-h200:
|
||||
needs: [check-changes, call-gate, wait-for-stage-b]
|
||||
if: |
|
||||
@@ -1482,6 +1531,9 @@ jobs:
|
||||
fi
|
||||
python3 run_suite.py --hw cuda --suite stage-c-test-deepep-8-gpu-h200 $CONTINUE_ON_ERROR_FLAG
|
||||
|
||||
- uses: ./.github/actions/upload-cuda-coredumps
|
||||
if: always()
|
||||
|
||||
stage-c-test-4-gpu-b200:
|
||||
needs: [check-changes, call-gate, wait-for-stage-b]
|
||||
if: |
|
||||
@@ -1532,6 +1584,11 @@ jobs:
|
||||
fi
|
||||
IS_BLACKWELL=1 python3 run_suite.py --hw cuda --suite stage-c-test-4-gpu-b200 --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --timeout-per-file 1800 $CONTINUE_ON_ERROR_FLAG
|
||||
|
||||
- uses: ./.github/actions/upload-cuda-coredumps
|
||||
if: always()
|
||||
with:
|
||||
artifact-suffix: ${{ matrix.part }}
|
||||
|
||||
stage-c-test-4-gpu-gb200:
|
||||
needs: [check-changes, call-gate, wait-for-stage-b, sgl-kernel-build-wheels-arm]
|
||||
if: |
|
||||
@@ -1579,6 +1636,9 @@ jobs:
|
||||
fi
|
||||
python3 run_suite.py --hw cuda --suite stage-c-test-4-gpu-gb200 --timeout-per-file 3600 $CONTINUE_ON_ERROR_FLAG
|
||||
|
||||
- uses: ./.github/actions/upload-cuda-coredumps
|
||||
if: always()
|
||||
|
||||
pr-test-finish:
|
||||
needs:
|
||||
[
|
||||
|
||||
Reference in New Issue
Block a user