From 83a475e8d7de58c032d3e13551208cc1d5549ea2 Mon Sep 17 00:00:00 2001 From: Liangsheng Yin Date: Tue, 17 Feb 2026 14:49:26 -0800 Subject: [PATCH] feat: add cuda core dump CI warpper (#18909) --- .../actions/upload-cuda-coredumps/action.yml | 27 ++++++ .github/workflows/nightly-test-nvidia.yml | 47 ++++++++++ .github/workflows/pr-test.yml | 60 ++++++++++++ .../sglang/srt/debug_utils/cuda_coredump.py | 93 +++++++++++++++++++ python/sglang/srt/environ.py | 7 ++ python/sglang/test/ci/ci_utils.py | 11 ++- .../debug_utils/test_cuda_coredump_smoke.py | 29 ++++++ 7 files changed, 272 insertions(+), 2 deletions(-) create mode 100644 .github/actions/upload-cuda-coredumps/action.yml create mode 100644 python/sglang/srt/debug_utils/cuda_coredump.py create mode 100644 test/registered/debug_utils/test_cuda_coredump_smoke.py diff --git a/.github/actions/upload-cuda-coredumps/action.yml b/.github/actions/upload-cuda-coredumps/action.yml new file mode 100644 index 000000000..0e9fdde27 --- /dev/null +++ b/.github/actions/upload-cuda-coredumps/action.yml @@ -0,0 +1,27 @@ +name: Upload CUDA Coredumps +description: Upload CUDA coredump files as artifacts and clean up the directory. + +inputs: + artifact-suffix: + description: Suffix appended to the artifact name (e.g. matrix partition id) + required: false + default: "" + retention-days: + description: Number of days to retain the artifact + required: false + default: "7" + +runs: + using: composite + steps: + - name: Upload CUDA coredumps + uses: actions/upload-artifact@v4 + with: + name: cuda-coredumps-${{ github.job }}${{ inputs.artifact-suffix && format('-{0}', inputs.artifact-suffix) }} + path: ${{ env.SGLANG_CUDA_COREDUMP_DIR || '/tmp/sglang_cuda_coredumps' }}/ + retention-days: ${{ inputs.retention-days }} + if-no-files-found: ignore + + - name: Cleanup CUDA coredumps + shell: bash + run: rm -rf "${{ env.SGLANG_CUDA_COREDUMP_DIR || '/tmp/sglang_cuda_coredumps' }}" diff --git a/.github/workflows/nightly-test-nvidia.yml b/.github/workflows/nightly-test-nvidia.yml index 3a4472c42..544276399 100644 --- a/.github/workflows/nightly-test-nvidia.yml +++ b/.github/workflows/nightly-test-nvidia.yml @@ -44,6 +44,7 @@ concurrency: env: SGLANG_IS_IN_CI: true + SGLANG_CUDA_COREDUMP: "1" HF_HUB_DOWNLOAD_TIMEOUT: 300 HF_HUB_ETAG_TIMEOUT: 300 @@ -68,6 +69,9 @@ jobs: cd test python3 run_suite.py --hw cuda --suite nightly-1-gpu --nightly --continue-on-error + - uses: ./.github/actions/upload-cuda-coredumps + if: always() + # General tests - 4 GPU H100 nightly-test-general-4-gpu-h100: if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-general-4-gpu-h100') @@ -88,6 +92,9 @@ jobs: cd test python3 run_suite.py --hw cuda --suite nightly-4-gpu --nightly --continue-on-error + - uses: ./.github/actions/upload-cuda-coredumps + if: always() + # General tests - 8 GPU H200 nightly-test-general-8-gpu-h200: if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-general-8-gpu-h200') @@ -167,6 +174,11 @@ jobs: retention-days: 5 if-no-files-found: ignore + - uses: ./.github/actions/upload-cuda-coredumps + if: always() + with: + artifact-suffix: ${{ matrix.partition }} + # General tests - 8 GPU H20 nightly-test-general-8-gpu-h20: if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-general-8-gpu-h20') @@ -191,6 +203,9 @@ jobs: cd test python3 run_suite.py --hw cuda --suite nightly-8-gpu-h20 --nightly --continue-on-error + - uses: ./.github/actions/upload-cuda-coredumps + if: always() + # General tests - 8 GPU B200 nightly-test-general-8-gpu-b200: if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-general-8-gpu-b200') @@ -259,6 +274,11 @@ jobs: retention-days: 5 if-no-files-found: ignore + - uses: ./.github/actions/upload-cuda-coredumps + if: always() + with: + artifact-suffix: ${{ matrix.partition }} + # Text model accuracy tests nightly-test-text-accuracy-2-gpu-runner: if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-text-accuracy-2-gpu-runner') @@ -279,6 +299,9 @@ jobs: cd test python3 run_suite.py --hw cuda --suite nightly-eval-text-2-gpu --nightly --continue-on-error --timeout-per-file 4500 + - uses: ./.github/actions/upload-cuda-coredumps + if: always() + # Text model performance tests nightly-test-text-perf-2-gpu-runner: if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-text-perf-2-gpu-runner') @@ -312,6 +335,9 @@ jobs: run: | python3 scripts/ci/utils/publish_traces.py --traces-dir test/performance_profiles_text_models + - uses: ./.github/actions/upload-cuda-coredumps + if: always() + # VLM accuracy tests nightly-test-vlm-accuracy-2-gpu-runner: if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-vlm-accuracy-2-gpu-runner') @@ -332,6 +358,9 @@ jobs: cd test python3 run_suite.py --hw cuda --suite nightly-eval-vlm-2-gpu --nightly --continue-on-error --timeout-per-file 9000 + - uses: ./.github/actions/upload-cuda-coredumps + if: always() + # VLM performance tests nightly-test-vlm-perf-2-gpu-runner: if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-vlm-perf-2-gpu-runner') @@ -365,6 +394,9 @@ jobs: run: | python3 scripts/ci/utils/publish_traces.py --traces-dir test/performance_profiles_vlms + - uses: ./.github/actions/upload-cuda-coredumps + if: always() + # diffusion performance tests nightly-test-multimodal-server-1-gpu: if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-multimodal-server-1-gpu') @@ -417,6 +449,10 @@ jobs: retention-days: 90 if-no-files-found: ignore + - uses: ./.github/actions/upload-cuda-coredumps + if: always() + with: + artifact-suffix: ${{ matrix.part }} nightly-test-multimodal-server-2-gpu: if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-multimodal-server-2-gpu') @@ -469,6 +505,11 @@ jobs: retention-days: 90 if-no-files-found: ignore + - uses: ./.github/actions/upload-cuda-coredumps + if: always() + with: + artifact-suffix: ${{ matrix.part }} + # B200 Performance tests - 4 GPU nightly-test-perf-4-gpu-b200: if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-perf-4-gpu-b200') @@ -489,6 +530,9 @@ jobs: cd test python3 run_suite.py --hw cuda --suite nightly-4-gpu-b200 --nightly --continue-on-error --timeout-per-file 12000 + - uses: ./.github/actions/upload-cuda-coredumps + if: always() + # Specialized B200 tests - 8 GPU, for specific backends and configs nightly-test-specialized-8-gpu-b200: if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-perf-8-gpu-b200') @@ -513,6 +557,9 @@ jobs: cd test python3 run_suite.py --hw cuda --suite nightly-8-gpu-b200 --nightly --continue-on-error --timeout-per-file 2400 + - uses: ./.github/actions/upload-cuda-coredumps + if: always() + # Consolidate performance metrics from all jobs consolidate-metrics: if: github.repository == 'sgl-project/sglang' && always() diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index 109a3d02f..3719c1779 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -63,6 +63,7 @@ concurrency: env: SGLANG_IS_IN_CI: true + SGLANG_CUDA_COREDUMP: "1" SGLANG_JIT_DEEPGEMM_FAST_WARMUP: true permissions: @@ -874,6 +875,9 @@ jobs: # temporarily put backend-independent cpu tests here python3 run_suite.py --hw cpu --suite default $CONTINUE_ON_ERROR_FLAG + - uses: ./.github/actions/upload-cuda-coredumps + if: always() + stage-a-cpu-only: needs: [check-changes, call-gate] if: | @@ -976,6 +980,11 @@ jobs: fi python3 run_suite.py --hw cuda --suite stage-b-test-small-1-gpu --auto-partition-id ${{ matrix.partition }} --auto-partition-size 8 $CONTINUE_ON_ERROR_FLAG + - uses: ./.github/actions/upload-cuda-coredumps + if: always() + with: + artifact-suffix: ${{ matrix.partition }} + # Runs on H100 (80GB, SM90) - tests that don't pass on 5090 (FA3, FP8, high VRAM, etc.) stage-b-test-large-1-gpu: needs: [check-changes, call-gate, wait-for-stage-a, sgl-kernel-build-wheels] @@ -1027,6 +1036,11 @@ jobs: fi python3 run_suite.py --hw cuda --suite stage-b-test-large-1-gpu --auto-partition-id ${{ matrix.partition }} --auto-partition-size 14 --timeout-per-file 1800 $CONTINUE_ON_ERROR_FLAG + - uses: ./.github/actions/upload-cuda-coredumps + if: always() + with: + artifact-suffix: ${{ matrix.partition }} + stage-b-test-large-2-gpu: needs: [check-changes, call-gate, wait-for-stage-a, sgl-kernel-build-wheels] if: | @@ -1079,6 +1093,11 @@ jobs: fi python3 run_suite.py --hw cuda --suite stage-b-test-large-2-gpu --auto-partition-id ${{ matrix.partition }} --auto-partition-size 4 $CONTINUE_ON_ERROR_FLAG + - uses: ./.github/actions/upload-cuda-coredumps + if: always() + with: + artifact-suffix: ${{ matrix.partition }} + stage-b-test-4-gpu-b200: needs: [check-changes, call-gate, wait-for-stage-a, sgl-kernel-build-wheels] if: | @@ -1132,6 +1151,9 @@ jobs: run: | IS_BLACKWELL=1 python3 -m pytest -q python/sglang/jit_kernel/tests/test_flash_attention_4.py + - uses: ./.github/actions/upload-cuda-coredumps + if: always() + multimodal-gen-test-1-gpu: needs: [check-changes, call-gate, sgl-kernel-build-wheels] if: | @@ -1184,6 +1206,10 @@ jobs: --total-partitions 2 \ $CONTINUE_ON_ERROR_FLAG + - uses: ./.github/actions/upload-cuda-coredumps + if: always() + with: + artifact-suffix: ${{ matrix.part }} multimodal-gen-test-2-gpu: needs: [check-changes, call-gate, sgl-kernel-build-wheels] @@ -1238,6 +1264,11 @@ jobs: --total-partitions 2 \ $CONTINUE_ON_ERROR_FLAG + - uses: ./.github/actions/upload-cuda-coredumps + if: always() + with: + artifact-suffix: ${{ matrix.part }} + stage-c-test-4-gpu-h100: needs: [check-changes, call-gate, wait-for-stage-b] if: | @@ -1287,6 +1318,11 @@ jobs: fi python3 run_suite.py --hw cuda --suite stage-c-test-4-gpu-h100 --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 $CONTINUE_ON_ERROR_FLAG + - uses: ./.github/actions/upload-cuda-coredumps + if: always() + with: + artifact-suffix: ${{ matrix.part }} + stage-c-test-8-gpu-h200: needs: [check-changes, call-gate, wait-for-stage-b] if: | @@ -1342,6 +1378,11 @@ jobs: fi python3 run_suite.py --hw cuda --suite stage-c-test-8-gpu-h200 --auto-partition-id ${{ matrix.part }} --auto-partition-size 4 $CONTINUE_ON_ERROR_FLAG + - uses: ./.github/actions/upload-cuda-coredumps + if: always() + with: + artifact-suffix: ${{ matrix.part }} + stage-c-test-8-gpu-h20: needs: [check-changes, call-gate, wait-for-stage-b] if: | @@ -1392,6 +1433,11 @@ jobs: fi python3 run_suite.py --hw cuda --suite stage-c-test-8-gpu-h20 --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 $CONTINUE_ON_ERROR_FLAG + - uses: ./.github/actions/upload-cuda-coredumps + if: always() + with: + artifact-suffix: ${{ matrix.part }} + stage-c-test-deepep-4-gpu: needs: [check-changes, call-gate, wait-for-stage-b] if: | @@ -1437,6 +1483,9 @@ jobs: fi python3 run_suite.py --hw cuda --suite stage-c-test-deepep-4-gpu $CONTINUE_ON_ERROR_FLAG + - uses: ./.github/actions/upload-cuda-coredumps + if: always() + stage-c-test-deepep-8-gpu-h200: needs: [check-changes, call-gate, wait-for-stage-b] if: | @@ -1482,6 +1531,9 @@ jobs: fi python3 run_suite.py --hw cuda --suite stage-c-test-deepep-8-gpu-h200 $CONTINUE_ON_ERROR_FLAG + - uses: ./.github/actions/upload-cuda-coredumps + if: always() + stage-c-test-4-gpu-b200: needs: [check-changes, call-gate, wait-for-stage-b] if: | @@ -1532,6 +1584,11 @@ jobs: fi IS_BLACKWELL=1 python3 run_suite.py --hw cuda --suite stage-c-test-4-gpu-b200 --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --timeout-per-file 1800 $CONTINUE_ON_ERROR_FLAG + - uses: ./.github/actions/upload-cuda-coredumps + if: always() + with: + artifact-suffix: ${{ matrix.part }} + stage-c-test-4-gpu-gb200: needs: [check-changes, call-gate, wait-for-stage-b, sgl-kernel-build-wheels-arm] if: | @@ -1579,6 +1636,9 @@ jobs: fi python3 run_suite.py --hw cuda --suite stage-c-test-4-gpu-gb200 --timeout-per-file 3600 $CONTINUE_ON_ERROR_FLAG + - uses: ./.github/actions/upload-cuda-coredumps + if: always() + pr-test-finish: needs: [ diff --git a/python/sglang/srt/debug_utils/cuda_coredump.py b/python/sglang/srt/debug_utils/cuda_coredump.py new file mode 100644 index 000000000..535e70907 --- /dev/null +++ b/python/sglang/srt/debug_utils/cuda_coredump.py @@ -0,0 +1,93 @@ +"""CUDA coredump helpers. + +When SGLANG_CUDA_COREDUMP=1, this module injects CUDA coredump environment +variables into the current process so that GPU exceptions (e.g. illegal +memory access) produce lightweight coredump files for post-mortem analysis +with cuda-gdb. + +The injection happens at module import time via _inject_env() on a +best-effort basis. If any CUDA_* variable is already present in the +environment (e.g. set by the user in the shell), injection is skipped for +that variable and a warning is logged. For strict guarantees, set the +CUDA_* env vars in the shell before launching Python. +""" + +import glob +import logging +import os + +from sglang.srt.environ import envs + +logger = logging.getLogger(__name__) + +_CUDA_COREDUMP_FLAGS = ( + "skip_nonrelocated_elf_images,skip_global_memory," + "skip_shared_memory,skip_local_memory,skip_constbank_memory" +) + + +def is_enabled() -> bool: + return envs.SGLANG_CUDA_COREDUMP.get() + + +def get_dump_dir() -> str: + return envs.SGLANG_CUDA_COREDUMP_DIR.get() + + +def _inject_env(): + """Inject CUDA coredump environment variables into the current process. + If a CUDA_* variable is already present, skip it and log a warning.""" + dump_dir = get_dump_dir() + os.makedirs(dump_dir, exist_ok=True) + + env_vars = { + "CUDA_ENABLE_COREDUMP_ON_EXCEPTION": "1", + "CUDA_COREDUMP_SHOW_PROGRESS": "1", + "CUDA_COREDUMP_GENERATION_FLAGS": _CUDA_COREDUMP_FLAGS, + "CUDA_COREDUMP_FILE": f"{dump_dir}/cuda_coredump_%h.%p.%t", + } + for key, value in env_vars.items(): + if key in os.environ: + logger.warning( + "CUDA coredump env var %s is already set to '%s', " + "skipping injection of '%s'.", + key, + os.environ[key], + value, + ) + else: + os.environ[key] = value + + +def cleanup_dump_dir(): + """Remove stale coredump files from the dump directory.""" + dump_dir = get_dump_dir() + for f in glob.glob(os.path.join(dump_dir, "cuda_coredump_*")): + os.remove(f) + + +def report(): + """Log any CUDA coredump files found after a test failure.""" + dump_dir = get_dump_dir() + coredump_files = glob.glob(os.path.join(dump_dir, "cuda_coredump_*")) + if not coredump_files: + return + + logger.info(f"\n{'='*60}") + logger.info(f"CUDA coredump(s) detected ({len(coredump_files)} file(s)):") + for f in coredump_files: + size_mb = os.path.getsize(f) / (1024 * 1024) + logger.info(f" {f} ({size_mb:.1f} MB)") + logger.info("Use cuda-gdb to analyze: cuda-gdb -c ") + + run_id = os.environ.get("GITHUB_RUN_ID") + if run_id: + repo = os.environ.get("GITHUB_REPOSITORY", "sgl-project/sglang") + logger.info(f"Download from CI: gh run download {run_id} --repo {repo}") + + logger.info(f"{'='*60}\n") + + +# Auto-inject CUDA coredump env vars at import time. +if is_enabled(): + _inject_env() diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index 593fd03b5..8293796a2 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -176,6 +176,8 @@ class Envs: # SGLang CI SGLANG_IS_IN_CI = EnvBool(False) SGLANG_IS_IN_CI_AMD = EnvBool(False) + SGLANG_CUDA_COREDUMP = EnvBool(False) + SGLANG_CUDA_COREDUMP_DIR = EnvStr("/tmp/sglang_cuda_coredumps") SGLANG_TEST_MAX_RETRY = EnvInt(None) # Constrained Decoding (Grammar) @@ -575,6 +577,11 @@ _warn_deprecated_env_to_cli_flag( "Please use '--prefill-delayer-token-usage-low-watermark' instead.", ) +# Import cuda_coredump to trigger auto-injection of CUDA env vars +# when SGLANG_CUDA_COREDUMP=1. Best-effort; for strict guarantees, +# set CUDA_* env vars in the shell before launching Python. +import sglang.srt.debug_utils.cuda_coredump # noqa: F401, E402 + def example_with_exit_stack(): # Use this style of context manager in unit test diff --git a/python/sglang/test/ci/ci_utils.py b/python/sglang/test/ci/ci_utils.py index 323910367..a17498e06 100644 --- a/python/sglang/test/ci/ci_utils.py +++ b/python/sglang/test/ci/ci_utils.py @@ -7,6 +7,7 @@ import time from dataclasses import dataclass from typing import Callable, List, Optional, Union +from sglang.srt.debug_utils import cuda_coredump from sglang.srt.utils.common import kill_process_tree from sglang.test.ci.ci_register import CIRegistry @@ -130,6 +131,10 @@ def run_unittest_files( max_attempts: Maximum number of attempts per file including initial run (default: 2). retry_wait_seconds: Seconds to wait between retries (default: 60). """ + coredump_enabled = cuda_coredump.is_enabled() + if coredump_enabled: + cuda_coredump.cleanup_dump_dir() + tic = time.perf_counter() success = True passed_tests = [] @@ -161,7 +166,6 @@ def run_unittest_files( ["python3", full_path], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - env=os.environ, text=True, errors="ignore", # Ignore non-UTF-8 bytes to prevent UnicodeDecodeError ) @@ -172,7 +176,7 @@ def run_unittest_files( process.wait() else: process = subprocess.Popen( - ["python3", full_path], stdout=None, stderr=None, env=os.environ + ["python3", full_path], stdout=None, stderr=None ) process.wait() @@ -258,6 +262,9 @@ def run_unittest_files( elapsed_total = time.perf_counter() - tic + if coredump_enabled and not success: + cuda_coredump.report() + if success: logger.info(f"Success. Time elapsed: {elapsed_total:.2f}s") else: diff --git a/test/registered/debug_utils/test_cuda_coredump_smoke.py b/test/registered/debug_utils/test_cuda_coredump_smoke.py new file mode 100644 index 000000000..09bde37a5 --- /dev/null +++ b/test/registered/debug_utils/test_cuda_coredump_smoke.py @@ -0,0 +1,29 @@ +"""Smoke test: intentionally trigger a CUDA illegal memory access +to verify the coredump collection pipeline works end-to-end. + +Manual use: python3 test/registered/debug_utils/test_cuda_coredump_smoke.py +""" + +import unittest + +import torch + +from sglang.test.ci.ci_register import register_cuda_ci + +register_cuda_ci( + est_time=10, + suite="stage-a-test-1", + disabled="Manual only: triggers intentional CUDA crash for coredump verification", +) + + +class TestCudaCoredumpSmoke(unittest.TestCase): + def test_trigger_illegal_memory_access(self): + x = torch.zeros(10, device="cuda") + y = torch.arange(10, device="cuda") + x[y * y] = 1 + torch.cuda.synchronize() + + +if __name__ == "__main__": + unittest.main()