feat: add cuda core dump CI warpper (#18909)

2026-02-17 14:49:26 -08:00
parent 9a7d6be567
commit 83a475e8d7
7 changed files with 272 additions and 2 deletions
--- a/.github/actions/upload-cuda-coredumps/action.yml
+++ b/.github/actions/upload-cuda-coredumps/action.yml
@@ -0,0 +1,27 @@
+name: Upload CUDA Coredumps
+description: Upload CUDA coredump files as artifacts and clean up the directory.
+
+inputs:
+  artifact-suffix:
+    description: Suffix appended to the artifact name (e.g. matrix partition id)
+    required: false
+    default: ""
+  retention-days:
+    description: Number of days to retain the artifact
+    required: false
+    default: "7"
+
+runs:
+  using: composite
+  steps:
+    - name: Upload CUDA coredumps
+      uses: actions/upload-artifact@v4
+      with:
+        name: cuda-coredumps-${{ github.job }}${{ inputs.artifact-suffix && format('-{0}', inputs.artifact-suffix) }}
+        path: ${{ env.SGLANG_CUDA_COREDUMP_DIR || '/tmp/sglang_cuda_coredumps' }}/
+        retention-days: ${{ inputs.retention-days }}
+        if-no-files-found: ignore
+
+    - name: Cleanup CUDA coredumps
+      shell: bash
+      run: rm -rf "${{ env.SGLANG_CUDA_COREDUMP_DIR || '/tmp/sglang_cuda_coredumps' }}"
--- a/.github/workflows/nightly-test-nvidia.yml
+++ b/.github/workflows/nightly-test-nvidia.yml
@@ -44,6 +44,7 @@ concurrency:

 env:
  SGLANG_IS_IN_CI: true
+  SGLANG_CUDA_COREDUMP: "1"
  HF_HUB_DOWNLOAD_TIMEOUT: 300
  HF_HUB_ETAG_TIMEOUT: 300

@@ -68,6 +69,9 @@ jobs:
          cd test
          python3 run_suite.py --hw cuda --suite nightly-1-gpu --nightly --continue-on-error

+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: always()
+
  # General tests - 4 GPU H100
  nightly-test-general-4-gpu-h100:
    if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-general-4-gpu-h100')
@@ -88,6 +92,9 @@ jobs:
          cd test
          python3 run_suite.py --hw cuda --suite nightly-4-gpu --nightly --continue-on-error

+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: always()
+
  # General tests - 8 GPU H200
  nightly-test-general-8-gpu-h200:
    if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-general-8-gpu-h200')
@@ -167,6 +174,11 @@ jobs:
          retention-days: 5
          if-no-files-found: ignore

+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: always()
+        with:
+          artifact-suffix: ${{ matrix.partition }}
+
  # General tests - 8 GPU H20
  nightly-test-general-8-gpu-h20:
    if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-general-8-gpu-h20')
@@ -191,6 +203,9 @@ jobs:
          cd test
          python3 run_suite.py --hw cuda --suite nightly-8-gpu-h20 --nightly --continue-on-error

+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: always()
+
  # General tests - 8 GPU B200
  nightly-test-general-8-gpu-b200:
    if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-general-8-gpu-b200')
@@ -259,6 +274,11 @@ jobs:
          retention-days: 5
          if-no-files-found: ignore

+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: always()
+        with:
+          artifact-suffix: ${{ matrix.partition }}
+
  # Text model accuracy tests
  nightly-test-text-accuracy-2-gpu-runner:
    if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-text-accuracy-2-gpu-runner')
@@ -279,6 +299,9 @@ jobs:
          cd test
          python3 run_suite.py --hw cuda --suite nightly-eval-text-2-gpu --nightly --continue-on-error --timeout-per-file 4500

+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: always()
+
  # Text model performance tests
  nightly-test-text-perf-2-gpu-runner:
    if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-text-perf-2-gpu-runner')
@@ -312,6 +335,9 @@ jobs:
        run: |
          python3 scripts/ci/utils/publish_traces.py --traces-dir test/performance_profiles_text_models

+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: always()
+
  # VLM accuracy tests
  nightly-test-vlm-accuracy-2-gpu-runner:
    if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-vlm-accuracy-2-gpu-runner')
@@ -332,6 +358,9 @@ jobs:
          cd test
          python3 run_suite.py --hw cuda --suite nightly-eval-vlm-2-gpu --nightly --continue-on-error --timeout-per-file 9000

+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: always()
+
  # VLM performance tests
  nightly-test-vlm-perf-2-gpu-runner:
    if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-vlm-perf-2-gpu-runner')
@@ -365,6 +394,9 @@ jobs:
        run: |
          python3 scripts/ci/utils/publish_traces.py --traces-dir test/performance_profiles_vlms

+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: always()
+
  # diffusion performance tests
  nightly-test-multimodal-server-1-gpu:
    if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-multimodal-server-1-gpu')
@@ -417,6 +449,10 @@ jobs:
          retention-days: 90
          if-no-files-found: ignore

+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: always()
+        with:
+          artifact-suffix: ${{ matrix.part }}

  nightly-test-multimodal-server-2-gpu:
    if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-multimodal-server-2-gpu')
@@ -469,6 +505,11 @@ jobs:
          retention-days: 90
          if-no-files-found: ignore

+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: always()
+        with:
+          artifact-suffix: ${{ matrix.part }}
+
  # B200 Performance tests - 4 GPU
  nightly-test-perf-4-gpu-b200:
    if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-perf-4-gpu-b200')
@@ -489,6 +530,9 @@ jobs:
          cd test
          python3 run_suite.py --hw cuda --suite nightly-4-gpu-b200 --nightly --continue-on-error --timeout-per-file 12000

+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: always()
+
  # Specialized B200 tests - 8 GPU, for specific backends and configs
  nightly-test-specialized-8-gpu-b200:
    if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-perf-8-gpu-b200')
@@ -513,6 +557,9 @@ jobs:
          cd test
          python3 run_suite.py --hw cuda --suite nightly-8-gpu-b200 --nightly --continue-on-error --timeout-per-file 2400

+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: always()
+
  # Consolidate performance metrics from all jobs
  consolidate-metrics:
    if: github.repository == 'sgl-project/sglang' && always()
--- a/.github/workflows/pr-test.yml
+++ b/.github/workflows/pr-test.yml
@@ -63,6 +63,7 @@ concurrency:

 env:
  SGLANG_IS_IN_CI: true
+  SGLANG_CUDA_COREDUMP: "1"
  SGLANG_JIT_DEEPGEMM_FAST_WARMUP: true

 permissions:
@@ -874,6 +875,9 @@ jobs:
          # temporarily put backend-independent cpu tests here
          python3 run_suite.py --hw cpu --suite default $CONTINUE_ON_ERROR_FLAG

+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: always()
+
  stage-a-cpu-only:
    needs: [check-changes, call-gate]
    if: |
@@ -976,6 +980,11 @@ jobs:
          fi
          python3 run_suite.py --hw cuda --suite stage-b-test-small-1-gpu --auto-partition-id ${{ matrix.partition }} --auto-partition-size 8 $CONTINUE_ON_ERROR_FLAG

+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: always()
+        with:
+          artifact-suffix: ${{ matrix.partition }}
+
  # Runs on H100 (80GB, SM90) - tests that don't pass on 5090 (FA3, FP8, high VRAM, etc.)
  stage-b-test-large-1-gpu:
    needs: [check-changes, call-gate, wait-for-stage-a, sgl-kernel-build-wheels]
@@ -1027,6 +1036,11 @@ jobs:
          fi
          python3 run_suite.py --hw cuda --suite stage-b-test-large-1-gpu --auto-partition-id ${{ matrix.partition }} --auto-partition-size 14 --timeout-per-file 1800 $CONTINUE_ON_ERROR_FLAG

+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: always()
+        with:
+          artifact-suffix: ${{ matrix.partition }}
+
  stage-b-test-large-2-gpu:
    needs: [check-changes, call-gate, wait-for-stage-a, sgl-kernel-build-wheels]
    if: |
@@ -1079,6 +1093,11 @@ jobs:
          fi
          python3 run_suite.py --hw cuda --suite stage-b-test-large-2-gpu --auto-partition-id ${{ matrix.partition }} --auto-partition-size 4 $CONTINUE_ON_ERROR_FLAG

+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: always()
+        with:
+          artifact-suffix: ${{ matrix.partition }}
+
  stage-b-test-4-gpu-b200:
    needs: [check-changes, call-gate, wait-for-stage-a, sgl-kernel-build-wheels]
    if: |
@@ -1132,6 +1151,9 @@ jobs:
        run: |
          IS_BLACKWELL=1 python3 -m pytest -q python/sglang/jit_kernel/tests/test_flash_attention_4.py

+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: always()
+
  multimodal-gen-test-1-gpu:
    needs: [check-changes, call-gate, sgl-kernel-build-wheels]
    if: |
@@ -1184,6 +1206,10 @@ jobs:
            --total-partitions 2 \
            $CONTINUE_ON_ERROR_FLAG

+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: always()
+        with:
+          artifact-suffix: ${{ matrix.part }}

  multimodal-gen-test-2-gpu:
    needs: [check-changes, call-gate, sgl-kernel-build-wheels]
@@ -1238,6 +1264,11 @@ jobs:
            --total-partitions 2 \
            $CONTINUE_ON_ERROR_FLAG

+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: always()
+        with:
+          artifact-suffix: ${{ matrix.part }}
+
  stage-c-test-4-gpu-h100:
    needs: [check-changes, call-gate, wait-for-stage-b]
    if: |
@@ -1287,6 +1318,11 @@ jobs:
          fi
          python3 run_suite.py --hw cuda --suite stage-c-test-4-gpu-h100 --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 $CONTINUE_ON_ERROR_FLAG

+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: always()
+        with:
+          artifact-suffix: ${{ matrix.part }}
+
  stage-c-test-8-gpu-h200:
    needs: [check-changes, call-gate, wait-for-stage-b]
    if: |
@@ -1342,6 +1378,11 @@ jobs:
          fi
          python3 run_suite.py --hw cuda --suite stage-c-test-8-gpu-h200 --auto-partition-id ${{ matrix.part }} --auto-partition-size 4 $CONTINUE_ON_ERROR_FLAG

+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: always()
+        with:
+          artifact-suffix: ${{ matrix.part }}
+
  stage-c-test-8-gpu-h20:
    needs: [check-changes, call-gate, wait-for-stage-b]
    if: |
@@ -1392,6 +1433,11 @@ jobs:
          fi
          python3 run_suite.py --hw cuda --suite stage-c-test-8-gpu-h20 --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 $CONTINUE_ON_ERROR_FLAG

+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: always()
+        with:
+          artifact-suffix: ${{ matrix.part }}
+
  stage-c-test-deepep-4-gpu:
    needs: [check-changes, call-gate, wait-for-stage-b]
    if: |
@@ -1437,6 +1483,9 @@ jobs:
          fi
          python3 run_suite.py --hw cuda --suite stage-c-test-deepep-4-gpu $CONTINUE_ON_ERROR_FLAG

+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: always()
+
  stage-c-test-deepep-8-gpu-h200:
    needs: [check-changes, call-gate, wait-for-stage-b]
    if: |
@@ -1482,6 +1531,9 @@ jobs:
          fi
          python3 run_suite.py --hw cuda --suite stage-c-test-deepep-8-gpu-h200 $CONTINUE_ON_ERROR_FLAG

+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: always()
+
  stage-c-test-4-gpu-b200:
    needs: [check-changes, call-gate, wait-for-stage-b]
    if: |
@@ -1532,6 +1584,11 @@ jobs:
          fi
          IS_BLACKWELL=1 python3 run_suite.py --hw cuda --suite stage-c-test-4-gpu-b200 --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --timeout-per-file 1800 $CONTINUE_ON_ERROR_FLAG

+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: always()
+        with:
+          artifact-suffix: ${{ matrix.part }}
+
  stage-c-test-4-gpu-gb200:
    needs: [check-changes, call-gate, wait-for-stage-b, sgl-kernel-build-wheels-arm]
    if: |
@@ -1579,6 +1636,9 @@ jobs:
          fi
          python3 run_suite.py --hw cuda --suite stage-c-test-4-gpu-gb200 --timeout-per-file 3600 $CONTINUE_ON_ERROR_FLAG

+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: always()
+
  pr-test-finish:
    needs:
      [
--- a/python/sglang/srt/debug_utils/cuda_coredump.py
+++ b/python/sglang/srt/debug_utils/cuda_coredump.py
@@ -0,0 +1,93 @@
+"""CUDA coredump helpers.
+
+When SGLANG_CUDA_COREDUMP=1, this module injects CUDA coredump environment
+variables into the current process so that GPU exceptions (e.g. illegal
+memory access) produce lightweight coredump files for post-mortem analysis
+with cuda-gdb.
+
+The injection happens at module import time via _inject_env() on a
+best-effort basis.  If any CUDA_* variable is already present in the
+environment (e.g. set by the user in the shell), injection is skipped for
+that variable and a warning is logged.  For strict guarantees, set the
+CUDA_* env vars in the shell before launching Python.
+"""
+
+import glob
+import logging
+import os
+
+from sglang.srt.environ import envs
+
+logger = logging.getLogger(__name__)
+
+_CUDA_COREDUMP_FLAGS = (
+    "skip_nonrelocated_elf_images,skip_global_memory,"
+    "skip_shared_memory,skip_local_memory,skip_constbank_memory"
+)
+
+
+def is_enabled() -> bool:
+    return envs.SGLANG_CUDA_COREDUMP.get()
+
+
+def get_dump_dir() -> str:
+    return envs.SGLANG_CUDA_COREDUMP_DIR.get()
+
+
+def _inject_env():
+    """Inject CUDA coredump environment variables into the current process.
+    If a CUDA_* variable is already present, skip it and log a warning."""
+    dump_dir = get_dump_dir()
+    os.makedirs(dump_dir, exist_ok=True)
+
+    env_vars = {
+        "CUDA_ENABLE_COREDUMP_ON_EXCEPTION": "1",
+        "CUDA_COREDUMP_SHOW_PROGRESS": "1",
+        "CUDA_COREDUMP_GENERATION_FLAGS": _CUDA_COREDUMP_FLAGS,
+        "CUDA_COREDUMP_FILE": f"{dump_dir}/cuda_coredump_%h.%p.%t",
+    }
+    for key, value in env_vars.items():
+        if key in os.environ:
+            logger.warning(
+                "CUDA coredump env var %s is already set to '%s', "
+                "skipping injection of '%s'.",
+                key,
+                os.environ[key],
+                value,
+            )
+        else:
+            os.environ[key] = value
+
+
+def cleanup_dump_dir():
+    """Remove stale coredump files from the dump directory."""
+    dump_dir = get_dump_dir()
+    for f in glob.glob(os.path.join(dump_dir, "cuda_coredump_*")):
+        os.remove(f)
+
+
+def report():
+    """Log any CUDA coredump files found after a test failure."""
+    dump_dir = get_dump_dir()
+    coredump_files = glob.glob(os.path.join(dump_dir, "cuda_coredump_*"))
+    if not coredump_files:
+        return
+
+    logger.info(f"\n{'='*60}")
+    logger.info(f"CUDA coredump(s) detected ({len(coredump_files)} file(s)):")
+    for f in coredump_files:
+        size_mb = os.path.getsize(f) / (1024 * 1024)
+        logger.info(f"  {f} ({size_mb:.1f} MB)")
+    logger.info("Use cuda-gdb to analyze: cuda-gdb -c <coredump_file>")
+
+    run_id = os.environ.get("GITHUB_RUN_ID")
+    if run_id:
+        repo = os.environ.get("GITHUB_REPOSITORY", "sgl-project/sglang")
+        logger.info(f"Download from CI: gh run download {run_id} --repo {repo}")
+
+    logger.info(f"{'='*60}\n")
+
+
+# Auto-inject CUDA coredump env vars at import time.
+if is_enabled():
+    _inject_env()
--- a/python/sglang/srt/environ.py
+++ b/python/sglang/srt/environ.py
@@ -176,6 +176,8 @@ class Envs:
    # SGLang CI
    SGLANG_IS_IN_CI = EnvBool(False)
    SGLANG_IS_IN_CI_AMD = EnvBool(False)
+    SGLANG_CUDA_COREDUMP = EnvBool(False)
+    SGLANG_CUDA_COREDUMP_DIR = EnvStr("/tmp/sglang_cuda_coredumps")
    SGLANG_TEST_MAX_RETRY = EnvInt(None)

    # Constrained Decoding (Grammar)
@@ -575,6 +577,11 @@ _warn_deprecated_env_to_cli_flag(
    "Please use '--prefill-delayer-token-usage-low-watermark' instead.",
 )

+# Import cuda_coredump to trigger auto-injection of CUDA env vars
+# when SGLANG_CUDA_COREDUMP=1. Best-effort; for strict guarantees,
+# set CUDA_* env vars in the shell before launching Python.
+import sglang.srt.debug_utils.cuda_coredump  # noqa: F401, E402
+

 def example_with_exit_stack():
    # Use this style of context manager in unit test
--- a/python/sglang/test/ci/ci_utils.py
+++ b/python/sglang/test/ci/ci_utils.py
@@ -7,6 +7,7 @@ import time
 from dataclasses import dataclass
 from typing import Callable, List, Optional, Union

+from sglang.srt.debug_utils import cuda_coredump
 from sglang.srt.utils.common import kill_process_tree
 from sglang.test.ci.ci_register import CIRegistry

@@ -130,6 +131,10 @@ def run_unittest_files(
        max_attempts: Maximum number of attempts per file including initial run (default: 2).
        retry_wait_seconds: Seconds to wait between retries (default: 60).
    """
+    coredump_enabled = cuda_coredump.is_enabled()
+    if coredump_enabled:
+        cuda_coredump.cleanup_dump_dir()
+
    tic = time.perf_counter()
    success = True
    passed_tests = []
@@ -161,7 +166,6 @@ def run_unittest_files(
                    ["python3", full_path],
                    stdout=subprocess.PIPE,
                    stderr=subprocess.STDOUT,
-                    env=os.environ,
                    text=True,
                    errors="ignore",  # Ignore non-UTF-8 bytes to prevent UnicodeDecodeError
                )
@@ -172,7 +176,7 @@ def run_unittest_files(
                process.wait()
            else:
                process = subprocess.Popen(
-                    ["python3", full_path], stdout=None, stderr=None, env=os.environ
+                    ["python3", full_path], stdout=None, stderr=None
                )
                process.wait()

@@ -258,6 +262,9 @@ def run_unittest_files(

    elapsed_total = time.perf_counter() - tic

+    if coredump_enabled and not success:
+        cuda_coredump.report()
+
    if success:
        logger.info(f"Success. Time elapsed: {elapsed_total:.2f}s")
    else:
--- a/test/registered/debug_utils/test_cuda_coredump_smoke.py
+++ b/test/registered/debug_utils/test_cuda_coredump_smoke.py
@@ -0,0 +1,29 @@
+"""Smoke test: intentionally trigger a CUDA illegal memory access
+to verify the coredump collection pipeline works end-to-end.
+
+Manual use:  python3 test/registered/debug_utils/test_cuda_coredump_smoke.py
+"""
+
+import unittest
+
+import torch
+
+from sglang.test.ci.ci_register import register_cuda_ci
+
+register_cuda_ci(
+    est_time=10,
+    suite="stage-a-test-1",
+    disabled="Manual only: triggers intentional CUDA crash for coredump verification",
+)
+
+
+class TestCudaCoredumpSmoke(unittest.TestCase):
+    def test_trigger_illegal_memory_access(self):
+        x = torch.zeros(10, device="cuda")
+        y = torch.arange(10, device="cuda")
+        x[y * y] = 1
+        torch.cuda.synchronize()
+
+
+if __name__ == "__main__":
+    unittest.main()