feat: add cuda core dump CI warpper (#18909)

python/sglang/srt/debug_utils/cuda_coredump.py

@@ -0,0 +1,93 @@
+"""CUDA coredump helpers.
+
+When SGLANG_CUDA_COREDUMP=1, this module injects CUDA coredump environment
+variables into the current process so that GPU exceptions (e.g. illegal
+memory access) produce lightweight coredump files for post-mortem analysis
+with cuda-gdb.
+
+The injection happens at module import time via _inject_env() on a
+best-effort basis.  If any CUDA_* variable is already present in the
+environment (e.g. set by the user in the shell), injection is skipped for
+that variable and a warning is logged.  For strict guarantees, set the
+CUDA_* env vars in the shell before launching Python.
+"""
+
+import glob
+import logging
+import os
+
+from sglang.srt.environ import envs
+
+logger = logging.getLogger(__name__)
+
+_CUDA_COREDUMP_FLAGS = (
+    "skip_nonrelocated_elf_images,skip_global_memory,"
+    "skip_shared_memory,skip_local_memory,skip_constbank_memory"
+)
+
+
+def is_enabled() -> bool:
+    return envs.SGLANG_CUDA_COREDUMP.get()
+
+
+def get_dump_dir() -> str:
+    return envs.SGLANG_CUDA_COREDUMP_DIR.get()
+
+
+def _inject_env():
+    """Inject CUDA coredump environment variables into the current process.
+    If a CUDA_* variable is already present, skip it and log a warning."""
+    dump_dir = get_dump_dir()
+    os.makedirs(dump_dir, exist_ok=True)
+
+    env_vars = {
+        "CUDA_ENABLE_COREDUMP_ON_EXCEPTION": "1",
+        "CUDA_COREDUMP_SHOW_PROGRESS": "1",
+        "CUDA_COREDUMP_GENERATION_FLAGS": _CUDA_COREDUMP_FLAGS,
+        "CUDA_COREDUMP_FILE": f"{dump_dir}/cuda_coredump_%h.%p.%t",
+    }
+    for key, value in env_vars.items():
+        if key in os.environ:
+            logger.warning(
+                "CUDA coredump env var %s is already set to '%s', "
+                "skipping injection of '%s'.",
+                key,
+                os.environ[key],
+                value,
+            )
+        else:
+            os.environ[key] = value
+
+
+def cleanup_dump_dir():
+    """Remove stale coredump files from the dump directory."""
+    dump_dir = get_dump_dir()
+    for f in glob.glob(os.path.join(dump_dir, "cuda_coredump_*")):
+        os.remove(f)
+
+
+def report():
+    """Log any CUDA coredump files found after a test failure."""
+    dump_dir = get_dump_dir()
+    coredump_files = glob.glob(os.path.join(dump_dir, "cuda_coredump_*"))
+    if not coredump_files:
+        return
+
+    logger.info(f"\n{'='*60}")
+    logger.info(f"CUDA coredump(s) detected ({len(coredump_files)} file(s)):")
+    for f in coredump_files:
+        size_mb = os.path.getsize(f) / (1024 * 1024)
+        logger.info(f"  {f} ({size_mb:.1f} MB)")
+    logger.info("Use cuda-gdb to analyze: cuda-gdb -c <coredump_file>")
+
+    run_id = os.environ.get("GITHUB_RUN_ID")
+    if run_id:
+        repo = os.environ.get("GITHUB_REPOSITORY", "sgl-project/sglang")
+        logger.info(f"Download from CI: gh run download {run_id} --repo {repo}")
+
+    logger.info(f"{'='*60}\n")
+
+
+# Auto-inject CUDA coredump env vars at import time.
+if is_enabled():
+    _inject_env()

python/sglang/srt/environ.py

@@ -176,6 +176,8 @@ class Envs:
     # SGLang CI
     SGLANG_IS_IN_CI = EnvBool(False)
     SGLANG_IS_IN_CI_AMD = EnvBool(False)
+    SGLANG_CUDA_COREDUMP = EnvBool(False)
+    SGLANG_CUDA_COREDUMP_DIR = EnvStr("/tmp/sglang_cuda_coredumps")
     SGLANG_TEST_MAX_RETRY = EnvInt(None)
 
     # Constrained Decoding (Grammar)
@@ -575,6 +577,11 @@ _warn_deprecated_env_to_cli_flag(
     "Please use '--prefill-delayer-token-usage-low-watermark' instead.",
 )
 
+# Import cuda_coredump to trigger auto-injection of CUDA env vars
+# when SGLANG_CUDA_COREDUMP=1. Best-effort; for strict guarantees,
+# set CUDA_* env vars in the shell before launching Python.
+import sglang.srt.debug_utils.cuda_coredump  # noqa: F401, E402
+
 
 def example_with_exit_stack():
     # Use this style of context manager in unit test

python/sglang/test/ci/ci_utils.py

@@ -7,6 +7,7 @@ import time
 from dataclasses import dataclass
 from typing import Callable, List, Optional, Union
 
+from sglang.srt.debug_utils import cuda_coredump
 from sglang.srt.utils.common import kill_process_tree
 from sglang.test.ci.ci_register import CIRegistry
 
@@ -130,6 +131,10 @@ def run_unittest_files(
         max_attempts: Maximum number of attempts per file including initial run (default: 2).
         retry_wait_seconds: Seconds to wait between retries (default: 60).
     """
+    coredump_enabled = cuda_coredump.is_enabled()
+    if coredump_enabled:
+        cuda_coredump.cleanup_dump_dir()
+
     tic = time.perf_counter()
     success = True
     passed_tests = []
@@ -161,7 +166,6 @@ def run_unittest_files(
                     ["python3", full_path],
                     stdout=subprocess.PIPE,
                     stderr=subprocess.STDOUT,
-                    env=os.environ,
                     text=True,
                     errors="ignore",  # Ignore non-UTF-8 bytes to prevent UnicodeDecodeError
                 )
@@ -172,7 +176,7 @@ def run_unittest_files(
                 process.wait()
             else:
                 process = subprocess.Popen(
-                    ["python3", full_path], stdout=None, stderr=None, env=os.environ
+                    ["python3", full_path], stdout=None, stderr=None
                 )
                 process.wait()
 
@@ -258,6 +262,9 @@ def run_unittest_files(
 
     elapsed_total = time.perf_counter() - tic
 
+    if coredump_enabled and not success:
+        cuda_coredump.report()
+
     if success:
         logger.info(f"Success. Time elapsed: {elapsed_total:.2f}s")
     else: