feat: add cuda core dump CI warpper (#18909)

This commit is contained in:
Liangsheng Yin
2026-02-17 14:49:26 -08:00
committed by GitHub
parent 9a7d6be567
commit 83a475e8d7
7 changed files with 272 additions and 2 deletions

View File

@@ -7,6 +7,7 @@ import time
from dataclasses import dataclass
from typing import Callable, List, Optional, Union
from sglang.srt.debug_utils import cuda_coredump
from sglang.srt.utils.common import kill_process_tree
from sglang.test.ci.ci_register import CIRegistry
@@ -130,6 +131,10 @@ def run_unittest_files(
max_attempts: Maximum number of attempts per file including initial run (default: 2).
retry_wait_seconds: Seconds to wait between retries (default: 60).
"""
coredump_enabled = cuda_coredump.is_enabled()
if coredump_enabled:
cuda_coredump.cleanup_dump_dir()
tic = time.perf_counter()
success = True
passed_tests = []
@@ -161,7 +166,6 @@ def run_unittest_files(
["python3", full_path],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
env=os.environ,
text=True,
errors="ignore", # Ignore non-UTF-8 bytes to prevent UnicodeDecodeError
)
@@ -172,7 +176,7 @@ def run_unittest_files(
process.wait()
else:
process = subprocess.Popen(
["python3", full_path], stdout=None, stderr=None, env=os.environ
["python3", full_path], stdout=None, stderr=None
)
process.wait()
@@ -258,6 +262,9 @@ def run_unittest_files(
elapsed_total = time.perf_counter() - tic
if coredump_enabled and not success:
cuda_coredump.report()
if success:
logger.info(f"Success. Time elapsed: {elapsed_total:.2f}s")
else: