feat: add cuda core dump CI warpper (#18909)
This commit is contained in:
@@ -7,6 +7,7 @@ import time
|
||||
from dataclasses import dataclass
|
||||
from typing import Callable, List, Optional, Union
|
||||
|
||||
from sglang.srt.debug_utils import cuda_coredump
|
||||
from sglang.srt.utils.common import kill_process_tree
|
||||
from sglang.test.ci.ci_register import CIRegistry
|
||||
|
||||
@@ -130,6 +131,10 @@ def run_unittest_files(
|
||||
max_attempts: Maximum number of attempts per file including initial run (default: 2).
|
||||
retry_wait_seconds: Seconds to wait between retries (default: 60).
|
||||
"""
|
||||
coredump_enabled = cuda_coredump.is_enabled()
|
||||
if coredump_enabled:
|
||||
cuda_coredump.cleanup_dump_dir()
|
||||
|
||||
tic = time.perf_counter()
|
||||
success = True
|
||||
passed_tests = []
|
||||
@@ -161,7 +166,6 @@ def run_unittest_files(
|
||||
["python3", full_path],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
env=os.environ,
|
||||
text=True,
|
||||
errors="ignore", # Ignore non-UTF-8 bytes to prevent UnicodeDecodeError
|
||||
)
|
||||
@@ -172,7 +176,7 @@ def run_unittest_files(
|
||||
process.wait()
|
||||
else:
|
||||
process = subprocess.Popen(
|
||||
["python3", full_path], stdout=None, stderr=None, env=os.environ
|
||||
["python3", full_path], stdout=None, stderr=None
|
||||
)
|
||||
process.wait()
|
||||
|
||||
@@ -258,6 +262,9 @@ def run_unittest_files(
|
||||
|
||||
elapsed_total = time.perf_counter() - tic
|
||||
|
||||
if coredump_enabled and not success:
|
||||
cuda_coredump.report()
|
||||
|
||||
if success:
|
||||
logger.info(f"Success. Time elapsed: {elapsed_total:.2f}s")
|
||||
else:
|
||||
|
||||
Reference in New Issue
Block a user