chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 11 Packages Projects Releases Wiki Activity

add runtime check for PyTorch 2.9.1 + CuDNN < 9.15 to prevent Conv3d performance issues (#14119)

Browse Source
This commit is contained in:
Yuhao Yang
2025-11-29 23:05:54 +08:00
committed by GitHub
parent 4cafc835d3
commit f03ea34a3d
2 changed files with 46 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
python/sglang/srt/server_args.py
View File

@@ -3979,6 +3979,11 @@ class ServerArgs:
# Check LoRA
self.check_lora_server_args()
# torch 2.9.1 has compatibility issues with cuDNN 9.14 and below,
# causing extremely slow nn.Conv3d performance.
# TODO(yhyang201): Remove this check when sglang no longer uses torch 2.9.1.
self.check_torch_2_9_1_cudnn_compatibility()
# Check speculative decoding
if self.speculative_algorithm is not None:
assert (
@@ -4054,6 +4059,46 @@ class ServerArgs:
if self.model_impl == "mindspore":
assert is_npu(), "MindSpore model impl is only supported on Ascend npu."
def check_torch_2_9_1_cudnn_compatibility(self):
if self.get_model_config().is_multimodal:
import torch
torch_version = torch.__version__.split("+", 1)[0]
if torch_version == "2.9.1":
cudnn_version = None
try:
cudnn_version = torch.backends.cudnn.version()
except Exception:
cudnn_version = None
if cudnn_version is not None:
version_float = float(str(cudnn_version)[:3]) / 100
if version_float < 9.15:
RED = "\033[91m"
BOLD = "\033[1m"
RESET = "\033[0m"
msg = (
f"{RED}{BOLD}"
"CRITICAL WARNING: PyTorch 2.9.1 & CuDNN Compatibility Issue Detected\n"
"--------------------------------------------------------------------------------\n"
f"Current Environment: PyTorch {torch.__version__} | CuDNN {version_float:.2f}\n\n"
"Issue: There is a KNOWN BUG in PyTorch 2.9.1's `nn.Conv3d` implementation\n"
" when used with CuDNN versions older than 9.15. This can cause\n"
" SEVERE PERFORMANCE DEGRADATION and EXCESSIVE MEMORY USAGE.\n\n"
"Reference: https://github.com/pytorch/pytorch/issues/168167\n\n"
"Solution: You MUST upgrade CuDNN to version 9.15+ to ensure correctness.\n\n"
"Run the following command immediately to fix:\n"
" pip install nvidia-cudnn-cu12==9.16.0.29\n"
"--------------------------------------------------------------------------------\n"
f"{RESET}"
)
raise RuntimeError(msg)
else:
RED = "\033[91m"
RESET = "\033[0m"
logger.warning(
f"{RED}WARNING: Could not determine CuDNN version for torch==2.9.1. Please ensure CuDNN >= 9.15 to avoid nn.Conv3d bugs.{RESET}"
)
def check_lora_server_args(self):
assert self.max_loras_per_batch > 0, "max_loras_per_batch must be positive"