chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[Auto Sync] Update server_args.py (20251223) (#15700)

Browse Source 
Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
Co-authored-by: Stefan He <hebiaobuaa@gmail.com>
This commit is contained in:
Lianmin Zheng
2025-12-24 14:55:14 -08:00
committed by GitHub
parent 5454d2a7ae
commit 92ddc46824
1 changed files with 8 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
python/sglang/srt/server_args.py
View File

@@ -549,7 +549,7 @@ class ServerArgs:
enable_piecewise_cuda_graph: bool = False
enable_torch_compile_debug_mode: bool = False
torch_compile_max_bs: int = 32
piecewise_cuda_graph_max_tokens: int = 4096
piecewise_cuda_graph_max_tokens: Optional[int] = None
piecewise_cuda_graph_tokens: Optional[List[int]] = None
piecewise_cuda_graph_compiler: str = "eager"
torchao_config: str = ""
@@ -895,6 +895,12 @@ class ServerArgs:
else:
self.cuda_graph_max_bs = max(self.cuda_graph_bs)
if self.piecewise_cuda_graph_max_tokens is None:
# Capture piecewise cuda graph tokens up to the chunked prefill size. Two benefits:
# 1. cuda graph acceleration for all prefill lengths.
# 2. do not need more temporary memory for activations. Less fragmentation.
self.piecewise_cuda_graph_max_tokens = self.chunked_prefill_size
if self.piecewise_cuda_graph_tokens is None:
self.piecewise_cuda_graph_tokens = (
self._generate_piecewise_cuda_graph_tokens()
@@ -936,7 +942,7 @@ class ServerArgs:
# For piecewise cuda graphs
if self.enable_piecewise_cuda_graph:
reserved_mem += self.piecewise_cuda_graph_max_tokens // 4
reserved_mem += self.piecewise_cuda_graph_max_tokens // 8
self.mem_fraction_static = (
round((gpu_mem - reserved_mem) / gpu_mem, 3)