[CPU] Adding Memory Capacity Acquisition Functionality (#11102)

This commit is contained in:
Zaili Wang
2025-09-30 08:41:20 -07:00
committed by GitHub
parent 9710f718fb
commit 229d2b95f1
2 changed files with 34 additions and 2 deletions
+6 -2
View File
@@ -639,7 +639,7 @@ class ServerArgs:
if self.cuda_graph_max_bs > 300:
reserved_mem += self.cuda_graph_max_bs * self.dp_size * 1.5
if gpu_mem > 60 * 1024:
if gpu_mem is not None and gpu_mem > 60 * 1024:
reserved_mem = max(reserved_mem, 10 * 1024)
if self.speculative_algorithm is not None:
@@ -650,7 +650,11 @@ class ServerArgs:
# eagle draft models and cuda graphs
reserved_mem += 2 * 1024
self.mem_fraction_static = round((gpu_mem - reserved_mem) / gpu_mem, 3)
self.mem_fraction_static = (
round((gpu_mem - reserved_mem) / gpu_mem, 3)
if gpu_mem is not None
else 0.88
)
# Lazy init to avoid circular import
# Multimodal models need more memory for the image processor