[CPU] Adding Memory Capacity Acquisition Functionality (#11102)
This commit is contained in:
@@ -639,7 +639,7 @@ class ServerArgs:
|
||||
if self.cuda_graph_max_bs > 300:
|
||||
reserved_mem += self.cuda_graph_max_bs * self.dp_size * 1.5
|
||||
|
||||
if gpu_mem > 60 * 1024:
|
||||
if gpu_mem is not None and gpu_mem > 60 * 1024:
|
||||
reserved_mem = max(reserved_mem, 10 * 1024)
|
||||
|
||||
if self.speculative_algorithm is not None:
|
||||
@@ -650,7 +650,11 @@ class ServerArgs:
|
||||
# eagle draft models and cuda graphs
|
||||
reserved_mem += 2 * 1024
|
||||
|
||||
self.mem_fraction_static = round((gpu_mem - reserved_mem) / gpu_mem, 3)
|
||||
self.mem_fraction_static = (
|
||||
round((gpu_mem - reserved_mem) / gpu_mem, 3)
|
||||
if gpu_mem is not None
|
||||
else 0.88
|
||||
)
|
||||
|
||||
# Lazy init to avoid circular import
|
||||
# Multimodal models need more memory for the image processor
|
||||
|
||||
Reference in New Issue
Block a user