[SpecV2-Mamba]: Refactor additional_ratio calculation when init mamba pool (#19660)
This commit is contained in:
@@ -342,10 +342,11 @@ class ModelRunnerKVCacheMixin:
|
||||
|
||||
additional_ratio = 0
|
||||
if self.server_args.enable_mamba_extra_buffer():
|
||||
if not self.spec_algorithm.is_none():
|
||||
additional_ratio = MAMBA_CACHE_V2_ADDITIONAL_RATIO_NO_OVERLAP
|
||||
else:
|
||||
# ping-pong buffer size is 2 when overlap schedule is on, 1 otherwise.
|
||||
if not self.server_args.disable_overlap_schedule:
|
||||
additional_ratio = MAMBA_CACHE_V2_ADDITIONAL_RATIO_OVERLAP
|
||||
else:
|
||||
additional_ratio = MAMBA_CACHE_V2_ADDITIONAL_RATIO_NO_OVERLAP
|
||||
|
||||
return MAMBA_CACHE_SIZE_MAX_RUNNING_REQUESTS_RATIO + additional_ratio
|
||||
|
||||
|
||||
Reference in New Issue
Block a user