chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 11 Packages Projects Releases Wiki Activity

[Core] Unify max_num_reqs dp_size division for pool sizing (#20063)

Browse Source 
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Liangsheng Yin
2026-03-06 16:12:59 -08:00
committed by GitHub
parent 7a12255b6e
commit ddcecdea49
1 changed files with 8 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
python/sglang/srt/model_executor/model_runner_kv_cache_mixin.py
View File

@@ -676,20 +676,18 @@ class ModelRunnerKVCacheMixin:
)
def init_memory_pool(self: ModelRunner, pre_model_load_memory: int):
max_num_reqs = self.server_args.max_running_requests
max_total_tokens = self.server_args.max_total_tokens
self.max_total_num_tokens = self.profile_max_num_token(pre_model_load_memory)
if max_num_reqs is None:
max_num_reqs = min(
max(
int(
self.max_total_num_tokens / self.model_config.context_len * 512
),
2048,
),
4096,
# Resolve max_num_reqs (per dp worker)
max_num_reqs = self.server_args.max_running_requests
if max_num_reqs is not None:
max_num_reqs = max_num_reqs // self.dp_size
else:
estimated = int(
self.max_total_num_tokens / self.model_config.context_len * 512
)
max_num_reqs = max(min(estimated, 4096), 2048)
if self.mambaish_config is not None:
ratio = self._calculate_mamba_ratio()
@@ -699,15 +697,6 @@ class ModelRunnerKVCacheMixin:
max_num_reqs, self.server_args.max_mamba_cache_size // ratio
)
# for dp attention, we need control the max_num_reqs for speculative decoding mamba space
if (
not self.spec_algorithm.is_none()
and self.server_args.enable_dp_attention
):
max_num_reqs = min(
max_num_reqs, self.server_args.max_running_requests // self.dp_size
)
if max_total_tokens is not None:
if max_total_tokens > self.max_total_num_tokens:
logging.warning(