From ddcecdea49829480f8a96d5c6c5ae0b099346e15 Mon Sep 17 00:00:00 2001 From: Liangsheng Yin Date: Fri, 6 Mar 2026 16:12:59 -0800 Subject: [PATCH] [Core] Unify `max_num_reqs` `dp_size` division for pool sizing (#20063) Co-authored-by: Claude Opus 4.6 (1M context) --- .../model_runner_kv_cache_mixin.py | 27 ++++++------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/python/sglang/srt/model_executor/model_runner_kv_cache_mixin.py b/python/sglang/srt/model_executor/model_runner_kv_cache_mixin.py index 60da20313..c46ff0a38 100644 --- a/python/sglang/srt/model_executor/model_runner_kv_cache_mixin.py +++ b/python/sglang/srt/model_executor/model_runner_kv_cache_mixin.py @@ -676,20 +676,18 @@ class ModelRunnerKVCacheMixin: ) def init_memory_pool(self: ModelRunner, pre_model_load_memory: int): - max_num_reqs = self.server_args.max_running_requests max_total_tokens = self.server_args.max_total_tokens self.max_total_num_tokens = self.profile_max_num_token(pre_model_load_memory) - if max_num_reqs is None: - max_num_reqs = min( - max( - int( - self.max_total_num_tokens / self.model_config.context_len * 512 - ), - 2048, - ), - 4096, + # Resolve max_num_reqs (per dp worker) + max_num_reqs = self.server_args.max_running_requests + if max_num_reqs is not None: + max_num_reqs = max_num_reqs // self.dp_size + else: + estimated = int( + self.max_total_num_tokens / self.model_config.context_len * 512 ) + max_num_reqs = max(min(estimated, 4096), 2048) if self.mambaish_config is not None: ratio = self._calculate_mamba_ratio() @@ -699,15 +697,6 @@ class ModelRunnerKVCacheMixin: max_num_reqs, self.server_args.max_mamba_cache_size // ratio ) - # for dp attention, we need control the max_num_reqs for speculative decoding mamba space - if ( - not self.spec_algorithm.is_none() - and self.server_args.enable_dp_attention - ): - max_num_reqs = min( - max_num_reqs, self.server_args.max_running_requests // self.dp_size - ) - if max_total_tokens is not None: if max_total_tokens > self.max_total_num_tokens: logging.warning(