diff --git a/python/sglang/srt/model_executor/model_runner_kv_cache_mixin.py b/python/sglang/srt/model_executor/model_runner_kv_cache_mixin.py index 60da20313..c46ff0a38 100644 --- a/python/sglang/srt/model_executor/model_runner_kv_cache_mixin.py +++ b/python/sglang/srt/model_executor/model_runner_kv_cache_mixin.py @@ -676,20 +676,18 @@ class ModelRunnerKVCacheMixin: ) def init_memory_pool(self: ModelRunner, pre_model_load_memory: int): - max_num_reqs = self.server_args.max_running_requests max_total_tokens = self.server_args.max_total_tokens self.max_total_num_tokens = self.profile_max_num_token(pre_model_load_memory) - if max_num_reqs is None: - max_num_reqs = min( - max( - int( - self.max_total_num_tokens / self.model_config.context_len * 512 - ), - 2048, - ), - 4096, + # Resolve max_num_reqs (per dp worker) + max_num_reqs = self.server_args.max_running_requests + if max_num_reqs is not None: + max_num_reqs = max_num_reqs // self.dp_size + else: + estimated = int( + self.max_total_num_tokens / self.model_config.context_len * 512 ) + max_num_reqs = max(min(estimated, 4096), 2048) if self.mambaish_config is not None: ratio = self._calculate_mamba_ratio() @@ -699,15 +697,6 @@ class ModelRunnerKVCacheMixin: max_num_reqs, self.server_args.max_mamba_cache_size // ratio ) - # for dp attention, we need control the max_num_reqs for speculative decoding mamba space - if ( - not self.spec_algorithm.is_none() - and self.server_args.enable_dp_attention - ): - max_num_reqs = min( - max_num_reqs, self.server_args.max_running_requests // self.dp_size - ) - if max_total_tokens is not None: if max_total_tokens > self.max_total_num_tokens: logging.warning(