From 07da4bed7b31cdb28f582bbfddd852373def51f1 Mon Sep 17 00:00:00 2001 From: Yilong Zhao <74357408+happierpig@users.noreply.github.com> Date: Fri, 27 Feb 2026 02:14:46 -0800 Subject: [PATCH] [cache] add conservative estimation (#19482) --- python/sglang/srt/managers/schedule_policy.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/python/sglang/srt/managers/schedule_policy.py b/python/sglang/srt/managers/schedule_policy.py index c339de20f..1b137e52b 100644 --- a/python/sglang/srt/managers/schedule_policy.py +++ b/python/sglang/srt/managers/schedule_policy.py @@ -513,8 +513,10 @@ class PrefillAdder: # TODO(lsyin): check this workaround logic, which only ensures the prefill will not out of memory, and may be too conservative extend_input_len = self.ceil_paged_tokens(extend_input_len) - self.rem_total_token_offset += extend_input_len + max_new_tokens - self.cur_rem_token_offset += extend_input_len + # alloc_extend reserves an extra page_size per request to make sure the budget doesn't over-commit + page_overhead = self.page_size + self.rem_total_token_offset += extend_input_len + max_new_tokens + page_overhead + self.cur_rem_token_offset += extend_input_len + page_overhead self.rem_input_tokens -= extend_input_len if self.dllm_config is not None: