chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 11 Packages Projects Releases Wiki Activity

Retry removing wrong logic about max total token in spec decoding (#15748)

Browse Source
This commit is contained in:
Liangsheng Yin
2025-12-24 21:13:26 +08:00
committed by GitHub
parent e245cac0c5
commit d5fbbfd988
1 changed files with 0 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
python/sglang/srt/model_executor/model_runner.py
View File

@@ -1876,23 +1876,7 @@ class ModelRunner:
self.max_total_num_tokens = self.server_args.draft_runner_cache_size
max_num_reqs = self.server_args.max_num_reqs
else:
# We are sharing the `token_to_kv_pool`, and both verify and draft tokens
# can be concurrently allocated, so we should give a headroom for it.
extra_tokens = (
# draft
max_num_reqs
* self.server_args.speculative_num_steps
* self.server_args.speculative_eagle_topk
# verify
+ max_num_reqs * self.server_args.speculative_num_draft_tokens
# buffer
+ 100
)
# Target worker and draft worker shares the same indices for the
# token_to_kv_pool, so we should make sure to match max_total_num_tokens.
self.max_total_num_tokens += extra_tokens
self.server_args.draft_runner_cache_size = self.max_total_num_tokens
self.server_args.max_num_reqs = max_num_reqs
if max_total_tokens is not None: