From 7541da15d20d1cd3170b63f54fc03ba57fccca15 Mon Sep 17 00:00:00 2001 From: gaopengff Date: Fri, 30 Jan 2026 13:28:17 +0800 Subject: [PATCH] Fix prefill latency performance drop of bench serving (#14592) --- python/sglang/bench_serving.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/python/sglang/bench_serving.py b/python/sglang/bench_serving.py index a00dec41c..8ccde20cc 100644 --- a/python/sglang/bench_serving.py +++ b/python/sglang/bench_serving.py @@ -1534,6 +1534,12 @@ def sample_random_requests( num=num_prompts, ) + if return_text: + # Need to truncate input_len as server encode will add special token. + num_special_tokens = int(tokenizer.num_special_tokens_to_add()) + for i in range(num_prompts): + input_lens[i] = max(0, input_lens[i] - num_special_tokens) + if random_sample: # Sample token ids from ShareGPT and repeat/truncate them to satisfy the input_lens