From f4393bf3f68889cf4bb35618301b5dc7e8dc22e9 Mon Sep 17 00:00:00 2001 From: gaopengff Date: Mon, 16 Mar 2026 11:05:36 +0800 Subject: [PATCH] Fix correctness test issue for bench_one_batch (#20650) --- python/sglang/bench_one_batch.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/python/sglang/bench_one_batch.py b/python/sglang/bench_one_batch.py index 8cf0aee1b..6ddc14c8d 100644 --- a/python/sglang/bench_one_batch.py +++ b/python/sglang/bench_one_batch.py @@ -71,6 +71,7 @@ from sglang.srt.layers.quantization.fp4_utils import initialize_fp4_gemm_config from sglang.srt.layers.quantization.fp8_utils import initialize_fp8_gemm_config from sglang.srt.managers.schedule_batch import Req, ScheduleBatch from sglang.srt.managers.scheduler_dp_attn_mixin import prepare_mlp_sync_batch_raw +from sglang.srt.mem_cache.base_prefix_cache import EvictParams from sglang.srt.model_executor.forward_batch_info import ForwardBatch from sglang.srt.model_executor.model_runner import ModelRunner from sglang.srt.sampling.sampling_params import SamplingParams @@ -286,6 +287,16 @@ def load_model(server_args, port_args, gpu_id, tp_rank): def prepare_inputs_for_correctness_test(bench_args, tokenizer, custom_prompts): + if custom_prompts: + custom_input_len = len(custom_prompts) + bs = bench_args.batch_size[0] + if custom_input_len > bs: + logging.warning( + f"Custom input size ({custom_input_len}) is larger than batch_size ({bs}). " + f"Using the first {bs} prompts." + ) + custom_prompts = custom_prompts[:bs] + prompts = ( custom_prompts if custom_prompts @@ -376,6 +387,9 @@ class TreeCacheNamespace(SimpleNamespace): def is_tree_cache(self) -> bool: return not self.is_chunk_cache() + def evict(self, params: EvictParams): + pass + @torch.no_grad def extend(reqs, model_runner):