Fix correctness test issue for bench_one_batch (#20650)

python/sglang/bench_one_batch.py

@@ -71,6 +71,7 @@ from sglang.srt.layers.quantization.fp4_utils import initialize_fp4_gemm_config
 from sglang.srt.layers.quantization.fp8_utils import initialize_fp8_gemm_config
 from sglang.srt.managers.schedule_batch import Req, ScheduleBatch
 from sglang.srt.managers.scheduler_dp_attn_mixin import prepare_mlp_sync_batch_raw
+from sglang.srt.mem_cache.base_prefix_cache import EvictParams
 from sglang.srt.model_executor.forward_batch_info import ForwardBatch
 from sglang.srt.model_executor.model_runner import ModelRunner
 from sglang.srt.sampling.sampling_params import SamplingParams
@@ -286,6 +287,16 @@ def load_model(server_args, port_args, gpu_id, tp_rank):
 
 
 def prepare_inputs_for_correctness_test(bench_args, tokenizer, custom_prompts):
+    if custom_prompts:
+        custom_input_len = len(custom_prompts)
+        bs = bench_args.batch_size[0]
+        if custom_input_len > bs:
+            logging.warning(
+                f"Custom input size ({custom_input_len}) is larger than batch_size ({bs}). "
+                f"Using the first {bs} prompts."
+            )
+            custom_prompts = custom_prompts[:bs]
+
     prompts = (
         custom_prompts
         if custom_prompts
@@ -376,6 +387,9 @@ class TreeCacheNamespace(SimpleNamespace):
     def is_tree_cache(self) -> bool:
         return not self.is_chunk_cache()
 
+    def evict(self, params: EvictParams):
+        pass
+
 
 @torch.no_grad
 def extend(reqs, model_runner):