diff --git a/python/sglang/bench_serving.py b/python/sglang/bench_serving.py index 410af18f9..0a5550ff2 100644 --- a/python/sglang/bench_serving.py +++ b/python/sglang/bench_serving.py @@ -1626,6 +1626,7 @@ def calculate_metrics( dur_s: float, tokenizer: PreTrainedTokenizerBase, backend: str, + accept_length: Optional[float] = None, ) -> Tuple[BenchmarkMetrics, List[int]]: output_lens: List[int] = [] retokenized_output_lens: List[int] = [] @@ -1650,7 +1651,14 @@ def calculate_metrics( total_input_vision += input_requests[i].vision_prompt_len if output_len > 1: tpots.append((outputs[i].latency - outputs[i].ttft) / (output_len - 1)) - itls += outputs[i].itl + if ( + accept_length + and accept_length > 0 + and backend in ("sglang-oai", "sglang-oai-chat") + ): + itls += [v / accept_length for v in outputs[i].itl] + else: + itls += outputs[i].itl ttfts.append(outputs[i].ttft) e2e_latencies.append(outputs[i].latency) @@ -1929,6 +1937,7 @@ async def benchmark( dur_s=benchmark_duration, tokenizer=tokenizer, backend=backend, + accept_length=accept_length, ) print("\n{s:{c}^{n}}".format(s=" Serving Benchmark Result ", n=50, c="="))