diff --git a/python/sglang/bench_serving.py b/python/sglang/bench_serving.py index cf62da0a7..1b9fa5ae7 100644 --- a/python/sglang/bench_serving.py +++ b/python/sglang/bench_serving.py @@ -932,6 +932,7 @@ class BenchmarkMetrics: mean_e2e_latency_ms: float median_e2e_latency_ms: float std_e2e_latency_ms: float + p90_e2e_latency_ms: float p99_e2e_latency_ms: float concurrency: float max_output_tokens_per_s: float = 0.0 @@ -2081,6 +2082,7 @@ def calculate_metrics( mean_e2e_latency_ms=np.mean(e2e_latencies) * 1000, median_e2e_latency_ms=np.median(e2e_latencies) * 1000, std_e2e_latency_ms=np.std(e2e_latencies) * 1000, + p90_e2e_latency_ms=np.percentile(e2e_latencies, 90) * 1000, p99_e2e_latency_ms=np.percentile(e2e_latencies, 99) * 1000, concurrency=np.sum(e2e_latencies) / dur_s, max_output_tokens_per_s=max_output_tokens_per_s, @@ -2409,6 +2411,12 @@ async def benchmark( "Median E2E Latency (ms):", metrics.median_e2e_latency_ms ) ) + print( + "{:<40} {:<10.2f}".format("P90 E2E Latency (ms):", metrics.p90_e2e_latency_ms) + ) + print( + "{:<40} {:<10.2f}".format("P99 E2E Latency (ms):", metrics.p99_e2e_latency_ms) + ) print("{s:{c}^{n}}".format(s="Time to First Token", n=50, c="-")) print("{:<40} {:<10.2f}".format("Mean TTFT (ms):", metrics.mean_ttft_ms)) print("{:<40} {:<10.2f}".format("Median TTFT (ms):", metrics.median_ttft_ms)) @@ -2463,6 +2471,7 @@ async def benchmark( "mean_e2e_latency_ms": metrics.mean_e2e_latency_ms, "median_e2e_latency_ms": metrics.median_e2e_latency_ms, "std_e2e_latency_ms": metrics.std_e2e_latency_ms, + "p90_e2e_latency_ms": metrics.p90_e2e_latency_ms, "p99_e2e_latency_ms": metrics.p99_e2e_latency_ms, "mean_ttft_ms": metrics.mean_ttft_ms, "median_ttft_ms": metrics.median_ttft_ms,