From 1ed946680aeb1bbee6518fd4bf878b793d0f08ba Mon Sep 17 00:00:00 2001 From: Mick Date: Sun, 21 Dec 2025 13:37:45 +0800 Subject: [PATCH] [diffusion] bench: improve bench_serving by adding more controlling args (#15554) --- .../benchmarks/bench_serving.py | 94 +++++++++++++++---- .../runtime/entrypoints/openai/common_api.py | 14 +++ 2 files changed, 89 insertions(+), 19 deletions(-) diff --git a/python/sglang/multimodal_gen/benchmarks/bench_serving.py b/python/sglang/multimodal_gen/benchmarks/bench_serving.py index 63eaf5c4c..8960c1391 100644 --- a/python/sglang/multimodal_gen/benchmarks/bench_serving.py +++ b/python/sglang/multimodal_gen/benchmarks/bench_serving.py @@ -612,6 +612,17 @@ async def benchmark(args): # Wait for service wait_for_service(args.base_url) + # Fetch model info + try: + resp = requests.get(f"{args.base_url}/v1/model_info", timeout=5) + if resp.status_code == 200: + info = resp.json() + if "model_path" in info and info["model_path"]: + args.model = info["model_path"] + print(f"Updated model name from server: {args.model}") + except Exception as e: + print(f"Failed to fetch model info: {e}. Using default: {args.model}") + # Setup dataset if args.backend == "sglang-image": if args.task == "i2v": @@ -632,14 +643,21 @@ async def benchmark(args): else: raise ValueError(f"Unknown dataset: {args.dataset}") + print(f"Loading requests...") requests_list = dataset.get_requests() print(f"Prepared {len(requests_list)} requests from {args.dataset} dataset.") # Limit concurrency - semaphore = asyncio.Semaphore(args.max_concurrency) + if args.max_concurrency is not None: + semaphore = asyncio.Semaphore(args.max_concurrency) + else: + semaphore = None async def limited_request_func(req, session, pbar): - async with semaphore: + if semaphore: + async with semaphore: + return await request_func(req, session, pbar) + else: return await request_func(req, session, pbar) # Run benchmark @@ -650,8 +668,8 @@ async def benchmark(args): tasks = [] for req in requests_list: if args.request_rate != float("inf"): - # Simple rate limiting - interval = 1.0 / args.request_rate + # Poisson process: inter-arrival times follow exponential distribution + interval = np.random.exponential(1.0 / args.request_rate) await asyncio.sleep(interval) task = asyncio.create_task(limited_request_func(req, session, pbar)) @@ -665,19 +683,43 @@ async def benchmark(args): # Calculate metrics metrics = calculate_metrics(outputs, total_duration) - print("\n" + "=" * 40) - print("Benchmark Results") - print("=" * 40) - print(f"Backend: {args.backend}") - print(f"Model: {args.model}") - print(f"Dataset: {args.dataset}") - print(f"Total Duration: {metrics['duration']:.2f} s") - print(f"Throughput: {metrics['throughput_qps']:.2f} req/s") - print(f"Success Rate: {metrics['completed_requests']}/{len(requests_list)}") - print(f"Latency Mean: {metrics['latency_mean']:.4f} s") - print(f"Latency Median: {metrics['latency_median']:.4f} s") - print(f"Latency P99: {metrics['latency_p99']:.4f} s") - print("=" * 40) + print("\n{s:{c}^{n}}".format(s=" Serving Benchmark Result ", n=60, c="=")) + + # Section 1: Configuration + print("{:<40} {:<15}".format("Backend:", args.backend)) + print("{:<40} {:<15}".format("Model:", args.model)) + print("{:<40} {:<15}".format("Dataset:", args.dataset)) + print("{:<40} {:<15}".format("Task:", args.task)) + + # Section 2: Execution & Traffic + print(f"{'-' * 50}") + print("{:<40} {:<15.2f}".format("Benchmark duration (s):", metrics["duration"])) + print("{:<40} {:<15}".format("Request rate:", str(args.request_rate))) + print( + "{:<40} {:<15}".format( + "Max request concurrency:", + str(args.max_concurrency) if args.max_concurrency else "not set", + ) + ) + print( + "{:<40} {}/{:<15}".format( + "Successful requests:", metrics["completed_requests"], len(requests_list) + ) + ) + + # Section 3: Performance Metrics + print(f"{'-' * 50}") + + print( + "{:<40} {:<15.2f}".format( + "Request throughput (req/s):", metrics["throughput_qps"] + ) + ) + print("{:<40} {:<15.4f}".format("Latency Mean (s):", metrics["latency_mean"])) + print("{:<40} {:<15.4f}".format("Latency Median (s):", metrics["latency_median"])) + print("{:<40} {:<15.4f}".format("Latency P99 (s):", metrics["latency_p99"])) + + print("\n" + "=" * 60) if args.output_file: with open(args.output_file, "w") as f: @@ -729,10 +771,24 @@ if __name__ == "__main__": "--num-prompts", type=int, default=10, help="Number of prompts to benchmark." ) parser.add_argument( - "--max-concurrency", type=int, default=10, help="Maximum concurrent requests." + "--max-concurrency", + type=int, + default=1, + help="Maximum number of concurrent requests, default to `1`. This can be used " + "to help simulate an environment where a higher level component " + "is enforcing a maximum number of concurrent requests. While the " + "--request-rate argument controls the rate at which requests are " + "initiated, this argument will control how many are actually allowed " + "to execute at a time. This means that when used in combination, the " + "actual request rate may be lower than specified with --request-rate, " + "if the server is not processing requests fast enough to keep up.", ) parser.add_argument( - "--request-rate", type=float, default=float("inf"), help="Request rate (req/s)." + "--request-rate", + type=float, + default=float("inf"), + help="Number of requests per second. If this is inf, then all the requests are sent at time 0. " + "Otherwise, we use Poisson process to synthesize the request arrival times. Default is inf.", ) parser.add_argument("--width", type=int, default=None, help="Image/Video width.") parser.add_argument("--height", type=int, default=None, help="Image/Video height.") diff --git a/python/sglang/multimodal_gen/runtime/entrypoints/openai/common_api.py b/python/sglang/multimodal_gen/runtime/entrypoints/openai/common_api.py index 676520aa3..fc7623a42 100644 --- a/python/sglang/multimodal_gen/runtime/entrypoints/openai/common_api.py +++ b/python/sglang/multimodal_gen/runtime/entrypoints/openai/common_api.py @@ -8,6 +8,7 @@ from sglang.multimodal_gen.runtime.entrypoints.openai.utils import ( UnmergeLoraWeightsReq, ) from sglang.multimodal_gen.runtime.scheduler_client import async_scheduler_client +from sglang.multimodal_gen.runtime.server_args import get_global_server_args from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger router = APIRouter(prefix="/v1") @@ -95,3 +96,16 @@ async def unmerge_lora_weights( f"Successfully unmerged LoRA weights (target: {target})", "Failed to unmerge LoRA weights", ) + + +@router.get("/model_info") +async def model_info(): + """Get the model information.""" + server_args = get_global_server_args() + if not server_args: + raise HTTPException(status_code=500, detail="Server args not initialized") + + result = { + "model_path": server_args.model_path, + } + return result