[diffusion] bench: improve bench_serving by adding more controlling args (#15554)

python/sglang/multimodal_gen/benchmarks/bench_serving.py

@@ -612,6 +612,17 @@ async def benchmark(args):
     # Wait for service
     wait_for_service(args.base_url)
 
+    # Fetch model info
+    try:
+        resp = requests.get(f"{args.base_url}/v1/model_info", timeout=5)
+        if resp.status_code == 200:
+            info = resp.json()
+            if "model_path" in info and info["model_path"]:
+                args.model = info["model_path"]
+                print(f"Updated model name from server: {args.model}")
+    except Exception as e:
+        print(f"Failed to fetch model info: {e}. Using default: {args.model}")
+
     # Setup dataset
     if args.backend == "sglang-image":
         if args.task == "i2v":
@@ -632,14 +643,21 @@ async def benchmark(args):
     else:
         raise ValueError(f"Unknown dataset: {args.dataset}")
 
+    print(f"Loading requests...")
     requests_list = dataset.get_requests()
     print(f"Prepared {len(requests_list)} requests from {args.dataset} dataset.")
 
     # Limit concurrency
-    semaphore = asyncio.Semaphore(args.max_concurrency)
+    if args.max_concurrency is not None:
+        semaphore = asyncio.Semaphore(args.max_concurrency)
+    else:
+        semaphore = None
 
     async def limited_request_func(req, session, pbar):
-        async with semaphore:
+        if semaphore:
+            async with semaphore:
+                return await request_func(req, session, pbar)
+        else:
             return await request_func(req, session, pbar)
 
     # Run benchmark
@@ -650,8 +668,8 @@ async def benchmark(args):
         tasks = []
         for req in requests_list:
             if args.request_rate != float("inf"):
-                # Simple rate limiting
-                interval = 1.0 / args.request_rate
+                # Poisson process: inter-arrival times follow exponential distribution
+                interval = np.random.exponential(1.0 / args.request_rate)
                 await asyncio.sleep(interval)
 
             task = asyncio.create_task(limited_request_func(req, session, pbar))
@@ -665,19 +683,43 @@ async def benchmark(args):
     # Calculate metrics
     metrics = calculate_metrics(outputs, total_duration)
 
-    print("\n" + "=" * 40)
-    print("Benchmark Results")
-    print("=" * 40)
-    print(f"Backend: {args.backend}")
-    print(f"Model: {args.model}")
-    print(f"Dataset: {args.dataset}")
-    print(f"Total Duration: {metrics['duration']:.2f} s")
-    print(f"Throughput: {metrics['throughput_qps']:.2f} req/s")
-    print(f"Success Rate: {metrics['completed_requests']}/{len(requests_list)}")
-    print(f"Latency Mean: {metrics['latency_mean']:.4f} s")
-    print(f"Latency Median: {metrics['latency_median']:.4f} s")
-    print(f"Latency P99: {metrics['latency_p99']:.4f} s")
-    print("=" * 40)
+    print("\n{s:{c}^{n}}".format(s=" Serving Benchmark Result ", n=60, c="="))
+
+    # Section 1: Configuration
+    print("{:<40} {:<15}".format("Backend:", args.backend))
+    print("{:<40} {:<15}".format("Model:", args.model))
+    print("{:<40} {:<15}".format("Dataset:", args.dataset))
+    print("{:<40} {:<15}".format("Task:", args.task))
+
+    # Section 2: Execution & Traffic
+    print(f"{'-' * 50}")
+    print("{:<40} {:<15.2f}".format("Benchmark duration (s):", metrics["duration"]))
+    print("{:<40} {:<15}".format("Request rate:", str(args.request_rate)))
+    print(
+        "{:<40} {:<15}".format(
+            "Max request concurrency:",
+            str(args.max_concurrency) if args.max_concurrency else "not set",
+        )
+    )
+    print(
+        "{:<40} {}/{:<15}".format(
+            "Successful requests:", metrics["completed_requests"], len(requests_list)
+        )
+    )
+
+    # Section 3: Performance Metrics
+    print(f"{'-' * 50}")
+
+    print(
+        "{:<40} {:<15.2f}".format(
+            "Request throughput (req/s):", metrics["throughput_qps"]
+        )
+    )
+    print("{:<40} {:<15.4f}".format("Latency Mean (s):", metrics["latency_mean"]))
+    print("{:<40} {:<15.4f}".format("Latency Median (s):", metrics["latency_median"]))
+    print("{:<40} {:<15.4f}".format("Latency P99 (s):", metrics["latency_p99"]))
+
+    print("\n" + "=" * 60)
 
     if args.output_file:
         with open(args.output_file, "w") as f:
@@ -729,10 +771,24 @@ if __name__ == "__main__":
         "--num-prompts", type=int, default=10, help="Number of prompts to benchmark."
     )
     parser.add_argument(
-        "--max-concurrency", type=int, default=10, help="Maximum concurrent requests."
+        "--max-concurrency",
+        type=int,
+        default=1,
+        help="Maximum number of concurrent requests, default to `1`. This can be used "
+        "to help simulate an environment where a higher level component "
+        "is enforcing a maximum number of concurrent requests. While the "
+        "--request-rate argument controls the rate at which requests are "
+        "initiated, this argument will control how many are actually allowed "
+        "to execute at a time. This means that when used in combination, the "
+        "actual request rate may be lower than specified with --request-rate, "
+        "if the server is not processing requests fast enough to keep up.",
     )
     parser.add_argument(
-        "--request-rate", type=float, default=float("inf"), help="Request rate (req/s)."
+        "--request-rate",
+        type=float,
+        default=float("inf"),
+        help="Number of requests per second. If this is inf, then all the requests are sent at time 0. "
+        "Otherwise, we use Poisson process to synthesize the request arrival times. Default is inf.",
     )
     parser.add_argument("--width", type=int, default=None, help="Image/Video width.")
     parser.add_argument("--height", type=int, default=None, help="Image/Video height.")

python/sglang/multimodal_gen/runtime/entrypoints/openai/common_api.py

@@ -8,6 +8,7 @@ from sglang.multimodal_gen.runtime.entrypoints.openai.utils import (
     UnmergeLoraWeightsReq,
 )
 from sglang.multimodal_gen.runtime.scheduler_client import async_scheduler_client
+from sglang.multimodal_gen.runtime.server_args import get_global_server_args
 from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
 
 router = APIRouter(prefix="/v1")
@@ -95,3 +96,16 @@ async def unmerge_lora_weights(
         f"Successfully unmerged LoRA weights (target: {target})",
         "Failed to unmerge LoRA weights",
     )
+
+
+@router.get("/model_info")
+async def model_info():
+    """Get the model information."""
+    server_args = get_global_server_args()
+    if not server_args:
+        raise HTTPException(status_code=500, detail="Server args not initialized")
+
+    result = {
+        "model_path": server_args.model_path,
+    }
+    return result