chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 11 Packages Projects Releases Wiki Activity

CI: add server performance test for SGLang diffusion (#13091)

Browse Source 
Co-authored-by: Mick <mickjagger19@icloud.com>
This commit is contained in:
Adarsh Shirawalmath
2025-11-15 07:41:19 +05:30
committed by GitHub
parent a5be6ef98e
commit af373636da
13 changed files with 901 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
python/sglang/multimodal_gen/runtime/managers/gpu_worker.py
View File

@@ -28,7 +28,6 @@ from sglang.multimodal_gen.runtime.utils.logging_utils import (
logger = init_logger(__name__)
# ANSI color codes
CYAN = "\033[1;36m"
RESET = "\033[0;0m"
@@ -96,6 +95,16 @@ class GPUWorker:
req = batch[0]
output_batch = self.pipeline.forward(req, server_args)
if req.perf_logger:
logging_info = getattr(output_batch, "logging_info", None) or getattr(
req, "logging_info", None
)
if logging_info:
try:
req.perf_logger.log_stage_metrics(logging_info)
except Exception:
logger.exception(
"Failed to log stage metrics for request %s", req.request_id
)
req.perf_logger.log_total_duration("total_inference_time")
return output_batch

8
python/sglang/multimodal_gen/runtime/models/encoders/qwen2_5vl.py
View File

@@ -54,7 +54,13 @@ from sglang.multimodal_gen.runtime.utils.common import add_prefix
# limitations under the License.
"""Inference-only Qwen2-VL model compatible with HuggingFace weights."""
import logging
from typing import Callable, Iterable, Optional, Tuple, Union, Unpack
from typing import Callable, Iterable, Optional, Tuple, Union
try:
from typing import Unpack # type: ignore[attr-defined]
except ImportError:
# Python 3.10 and below
from typing_extensions import Unpack
import torch
import torch.nn as nn

3
python/sglang/multimodal_gen/runtime/pipelines/schedule_batch.py
View File

@@ -193,6 +193,9 @@ class Req:
VSA_sparsity: float = 0.0
perf_logger: PerformanceLogger | None = None
# stage logging
logging_info: PipelineLoggingInfo = field(default_factory=PipelineLoggingInfo)
# profile
profile: bool = False
num_profiled_timesteps: int = 8

24
python/sglang/multimodal_gen/runtime/pipelines/stages/base.py
View File

@@ -185,6 +185,8 @@ class PipelineStage(ABC):
raise
# Execute the actual stage logic
logging_info = getattr(batch, "logging_info", None)
if envs.SGL_DIFFUSION_STAGE_LOGGING:
logger.info("[%s] Starting execution", stage_name)
start_time = time.perf_counter()
@@ -197,7 +199,27 @@ class PipelineStage(ABC):
stage_name,
execution_time * 1000,
)
batch.logging_info.add_stage_execution_time(stage_name, execution_time)
if logging_info is not None:
try:
logging_info.add_stage_execution_time(
stage_name, execution_time
)
except Exception:
logger.warning(
"[%s] Failed to record stage timing on batch.logging_info",
stage_name,
exc_info=True,
)
perf_logger = getattr(batch, "perf_logger", None)
if perf_logger is not None:
try:
perf_logger.log_stage_metric(stage_name, execution_time * 1000)
except Exception:
logger.warning(
"[%s] Failed to log stage metric to performance logger",
stage_name,
exc_info=True,
)
except Exception as e:
execution_time = time.perf_counter() - start_time
logger.error(

65
python/sglang/multimodal_gen/runtime/utils/performance_logger.py
View File

@@ -6,11 +6,16 @@ import os
import subprocess
import time
from datetime import datetime
from typing import Any
from dateutil.tz import UTC
project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../"))
LOG_DIR = os.path.join(project_root, "logs")
LOG_DIR = os.environ.get("SGLANG_PERF_LOG_DIR")
if LOG_DIR:
LOG_DIR = os.path.abspath(LOG_DIR)
else:
project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../"))
LOG_DIR = os.path.join(project_root, "logs")
# Configure a specific logger for performance metrics
perf_logger = logging.getLogger("performance")
@@ -74,3 +79,59 @@ class PerformanceLogger:
"steps": self.step_timings,
}
perf_logger.info(json.dumps(log_entry))
def log_stage_metric(self, stage_name: str, duration_ms: float):
"""Logs a single pipeline stage timing entry."""
log_entry = {
"timestamp": datetime.now(UTC).isoformat(),
"request_id": self.request_id,
"commit_hash": self.commit_hash,
"tag": "pipeline_stage_metric",
"stage": stage_name,
"duration_ms": duration_ms,
}
perf_logger.info(json.dumps(log_entry))
def log_stage_metrics(self, stages: Any):
"""
Persist per-stage execution stats to performance.log.
Args:
stages: Either a PipelineLoggingInfo instance or any object exposing
a mapping of stage metadata via a `stages` attribute/dict.
"""
if stages is None:
return
if hasattr(stages, "stages"):
stage_items = getattr(stages, "stages", {}).items()
elif isinstance(stages, dict):
stage_items = stages.items()
else:
return
formatted_stages: list[dict[str, Any]] = []
for name, info in stage_items:
if not info:
continue
entry = {"name": name}
execution_time = info.get("execution_time")
if execution_time is not None:
entry["execution_time_ms"] = execution_time * 1000
for key, value in info.items():
if key == "execution_time":
continue
entry[key] = value
formatted_stages.append(entry)
if not formatted_stages:
return
log_entry = {
"timestamp": datetime.now(UTC).isoformat(),
"request_id": self.request_id,
"commit_hash": self.commit_hash,
"tag": "pipeline_stage_metrics",
"stages": formatted_stages,
}
perf_logger.info(json.dumps(log_entry))

53
python/sglang/multimodal_gen/test/server/conftest.py Normal file
View File

@@ -0,0 +1,53 @@
_GLOBAL_PERF_RESULTS = []
def pytest_sessionfinish(session):
"""
This hook is called by pytest at the end of the entire test session.
It prints a consolidated summary of all performance results.
"""
if not _GLOBAL_PERF_RESULTS:
return
print("\n\n" + "=" * 35 + " Performance Summary " + "=" * 35)
print(
f"{'Test Suite':<30} | {'Test Name':<20} | {'E2E (ms)':>12} | {'Avg Denoise (ms)':>18} | {'Median Denoise (ms)':>20}"
)
print(
"-" * 30
+ "-+-"
+ "-" * 20
+ "-+-"
+ "-" * 12
+ "-+-"
+ "-" * 18
+ "-+-"
+ "-" * 20
)
for entry in sorted(_GLOBAL_PERF_RESULTS, key=lambda x: x["class_name"]):
print(
f"{entry['class_name']:<30} | {entry['test_name']:<20} | {entry['e2e_ms']:>12.2f} | "
f"{entry['avg_denoise_ms']:>18.2f} | {entry['median_denoise_ms']:>20.2f}"
)
print("=" * 91)
print("\n\n" + "=" * 36 + " Detailed Reports " + "=" * 37)
for entry in sorted(_GLOBAL_PERF_RESULTS, key=lambda x: x["class_name"]):
print(f"\n--- Details for {entry['class_name']} / {entry['test_name']} ---")
stage_report = ", ".join(
f"{name}:{duration:.2f}ms"
for name, duration in entry.get("stage_metrics", {}).items()
)
if stage_report:
print(f" Stages: {stage_report}")
sampled_steps = entry.get("sampled_steps") or {}
if sampled_steps:
step_report = ", ".join(
f"{idx}:{duration:.2f}ms"
for idx, duration in sorted(sampled_steps.items())
)
print(f" Sampled Steps: {step_report}")
print("=" * 91)

82
python/sglang/multimodal_gen/test/server/perf_baselines.json Normal file
View File

@@ -0,0 +1,82 @@
{
"metadata": {
"model": "Qwen/Qwen-Image",
"hardware": "CI H100 80GB pool",
"description": "Reference numbers captured from the CI diffusion server baseline run"
},
"tolerances": {
"e2e": 0.25,
"stage": 0.3,
"denoise_step": 0.1,
"denoise_agg": 0.1
},
"sampling": {
"step_fractions": [
0.0,
0.2,
0.4,
0.6,
0.8,
1.0
],
"warmup_requests": {
"text": 1,
"image_edit": 0
}
},
"scenarios": {
"text_to_image": {
"notes": "Single-image generation using the default prompt",
"expected_e2e_ms": 74500.0,
"expected_avg_denoise_ms": 422.42,
"expected_median_denoise_ms": 410.62,
"stages_ms": {
"InputValidationStage": 0.1,
"TextEncodingStage": 834.2,
"ConditioningStage": 0.1,
"TimestepPreparationStage": 10.6,
"LatentPreparationStage": 5.2,
"DenoisingStage": 21202.6,
"DecodingStage": 476.12
},
"denoise_step_ms": {
"0": 1077.77, "1": 345.13, "2": 413.8, "3": 405.49, "4": 408.14, "5": 409.06,
"6": 408.85, "7": 410.53, "8": 407.51, "9": 409.44, "10": 408.65, "11": 410.14,
"12": 411.74, "13": 409.59, "14": 409.17, "15": 410.78, "16": 410.66, "17": 410.58,
"18": 411.27, "19": 410.51, "20": 409.03, "21": 410.16, "22": 409.42, "23": 411.03,
"24": 410.18, "25": 409.72, "26": 410.26, "27": 410.21, "28": 410.71, "29": 410.76,
"30": 411.06, "31": 410.1, "32": 410.55, "33": 410.77, "34": 410.74, "35": 411.75,
"36": 410.78, "37": 411.56, "38": 410.85, "39": 411.08, "40": 411.12, "41": 411.1,
"42": 411.09, "43": 410.87, "44": 411.37, "45": 411.68, "46": 411.0, "47": 410.09,
"48": 412.72, "49": 410.42
}
},
"image_edit": {
"notes": "single uploaded reference image, Qwen/Qwen-Image-Edit",
"expected_e2e_ms": 138500.0,
"expected_avg_denoise_ms": 720.0,
"expected_median_denoise_ms": 718.0,
"stages_ms": {
"InputValidationStage": 14,
"ImageEncodingStage": 990.0,
"ImageVAEEncodingStage": 252.76,
"ConditioningStage": 0.13,
"TimestepPreparationStage": 13.78,
"LatentPreparationStage": 9.18,
"DenoisingStage": 36000.0,
"DecodingStage": 645
},
"denoise_step_ms": {
"0": 720.0, "1": 720.0, "2": 720.0, "3": 720.0, "4": 720.0, "5": 720.0,
"6": 720.0, "7": 720.0, "8": 720.0, "9": 720.0, "10": 720.0, "11": 720.0,
"12": 720.0, "13": 720.0, "14": 720.0, "15": 720.0, "16": 720.0, "17": 720.0,
"18": 720.0, "19": 720.0, "20": 720.0, "21": 720.0, "22": 720.0, "23": 720.0,
"24": 720.0, "25": 720.0, "26": 720.0, "27": 720.0, "28": 720.0, "29": 720.0,
"30": 720.0, "31": 720.0, "32": 720.0, "33": 720.0, "34": 720.0, "35": 720.0,
"36": 720.0, "37": 720.0, "38": 720.0, "39": 720.0, "40": 720.0, "41": 720.0,
"42": 720.0, "43": 720.0, "44": 720.0, "45": 720.0, "46": 720.0, "47": 720.0,
"48": 720.0, "49": 720.0
}
}
}
}

490
python/sglang/multimodal_gen/test/server/test_server_performance.py Normal file
View File

@@ -0,0 +1,490 @@
# Server-based diffusion performance test:
# - Launches an sglang diffusion server via the CLI.
# - Issues an OpenAI-compatible Images API request.
# - Extracts all performance metrics from performance.log (no stdout parsing).
# - Verifies E2E, stage-level, and denoising-step latencies with configurable buffers.
from __future__ import annotations
import base64
import json
import os
import statistics
import subprocess
import tempfile
import time
from pathlib import Path
from typing import Any, Sequence
import pytest
from openai import OpenAI
from sglang.multimodal_gen.runtime.utils.common import kill_process_tree
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
from sglang.multimodal_gen.test.server.conftest import _GLOBAL_PERF_RESULTS
from sglang.multimodal_gen.test.test_utils import (
get_dynamic_server_port,
is_jpeg,
is_png,
prepare_perf_log,
read_perf_records,
sample_step_indices,
wait_for_perf_record,
wait_for_stage_metrics,
)
logger = init_logger(__name__)
_BASELINE_PATH = Path(__file__).with_name("perf_baselines.json")
with _BASELINE_PATH.open("r", encoding="utf-8") as _fh:
_BASELINE_CONFIG = json.load(_fh)
_SCENARIOS = _BASELINE_CONFIG["scenarios"]
_TEXT_SCENARIO = _SCENARIOS["text_to_image"]
_IMAGE_EDIT_SCENARIO = _SCENARIOS["image_edit"]
STEP_SAMPLE_FRACTIONS: Sequence[float] = tuple(
_BASELINE_CONFIG["sampling"]["step_fractions"]
)
_WARMUP_DEFAULTS = _BASELINE_CONFIG["sampling"].get("warmup_requests", {})
_DEFAULT_WARMUP_TEXT = int(_WARMUP_DEFAULTS.get("text", 1))
_DEFAULT_WARMUP_EDIT = int(_WARMUP_DEFAULTS.get("image_edit", 0))
_TOLERANCES = _BASELINE_CONFIG["tolerances"]
def _tolerance_from_env(var_name: str, default: float) -> float:
override = os.environ.get(var_name)
if override is not None:
return float(override)
return float(default)
E2E_TOLERANCE_RATIO = _tolerance_from_env("SGLANG_E2E_TOLERANCE", _TOLERANCES["e2e"])
STAGE_TOLERANCE_RATIO = _tolerance_from_env(
"SGLANG_STAGE_TIME_TOLERANCE", _TOLERANCES["stage"]
)
DENOISE_STEP_TOLERANCE_RATIO = _tolerance_from_env(
"SGLANG_DENOISE_STEP_TOLERANCE", _TOLERANCES["denoise_step"]
)
DENOISE_AGG_TOLERANCE_RATIO = _tolerance_from_env(
"SGLANG_DENOISE_AGG_TOLERANCE", _TOLERANCES["denoise_agg"]
)
def _decode_and_validate_image(b64_json: str) -> None:
image_bytes = base64.b64decode(b64_json)
assert is_png(image_bytes) or is_jpeg(
image_bytes
), "Warm-up image must be PNG or JPEG"
def _run_warmup_requests(cls, port: int) -> None:
warmup_text_requests = int(getattr(cls, "WARMUP_TEXT_REQUESTS", 1))
warmup_edit_requests = int(getattr(cls, "WARMUP_IMAGE_EDIT_REQUESTS", 0))
if warmup_text_requests <= 0 and warmup_edit_requests <= 0:
return
client = OpenAI(
api_key="sglang-anything",
base_url=f"http://localhost:{port}/v1",
)
prompt = getattr(cls, "PROMPT", "A colorful raccoon icon")
output_size = getattr(cls, "OUTPUT_SIZE", "1024x1024")
logger.info(
"[server-test] Running %s text warm-up(s) and %s edit warm-up(s)",
warmup_text_requests,
warmup_edit_requests,
)
for _ in range(warmup_text_requests):
result = client.images.generate(
model=getattr(cls, "MODEL_PATH"),
prompt=prompt,
n=1,
size=output_size,
response_format="b64_json",
)
_decode_and_validate_image(result.data[0].b64_json)
if warmup_edit_requests > 0:
edit_prompt = getattr(cls, "IMAGE_EDIT_PROMPT", None)
edit_path: Path | None = getattr(cls, "IMAGE_EDIT_PATH", None)
if not edit_prompt or not edit_path or not edit_path.exists():
logger.warning(
"[server-test] Skipping image-edit warm-up: prompt=%s path=%s exists=%s",
bool(edit_prompt),
edit_path,
edit_path.exists() if edit_path else False,
)
return
for _ in range(warmup_edit_requests):
with edit_path.open("rb") as fh:
result = client.images.edit(
model=getattr(cls, "MODEL_PATH"),
image=fh,
prompt=edit_prompt,
n=1,
size=output_size,
response_format="b64_json",
)
_decode_and_validate_image(result.data[0].b64_json)
@pytest.fixture(scope="class")
def diffusion_server(request):
cls = request.cls
log_dir, perf_log_path = prepare_perf_log(Path(__file__))
default_port = get_dynamic_server_port()
port = int(os.environ.get("SGLANG_TEST_SERVER_PORT", default_port))
port = getattr(cls, "SERVER_PORT", port)
model = getattr(cls, "MODEL_PATH")
wait_deadline = float(os.environ.get("SGLANG_TEST_WAIT_SECS", "1200"))
serve_extra_args = os.environ.get("SGLANG_TEST_SERVE_ARGS", "")
safe_model_name = model.replace("/", "_")
stdout_path = (
Path(tempfile.gettempdir()) / f"sgl_server_{port}_{safe_model_name}.log"
)
stdout_path.unlink(missing_ok=True)
base_command = [
"sglang",
"serve",
"--model-path",
model,
"--port",
str(port),
"--log-level=debug",
]
if serve_extra_args.strip():
base_command += serve_extra_args.strip().split()
env = os.environ.copy()
env["SGL_DIFFUSION_STAGE_LOGGING"] = "1"
env["SGLANG_PERF_LOG_DIR"] = log_dir.as_posix()
stdout_fh = stdout_path.open("w", encoding="utf-8", buffering=1)
process = subprocess.Popen(
base_command,
stdout=stdout_fh,
stderr=subprocess.STDOUT,
text=True,
bufsize=1,
env=env,
)
logger.info(
"[server-test] Starting diffusion server pid=%s, model=%s, log=%s",
process.pid,
model,
stdout_path.as_posix(),
)
start = time.time()
server_ready_message = "Application startup complete."
server_ready = False
while time.time() - start < wait_deadline:
if process.poll() is not None:
tail = ""
try:
tail = "\n".join(
stdout_path.read_text(
encoding="utf-8", errors="ignore"
).splitlines()[-200:]
)
except Exception:
pass
raise RuntimeError(
f"Server exited early (code {process.returncode}). Last logs:\n{tail}"
)
if stdout_path.exists():
try:
log_content = stdout_path.read_text(encoding="utf-8", errors="ignore")
if server_ready_message in log_content:
logger.info("[server-test] Server is fully loaded and ready.")
server_ready = True
break
except Exception as e:
logger.debug("Could not read server log file yet: %s", e)
logger.info(
"[server-test] Waiting for server to initialize... elapsed=%ss",
int(time.time() - start),
)
time.sleep(5)
if not server_ready:
tail = ""
try:
tail = "\n".join(
stdout_path.read_text(encoding="utf-8", errors="ignore").splitlines()[
-200:
]
)
except Exception:
pass
raise TimeoutError(
f"Server did not become ready within {wait_deadline}s. Last logs:\n{tail}"
)
ctx = {
"port": port,
"stdout_file": stdout_path,
"process": process,
"model": model,
"fh": stdout_fh,
"perf_log_path": perf_log_path,
"log_dir": log_dir,
}
request.cls.server_ctx = ctx
request.cls.perf_log_path = perf_log_path
grace = float(getattr(cls, "STARTUP_GRACE_SECONDS", 0.0) or 0.0)
if grace > 0:
logger.info(
"[server-test] Waiting %.1fs before warm-ups to let model settle", grace
)
time.sleep(grace)
try:
_run_warmup_requests(cls, port)
except Exception as exc:
logger.error("Warm-up requests failed: %s", exc)
kill_process_tree(process.pid)
raise
yield ctx
try:
kill_process_tree(process.pid)
except Exception:
pass
try:
stdout_fh.flush()
stdout_fh.close()
except Exception:
pass
@pytest.mark.usefixtures("diffusion_server")
class DiffusionPerfTestBase:
MODEL_PATH: str
# SERVER_PORT = int(os.environ.get("SGLANG_TEST_SERVER_PORT", "30100"))
PROMPT = "A Logo With Bold Large Text: SGL Diffusion"
IMAGE_EDIT_PROMPT: str | None = None
IMAGE_EDIT_PATH = Path(__file__).resolve().parents[1] / "test_files" / "girl.jpg"
OUTPUT_SIZE = "1024x1024"
WARMUP_TEXT_REQUESTS = _DEFAULT_WARMUP_TEXT
WARMUP_IMAGE_EDIT_REQUESTS = _DEFAULT_WARMUP_EDIT
STARTUP_GRACE_SECONDS = 0.0
STAGE_EXPECTATIONS: dict
STEP_EXPECTATIONS: dict
EXPECTED_E2E_MS: float
EXPECTED_AVG_DENOISE_MS: float
EXPECTED_MEDIAN_DENOISE_MS: float
_perf_results: list[dict[str, Any]] = []
@classmethod
def setup_class(cls):
cls._perf_results = []
@classmethod
def teardown_class(cls):
for result in cls._perf_results:
result["class_name"] = cls.__name__
_GLOBAL_PERF_RESULTS.append(result)
def _client(self) -> OpenAI:
return OpenAI(
api_key="sglang-anything",
base_url=f"http://localhost:{self.server_ctx['port']}/v1",
)
def _perf_log_path(self) -> Path:
return self.server_ctx["perf_log_path"]
def _record_result(self, test_name: str, summary: dict[str, Any]) -> None:
if not summary:
return
entry = {"test_name": test_name, **summary}
self.__class__._perf_results.append(entry)
def _run_and_collect_records(self, generate_fn) -> tuple[dict, dict]:
log_path = self._perf_log_path()
prev_len = len(read_perf_records(log_path))
generate_fn()
perf_record, _ = wait_for_perf_record(
"total_inference_time",
prev_len,
log_path,
)
stage_metrics, _ = wait_for_stage_metrics(
perf_record.get("request_id", ""),
prev_len,
len(self.STAGE_EXPECTATIONS),
log_path,
)
return perf_record, stage_metrics
def _generate_image(self):
client = self._client()
result = client.images.generate(
model=self.MODEL_PATH,
prompt=self.PROMPT,
n=1,
size=self.OUTPUT_SIZE,
response_format="b64_json",
)
image_bytes = base64.b64decode(result.data[0].b64_json)
assert is_png(image_bytes) or is_jpeg(
image_bytes
), "Generated image must be PNG or JPEG"
def _generate_image_edit(self):
if not self.IMAGE_EDIT_PROMPT:
pytest.skip("Image edit prompt not configured")
if not self.IMAGE_EDIT_PATH.exists():
pytest.skip(f"Image edit file missing: {self.IMAGE_EDIT_PATH}")
client = self._client()
with self.IMAGE_EDIT_PATH.open("rb") as fh:
result = client.images.edit(
model=self.MODEL_PATH,
image=fh,
prompt=self.IMAGE_EDIT_PROMPT,
n=1,
size=self.OUTPUT_SIZE,
response_format="b64_json",
)
image_bytes = base64.b64decode(result.data[0].b64_json)
assert is_png(image_bytes) or is_jpeg(
image_bytes
), "Edited image must be PNG or JPEG"
def _assert_metrics(self, perf_record: dict, stage_metrics: dict):
e2e_ms = float(perf_record.get("total_duration_ms", 0.0))
assert e2e_ms > 0, "E2E duration missing from perf log"
e2e_upper = self.EXPECTED_E2E_MS * (1 + E2E_TOLERANCE_RATIO)
assert (
e2e_ms <= e2e_upper
), f"E2E time {e2e_ms:.2f}ms exceeds allowed {e2e_upper:.2f}ms"
steps = [
step
for step in perf_record.get("steps", []) or []
if step.get("name") == "denoising_step_guided" and "duration_ms" in step
]
assert steps, "Denoising step timings missing from perf log"
durations = [float(step["duration_ms"]) for step in steps]
avg_duration = sum(durations) / len(durations)
median_duration = statistics.median(durations)
avg_upper = self.EXPECTED_AVG_DENOISE_MS * (1 + DENOISE_AGG_TOLERANCE_RATIO)
med_upper = self.EXPECTED_MEDIAN_DENOISE_MS * (1 + DENOISE_AGG_TOLERANCE_RATIO)
assert (
avg_duration <= avg_upper
), f"Avg denoise {avg_duration:.2f}ms exceeds {avg_upper:.2f}ms"
assert (
median_duration <= med_upper
), f"Median denoise {median_duration:.2f}ms exceeds {med_upper:.2f}ms"
avg_per_step = {
int(step.get("index")): float(step["duration_ms"])
for step in steps
if step.get("index") is not None
}
sample_indices = sample_step_indices(avg_per_step, STEP_SAMPLE_FRACTIONS)
sampled_steps = {idx: avg_per_step[idx] for idx in sample_indices}
for idx in sample_indices:
expected = self.STEP_EXPECTATIONS.get(idx)
if expected is None:
continue
actual = avg_per_step[idx]
upper_bound = expected * (1 + DENOISE_STEP_TOLERANCE_RATIO)
assert (
actual <= upper_bound
), f"Denoise step {idx} took {actual:.2f}ms > allowed {upper_bound:.2f}ms"
assert stage_metrics, "Stage metrics missing from performance log"
for stage, expected in self.STAGE_EXPECTATIONS.items():
actual = stage_metrics.get(stage)
assert actual is not None, f"Stage {stage} timing missing"
upper_bound = expected * (1 + STAGE_TOLERANCE_RATIO)
assert (
actual <= upper_bound
), f"Stage {stage} took {actual:.2f}ms > allowed {upper_bound:.2f}ms"
# Log to pytest console during the run for immediate feedback
logger.info(
"[Perf] %s/%s: E2E %.2f ms; Avg denoise %.2f ms; Median %.2f ms",
self.__class__.__name__,
perf_record.get("test_name", "test"),
e2e_ms,
avg_duration,
median_duration,
)
return {
"e2e_ms": e2e_ms,
"avg_denoise_ms": avg_duration,
"median_denoise_ms": median_duration,
"stage_metrics": stage_metrics,
"sampled_steps": sampled_steps,
}
class TestQwenImageGeneration(DiffusionPerfTestBase):
"""Performance tests for the Qwen/Qwen-image model."""
MODEL_PATH = "Qwen/Qwen-Image"
STARTUP_GRACE_SECONDS = 30.0
WARMUP_IMAGE_EDIT_REQUESTS = 0
STAGE_EXPECTATIONS = _TEXT_SCENARIO["stages_ms"]
STEP_EXPECTATIONS = {
int(k): v for k, v in _TEXT_SCENARIO["denoise_step_ms"].items()
}
EXPECTED_E2E_MS = float(_TEXT_SCENARIO["expected_e2e_ms"])
EXPECTED_AVG_DENOISE_MS = float(_TEXT_SCENARIO["expected_avg_denoise_ms"])
EXPECTED_MEDIAN_DENOISE_MS = float(_TEXT_SCENARIO["expected_median_denoise_ms"])
def test_text_to_image_performance(self):
perf_record, stage_metrics = self._run_and_collect_records(self._generate_image)
summary = self._assert_metrics(perf_record, stage_metrics)
self._record_result("text_to_image", summary)
class TestQwenImageEdit(DiffusionPerfTestBase):
"""Performance tests for the Qwen/Qwen-Image-Edit model."""
MODEL_PATH = "Qwen/Qwen-Image-Edit"
IMAGE_EDIT_PROMPT = "Convert 2D style to 3D style"
OUTPUT_SIZE = "1024x1536"
STARTUP_GRACE_SECONDS = 30.0
WARMUP_TEXT_REQUESTS = 0
WARMUP_IMAGE_EDIT_REQUESTS = 1
STAGE_EXPECTATIONS = _IMAGE_EDIT_SCENARIO["stages_ms"]
STEP_EXPECTATIONS = {
int(k): v for k, v in _IMAGE_EDIT_SCENARIO["denoise_step_ms"].items()
}
EXPECTED_E2E_MS = float(_IMAGE_EDIT_SCENARIO["expected_e2e_ms"])
EXPECTED_AVG_DENOISE_MS = float(_IMAGE_EDIT_SCENARIO["expected_avg_denoise_ms"])
EXPECTED_MEDIAN_DENOISE_MS = float(
_IMAGE_EDIT_SCENARIO["expected_median_denoise_ms"]
)
def test_image_edit_performance(self):
perf_record, stage_metrics = self._run_and_collect_records(
self._generate_image_edit
)
summary = self._assert_metrics(perf_record, stage_metrics)
self._record_result("image_edit", summary)

BIN
python/sglang/multimodal_gen/test/test_files/girl.jpg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 212 KiB

BIN
python/sglang/multimodal_gen/test/test_files/rabbit.jpg
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 262 KiB

138
python/sglang/multimodal_gen/test/test_utils.py
View File

@@ -1,5 +1,6 @@
# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo
import dataclasses
import json
import os
import shlex
import socket
@@ -7,11 +8,13 @@ import subprocess
import sys
import time
import unittest
from typing import Optional
from pathlib import Path
from typing import Optional, Sequence
from PIL import Image
from sglang.multimodal_gen.configs.sample.base import DataType
from sglang.multimodal_gen.runtime.utils.common import get_bool_env_var
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
logger = init_logger(__name__)
@@ -52,11 +55,37 @@ def probe_port(host="127.0.0.1", port=30010, timeout=2.0) -> bool:
return False
def is_in_ci() -> bool:
return get_bool_env_var("SGLANG_IS_IN_CI")
def get_dynamic_server_port() -> int:
cuda_devices = os.environ.get("CUDA_VISIBLE_DEVICES", "0")
if not cuda_devices:
cuda_devices = "0"
try:
first_device_id = int(cuda_devices.split(",")[0].strip()[0])
except (ValueError, IndexError):
first_device_id = 0
if is_in_ci():
base_port = 10000 + first_device_id * 2000
else:
base_port = 20000 + first_device_id * 1000
return base_port + 1000
def is_mp4(data):
idx = data.find(b"ftyp")
return 0 <= idx <= 32
def is_jpeg(data: bytes) -> bool:
# JPEG files start with: FF D8 FF
return data.startswith(b"\xff\xd8\xff")
def is_png(data):
# PNG files start with: 89 50 4E 47 0D 0A 1A 0A
return data.startswith(b"\x89PNG\r\n\x1a\n")
@@ -77,6 +106,113 @@ def check_image_size(ut, image, width, height):
ut.assertEqual(image.size, (width, height))
def get_perf_log_dir(start_file: Path) -> Path:
"""Mirror runtime/utils/performance_logger.py behaviour for locating logs."""
this_file = start_file.resolve()
root_logs = this_file.parents[3] / "logs"
fallback = this_file.parents[2] / "logs"
return root_logs if root_logs.exists() or not fallback.exists() else fallback
def _ensure_log_path(log_dir: Path) -> Path:
log_dir.mkdir(parents=True, exist_ok=True)
return log_dir / "performance.log"
def clear_perf_log(log_dir: Path) -> Path:
"""Delete the perf log file so tests can watch for fresh entries."""
log_path = _ensure_log_path(log_dir)
if log_path.exists():
log_path.unlink()
logger.info("[server-test] Monitoring perf log at %s", log_path.as_posix())
return log_path
def prepare_perf_log(start_file: Path) -> tuple[Path, Path]:
"""Convenience helper to resolve and clear the perf log in one call."""
log_dir = get_perf_log_dir(start_file)
log_path = clear_perf_log(log_dir)
return log_dir, log_path
def read_perf_records(log_path: Path) -> list[dict]:
if not log_path.exists():
return []
records: list[dict] = []
with log_path.open("r", encoding="utf-8") as fh:
for line in fh:
line = line.strip()
if not line:
continue
try:
records.append(json.loads(line))
except json.JSONDecodeError:
continue
return records
def wait_for_perf_record(
tag: str,
prev_len: int,
log_path: Path,
timeout: float = 120.0,
) -> tuple[dict, int]:
deadline = time.time() + timeout
while time.time() < deadline:
records = read_perf_records(log_path)
if len(records) > prev_len:
for rec in records[prev_len:]:
if rec.get("tag") == tag:
return rec, len(records)
time.sleep(0.5)
raise AssertionError(
f"Timeout waiting for perf log entry '{tag}' (start_len={prev_len})"
)
def wait_for_stage_metrics(
request_id: str,
prev_len: int,
expected_count: int,
log_path: Path,
timeout: float = 120.0,
) -> tuple[dict[str, float], int]:
deadline = time.time() + timeout
metrics: dict[str, float] = {}
while time.time() < deadline:
records = read_perf_records(log_path)
for rec in records[prev_len:]:
if (
rec.get("tag") == "pipeline_stage_metric"
and rec.get("request_id") == request_id
):
stage = rec.get("stage")
duration = rec.get("duration_ms")
if stage is not None and duration is not None:
metrics[str(stage)] = float(duration)
if len(metrics) >= expected_count:
return metrics, len(records)
time.sleep(0.5)
raise AssertionError(
f"Timeout waiting for stage metrics for request {request_id} "
f"(collected={len(metrics)} expected={expected_count})"
)
def sample_step_indices(
step_map: dict[int, float], fractions: Sequence[float]
) -> list[int]:
if not step_map:
return []
max_idx = max(step_map.keys())
indices = set()
for fraction in fractions:
idx = min(max_idx, max(0, int(round(fraction * max_idx))))
if idx in step_map:
indices.add(idx)
return sorted(indices)
@dataclasses.dataclass
class TestResult:
name: str