chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 11 Packages Projects Releases Wiki Activity

[diffusion] CI: add script for automatically generation ci perf baseline (#16389)

Browse Source
This commit is contained in:
Xiaoyu Zhang
2026-01-05 13:18:35 +08:00
committed by GitHub
parent 561a3e04f6
commit 520c048d55
4 changed files with 330 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
python/sglang/multimodal_gen/docs/ci_perf.md Normal file
View File

@@ -0,0 +1,30 @@
## Perf baseline generation script
`python/sglang/multimodal_gen/test/scripts/gen_perf_baselines.py` starts a local diffusion server, issues requests for selected test cases, aggregates stage/denoise-step/E2E timings from the perf log, and writes the results back to the `scenarios` section of `perf_baselines.json`.
### Usage
Update a single case:
```bash
python python/sglang/multimodal_gen/test/scripts/gen_perf_baselines.py --case qwen_image_t2i
```
Select by regex:
```bash
python python/sglang/multimodal_gen/test/scripts/gen_perf_baselines.py --match 'qwen_image_.*'
```
Run all keys from the baseline file `scenarios`:
```bash
python python/sglang/multimodal_gen/test/scripts/gen_perf_baselines.py --all-from-baseline
```
Specify input/output paths and timeout:
```bash
python python/sglang/multimodal_gen/test/scripts/gen_perf_baselines.py --baseline python/sglang/multimodal_gen/test/server/perf_baselines.json --out /tmp/perf_baselines.json --timeout 600
```

235
python/sglang/multimodal_gen/test/scripts/gen_perf_baselines.py Normal file
View File

@@ -0,0 +1,235 @@
import argparse
import inspect
import json
import os
import re
import sys
from pathlib import Path
from openai import OpenAI
from sglang.multimodal_gen.test.server.test_server_utils import (
ServerManager,
WarmupRunner,
download_image_from_url,
get_generate_fn,
)
from sglang.multimodal_gen.test.server.testcase_configs import (
BASELINE_CONFIG,
DiffusionTestCase,
)
from sglang.multimodal_gen.test.test_utils import (
get_dynamic_server_port,
is_image_url,
wait_for_req_perf_record,
)
def _all_cases() -> list[DiffusionTestCase]:
import sglang.multimodal_gen.test.server.testcase_configs as cfg
cases: list[DiffusionTestCase] = []
for _, v in inspect.getmembers(cfg):
if isinstance(v, list) and v and isinstance(v[0], DiffusionTestCase):
cases.extend(v)
seen: set[str] = set()
out: list[DiffusionTestCase] = []
for c in cases:
if c.id not in seen:
seen.add(c.id)
out.append(c)
return out
def _baseline_path() -> Path:
import sglang.multimodal_gen.test.server.testcase_configs as cfg
return Path(cfg.__file__).with_name("perf_baselines.json")
def _openai_client(port: int) -> OpenAI:
return OpenAI(api_key="sglang-anything", base_url=f"http://localhost:{port}/v1")
def _build_server_extra_args(case: DiffusionTestCase) -> str:
a = os.environ.get("SGLANG_TEST_SERVE_ARGS", "")
a += f" --num-gpus {case.server_args.num_gpus}"
if case.server_args.tp_size is not None:
a += f" --tp-size {case.server_args.tp_size}"
if case.server_args.ulysses_degree is not None:
a += f" --ulysses-degree {case.server_args.ulysses_degree}"
if case.server_args.dit_layerwise_offload:
a += " --dit-layerwise-offload true"
if case.server_args.ring_degree is not None:
a += f" --ring-degree {case.server_args.ring_degree}"
if case.server_args.lora_path:
a += f" --lora-path {case.server_args.lora_path}"
if case.server_args.enable_warmup:
a += " --enable-warmup"
return a
def _build_env_vars(case: DiffusionTestCase) -> dict[str, str]:
if case.server_args.enable_cache_dit:
return {"SGLANG_CACHE_DIT_ENABLED": "true"}
return {}
def _torch_cleanup() -> None:
try:
import gc
gc.collect()
except Exception:
pass
try:
import torch
if torch.cuda.is_available():
torch.cuda.synchronize()
torch.cuda.empty_cache()
except Exception:
pass
def _run_case(case: DiffusionTestCase) -> dict:
default_port = get_dynamic_server_port()
port = int(os.environ.get("SGLANG_TEST_SERVER_PORT", default_port))
mgr = ServerManager(
model=case.server_args.model_path,
port=port,
wait_deadline=float(os.environ.get("SGLANG_TEST_WAIT_SECS", "1200")),
extra_args=_build_server_extra_args(case),
env_vars=_build_env_vars(case),
)
ctx = mgr.start()
try:
sp = case.sampling_params
output_size = os.environ.get("SGLANG_TEST_OUTPUT_SIZE", sp.output_size)
w = WarmupRunner(
port=ctx.port,
model=case.server_args.model_path,
prompt=sp.prompt or "A colorful raccoon icon",
output_size=output_size,
output_format=sp.output_format,
)
if case.server_args.warmup > 0:
if sp.image_path and sp.prompt:
image_path_list = sp.image_path
if not isinstance(image_path_list, list):
image_path_list = [image_path_list]
new_image_path_list = []
for p in image_path_list:
if is_image_url(p):
new_image_path_list.append(download_image_from_url(str(p)))
else:
pp = Path(p)
if not pp.exists():
raise FileNotFoundError(str(pp))
new_image_path_list.append(pp)
w.run_edit_warmups(
count=case.server_args.warmup,
edit_prompt=sp.prompt,
image_path=new_image_path_list,
)
else:
w.run_text_warmups(case.server_args.warmup)
client = _openai_client(ctx.port)
gen = get_generate_fn(
model_path=case.server_args.model_path,
modality=case.server_args.modality,
sampling_params=sp,
)
rid = gen(case.id, client)
rec = wait_for_req_perf_record(
rid,
ctx.perf_log_path,
timeout=float(os.environ.get("SGLANG_PERF_TIMEOUT", "300")),
)
if rec is None:
raise RuntimeError(f"missing perf record: {case.id}")
from sglang.multimodal_gen.test.server.testcase_configs import (
PerformanceSummary,
)
perf = PerformanceSummary.from_req_perf_record(
rec, BASELINE_CONFIG.step_fractions
)
if case.server_args.modality == "video" and sp.num_frames and sp.num_frames > 0:
if "per_frame_generation" not in perf.stage_metrics:
perf.stage_metrics["per_frame_generation"] = perf.e2e_ms / sp.num_frames
return {
"stages_ms": {k: round(v, 2) for k, v in perf.stage_metrics.items()},
"denoise_step_ms": {
str(k): round(v, 2) for k, v in perf.all_denoise_steps.items()
},
"expected_e2e_ms": round(perf.e2e_ms, 2),
"expected_avg_denoise_ms": round(perf.avg_denoise_ms, 2),
"expected_median_denoise_ms": round(perf.median_denoise_ms, 2),
}
finally:
ctx.cleanup()
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("--baseline", default="")
ap.add_argument("--out", default="")
ap.add_argument("--match", default="")
ap.add_argument("--case", action="append", default=[])
ap.add_argument("--all-from-baseline", action="store_true")
ap.add_argument("--timeout", type=float, default=300.0)
args = ap.parse_args()
os.environ.setdefault("SGLANG_GEN_BASELINE", "1")
os.environ["SGLANG_PERF_TIMEOUT"] = str(args.timeout)
baseline_path = Path(args.baseline) if args.baseline else _baseline_path()
out_path = Path(args.out) if args.out else baseline_path
data = json.loads(baseline_path.read_text(encoding="utf-8"))
scenarios = data.setdefault("scenarios", {})
ids = set(args.case) if args.case else None
pat = re.compile(args.match) if args.match else None
if args.all_from_baseline:
ids = set(scenarios.keys())
pat = None
all_cases = _all_cases()
cases = []
for c in all_cases:
if ids and c.id not in ids:
continue
if pat and not pat.search(c.id):
continue
cases.append(c)
if args.all_from_baseline and ids:
case_ids = {c.id for c in all_cases}
missing = sorted([i for i in ids if i not in case_ids])
if missing:
sys.stderr.write(f"missing cases in testcase_configs.py: {len(missing)}\n")
if not cases:
return 0
for c in cases:
prev = scenarios.get(c.id, {})
note = prev.get("notes")
baseline = _run_case(c)
if note is not None:
baseline["notes"] = note
scenarios[c.id] = baseline
sys.stdout.write(f"{c.id}\n")
sys.stdout.flush()
_torch_cleanup()
out_path.write_text(json.dumps(data, indent=4) + "\n", encoding="utf-8")
return 0
if __name__ == "__main__":
sys.exit(main())

57
python/sglang/multimodal_gen/test/server/perf_baselines.json
View File

@@ -1562,6 +1562,63 @@
"expected_e2e_ms": 27624.8,
"expected_avg_denoise_ms": 518.23,
"expected_median_denoise_ms": 528.06
},
"qwen_image_edit_2511_ti2i": {
"stages_ms": {
"InputValidationStage": 55.15,
"ImageEncodingStage": 770.33,
"ImageVAEEncodingStage": 88.06,
"TimestepPreparationStage": 2.12,
"LatentPreparationStage": 0.14,
"ConditioningStage": 0.01,
"DenoisingStage": 23869.32,
"DecodingStage": 108.23
},
"denoise_step_ms": {
"0": 478.35,
"1": 608.56,
"2": 588.51,
"3": 607.26,
"4": 599.37,
"5": 595.19,
"6": 603.22,
"7": 594.48,
"8": 605.06,
"9": 597.63,
"10": 601.03,
"11": 597.18,
"12": 598.82,
"13": 600.05,
"14": 598.57,
"15": 601.4,
"16": 595.17,
"17": 599.21,
"18": 600.86,
"19": 600.93,
"20": 600.35,
"21": 600.63,
"22": 597.58,
"23": 600.73,
"24": 599.36,
"25": 600.48,
"26": 600.33,
"27": 599.34,
"28": 599.61,
"29": 599.71,
"30": 596.03,
"31": 599.85,
"32": 599.36,
"33": 601.58,
"34": 597.91,
"35": 600.79,
"36": 599.29,
"37": 601.64,
"38": 598.24,
"39": 599.87
},
"expected_e2e_ms": 24895.28,
"expected_avg_denoise_ms": 596.59,
"expected_median_denoise_ms": 599.66
}
}
}

8
python/sglang/multimodal_gen/test/server/testcase_configs.py
View File

@@ -373,6 +373,14 @@ ONE_GPU_CASES_A: list[DiffusionTestCase] = [
),
MULTI_IMAGE_TI2I_sampling_params,
),
DiffusionTestCase(
"qwen_image_edit_2511_ti2i",
DiffusionServerArgs(
model_path="Qwen/Qwen-Image-Edit-2511",
modality="image",
),
TI2I_sampling_params,
),
DiffusionTestCase(
"qwen_image_layered_i2i",
DiffusionServerArgs(