chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 11 Packages Projects Releases Wiki Activity

[diffusion] CI: make auxiliary coverage explicit and simplify testcases (#20983)

Browse Source
This commit is contained in:
Mick
2026-03-21 20:18:23 +08:00
committed by GitHub
parent a0862f00c2
commit 6dfa8a40bc
4 changed files with 228 additions and 167 deletions
Download Patch File Download Diff File Expand all files Collapse all files

164
python/sglang/multimodal_gen/test/server/perf_baselines.json
View File

@@ -103,7 +103,7 @@
"qwen_image_t2i_2_gpus": {
"stages_ms": {
"InputValidationStage": 0.04,
"TextEncodingStage": 693.2,
"TextEncodingStage": 800.0,
"TimestepPreparationStage": 2.84,
"LatentPreparationStage": 9.13,
"DenoisingStage": 24529.77,
@@ -292,7 +292,7 @@
"48": 497.49,
"49": 495.69
},
"expected_e2e_ms": 25832.82,
"expected_e2e_ms": 329129.82,
"expected_avg_denoise_ms": 489.43,
"expected_median_denoise_ms": 497.53
},
@@ -1281,7 +1281,7 @@
"TimestepPreparationStage": 58.66,
"LatentPreparationStage": 28.55,
"DmdDenoisingStage": 499.34,
"DecodingStage": 1924.01,
"DecodingStage": 3712.76,
"per_frame_generation": null
},
"denoise_step_ms": {
@@ -1843,7 +1843,7 @@
"flux_2_image_t2i_2_gpus": {
"stages_ms": {
"InputValidationStage": 0.05,
"TextEncodingStage": 518.88,
"TextEncodingStage": 600.0,
"ImageVAEEncodingStage": 0.0,
"LatentPreparationStage": 0.45,
"TimestepPreparationStage": 3.41,
@@ -1964,93 +1964,93 @@
},
"fsdp-inference": {
"stages_ms": {
"InputValidationStage": 0.04,
"TextEncodingStage": 411.12,
"TimestepPreparationStage": 1.44,
"LatentPreparationStage": 0.1,
"DenoisingStage": 1569.61,
"DecodingStage": 41.43
"InputValidationStage": 0.07,
"TextEncodingStage": 605.9,
"LatentPreparationStage": 0.2,
"TimestepPreparationStage": 61.38,
"DenoisingStage": 2180.15,
"DecodingStage": 8.15
},
"denoise_step_ms": {
"0": 165.33,
"1": 158.34,
"2": 167.65,
"3": 179.11,
"4": 183.98,
"5": 175.08,
"6": 178.34,
"7": 178.53,
"8": 178.08
"0": 97.92,
"1": 228.71,
"2": 267.25,
"3": 266.93,
"4": 265.8,
"5": 263.33,
"6": 262.05,
"7": 261.54,
"8": 261.6
},
"expected_e2e_ms": 2103.05,
"expected_avg_denoise_ms": 173.83,
"expected_median_denoise_ms": 178.08
"expected_e2e_ms": 3541.48,
"expected_avg_denoise_ms": 241.68,
"expected_median_denoise_ms": 262.05
},
"hunyuan3d_shape_gen": {
"stages_ms": {
"Hunyuan3DShapeBeforeDenoisingStage": 235.65,
"Hunyuan3DShapeDenoisingStage": 3452.51,
"Hunyuan3DShapeExportStage": 8819.6,
"Hunyuan3DShapeSaveStage": 752.4,
"Hunyuan3DPaintPreprocessStage": 218136.45,
"Hunyuan3DPaintTexGenStage": 10259.21,
"Hunyuan3DPaintPostprocessStage": 6387.55
"Hunyuan3DShapeBeforeDenoisingStage": 544.59,
"Hunyuan3DShapeDenoisingStage": 3306.16,
"Hunyuan3DShapeExportStage": 8488.42,
"Hunyuan3DShapeSaveStage": 859.23,
"Hunyuan3DPaintPreprocessStage": 256020.36,
"Hunyuan3DPaintTexGenStage": 23764.05,
"Hunyuan3DPaintPostprocessStage": 7095.01
},
"denoise_step_ms": {
"0": 150.72,
"1": 26.65,
"2": 65.91,
"3": 68.09,
"4": 76.12,
"5": 68.16,
"6": 61.19,
"7": 68.26,
"8": 67.92,
"9": 68.26,
"10": 68.06,
"11": 68.22,
"12": 68.11,
"13": 68.19,
"14": 69.58,
"15": 66.91,
"16": 68.03,
"17": 68.36,
"18": 68.49,
"19": 67.69,
"20": 68.19,
"21": 69.1,
"22": 67.78,
"23": 68.36,
"24": 68.19,
"25": 68.26,
"26": 68.06,
"27": 68.25,
"28": 68.39,
"29": 68.26,
"30": 68.05,
"31": 68.27,
"32": 68.2,
"33": 68.19,
"34": 68.02,
"35": 68.3,
"36": 68.2,
"37": 68.48,
"38": 68.23,
"39": 68.36,
"40": 67.9,
"41": 75.76,
"42": 62.04,
"43": 66.78,
"44": 67.85,
"45": 68.11,
"46": 67.92,
"47": 68.15,
"48": 67.89,
"49": 68.3
"0": 137.54,
"1": 31.13,
"2": 65.96,
"3": 65.31,
"4": 65.03,
"5": 65.08,
"6": 65.1,
"7": 65.48,
"8": 64.99,
"9": 65.53,
"10": 64.91,
"11": 65.44,
"12": 64.94,
"13": 65.42,
"14": 65.1,
"15": 65.55,
"16": 65.03,
"17": 65.46,
"18": 64.97,
"19": 65.39,
"20": 65.08,
"21": 65.47,
"22": 65.89,
"23": 64.66,
"24": 65.16,
"25": 65.59,
"26": 64.95,
"27": 65.65,
"28": 64.94,
"29": 65.49,
"30": 65.37,
"31": 65.58,
"32": 64.85,
"33": 65.53,
"34": 65.18,
"35": 65.53,
"36": 64.89,
"37": 65.54,
"38": 65.29,
"39": 65.41,
"40": 65.01,
"41": 65.54,
"42": 65.17,
"43": 65.49,
"44": 64.98,
"45": 65.36,
"46": 65.25,
"47": 65.38,
"48": 65.36,
"49": 66.03
},
"expected_e2e_ms": 248171.5,
"expected_avg_denoise_ms": 68.98,
"expected_median_denoise_ms": 68.19
"expected_e2e_ms": 300141.39,
"expected_avg_denoise_ms": 66.06,
"expected_median_denoise_ms": 65.36
},
"wan2_1_t2v_1.3b_frame_interp_2x": {
"stages_ms": {

44
python/sglang/multimodal_gen/test/server/test_server_common.py
View File

@@ -92,6 +92,9 @@ def diffusion_server(case: DiffusionTestCase) -> ServerContext:
if server_args.ring_degree is not None:
extra_args += f" --ring-degree {server_args.ring_degree}"
if server_args.cfg_parallel:
extra_args += " --enable-cfg-parallel"
# LoRA support
if server_args.lora_path:
extra_args += f" --lora-path {server_args.lora_path}"
@@ -835,7 +838,7 @@ Consider updating perf_baselines.json with the snippets below:
# Dynamic LoRA loading test - tests LayerwiseOffload + set_lora interaction
# Server starts WITHOUT lora_path, then set_lora is called after startup
if case.server_args.dynamic_lora_path and not is_gt_gen_mode:
if case.run_lora_dynamic_load_check and not is_gt_gen_mode:
self._test_dynamic_lora_loading(diffusion_server, case)
generate_fn = get_generate_fn(
@@ -871,27 +874,28 @@ Consider updating perf_baselines.json with the snippets below:
validate_mesh_correctness(mesh_path)
# Test /v1/models endpoint for router compatibility
self._test_v1_models_endpoint(diffusion_server, case)
self._test_t2v_rejects_input_reference(diffusion_server, case)
if case.run_models_api_check:
self._test_v1_models_endpoint(diffusion_server, case)
if case.run_t2v_input_reference_check:
self._test_t2v_rejects_input_reference(diffusion_server, case)
# LoRA API functionality test with E2E validation (only for LoRA-enabled cases)
if case.server_args.lora_path or case.server_args.dynamic_lora_path:
if case.run_lora_basic_api_check:
self._test_lora_api_functionality(diffusion_server, case, generate_fn)
# Test dynamic LoRA switching (requires a second LoRA adapter)
if case.server_args.second_lora_path:
self._test_lora_dynamic_switch_e2e(
diffusion_server,
case,
generate_fn,
case.server_args.second_lora_path,
)
if case.run_lora_dynamic_switch_check:
self._test_lora_dynamic_switch_e2e(
diffusion_server,
case,
generate_fn,
case.server_args.second_lora_path,
)
# Test multi-LoRA functionality
self._test_multi_lora_e2e(
diffusion_server,
case,
generate_fn,
case.server_args.lora_path,
case.server_args.second_lora_path,
)
if case.run_multi_lora_api_check:
self._test_multi_lora_e2e(
diffusion_server,
case,
generate_fn,
case.server_args.lora_path,
case.server_args.second_lora_path,
)

47
python/sglang/multimodal_gen/test/server/test_server_utils.py
View File

@@ -737,6 +737,28 @@ VALIDATOR_REGISTRY = {
}
def _extract_async_job_error_message(job: Any) -> str | None:
error = getattr(job, "error", None)
if error is None and isinstance(job, dict):
error = job.get("error")
if error is None:
return None
if isinstance(error, dict):
for key in ("message", "detail", "error"):
value = error.get(key)
if value:
return str(value)
return str(error)
message = getattr(error, "message", None)
if message:
return str(message)
return str(error)
def get_generate_fn(
model_path: str,
modality: str,
@@ -796,11 +818,25 @@ def get_generate_fn(
while True:
page = client.videos.list() # type: ignore[attr-defined]
item = next((v for v in page.data if v.id == video_id), None)
status = getattr(item, "status", None) if item is not None else None
if item and getattr(item, "status", None) == "completed":
if status == "completed":
job_completed = True
break
if status == "failed":
error_message = (
_extract_async_job_error_message(item) or "unknown error"
)
pytest.fail(
f"{case_id}: video job {video_id} failed early: {error_message}"
)
if status in {"cancelled", "deleted"}:
pytest.fail(
f"{case_id}: video job {video_id} ended with status={status}"
)
if time.time() > deadline:
break
@@ -837,6 +873,15 @@ def get_generate_fn(
# Validate output file
expected_width, expected_height = parse_dimensions(size)
if (
extra_body is not None
and extra_body.get("enable_upscaling")
and expected_width
and expected_height
):
scale = extra_body.get("upscaling_scale", 4)
expected_width *= scale
expected_height *= scale
validate_video_file(
tmp_path, expected_filename, expected_width, expected_height
)

140
python/sglang/multimodal_gen/test/server/testcase_configs.py
View File

@@ -247,6 +247,37 @@ class DiffusionTestCase:
server_args: DiffusionServerArgs
sampling_params: DiffusionSamplingParams
run_perf_check: bool = True
run_models_api_check: bool = True
run_t2v_input_reference_check: bool = True
run_lora_basic_api_check: bool = False
run_lora_dynamic_load_check: bool = False
run_lora_dynamic_switch_check: bool = False
run_multi_lora_api_check: bool = False
def __post_init__(self) -> None:
has_startup_lora = self.server_args.lora_path is not None
has_dynamic_lora = self.server_args.dynamic_lora_path is not None
has_second_lora = self.server_args.second_lora_path is not None
if self.run_lora_basic_api_check and not (has_startup_lora or has_dynamic_lora):
raise ValueError(
f"{self.id}: run_lora_basic_api_check requires lora_path or dynamic_lora_path"
)
if self.run_lora_dynamic_load_check and not has_dynamic_lora:
raise ValueError(
f"{self.id}: run_lora_dynamic_load_check requires dynamic_lora_path"
)
if self.run_lora_dynamic_switch_check and not has_second_lora:
raise ValueError(
f"{self.id}: run_lora_dynamic_switch_check requires second_lora_path"
)
if self.run_multi_lora_api_check and not (has_startup_lora and has_second_lora):
raise ValueError(
f"{self.id}: run_multi_lora_api_check requires lora_path and second_lora_path"
)
def sample_step_indices(
@@ -459,6 +490,9 @@ ONE_GPU_CASES_A: list[DiffusionTestCase] = [
second_lora_path="tarn59/pixel_art_style_lora_z_image_turbo",
),
T2I_sampling_params,
run_lora_basic_api_check=True,
run_lora_dynamic_switch_check=True,
run_multi_lora_api_check=True,
),
DiffusionTestCase(
"sana_image_t2i",
@@ -621,6 +655,8 @@ ONE_GPU_CASES_B: list[DiffusionTestCase] = [
DiffusionSamplingParams(
prompt="csetiarcane Nfj1nx with blue hair, a woman walking in a cyberpunk city at night",
),
run_lora_basic_api_check=True,
run_lora_dynamic_load_check=True,
),
# NOTE(mick): flaky
# DiffusionTestCase(
@@ -680,43 +716,44 @@ ONE_GPU_CASES_B: list[DiffusionTestCase] = [
),
TI2V_sampling_params,
),
# flaky
# === Helios T2V ===
DiffusionTestCase(
"helios_base_t2v",
DiffusionServerArgs(
model_path="BestWishYsh/Helios-Base",
modality="video",
),
DiffusionSamplingParams(
prompt=T2V_PROMPT,
output_size="640x384",
num_frames=33,
),
),
DiffusionTestCase(
"helios_mid_t2v",
DiffusionServerArgs(
model_path="BestWishYsh/Helios-Mid",
modality="video",
),
DiffusionSamplingParams(
prompt=T2V_PROMPT,
output_size="640x384",
num_frames=33,
),
),
DiffusionTestCase(
"helios_distilled_t2v",
DiffusionServerArgs(
model_path="BestWishYsh/Helios-Distilled",
modality="video",
),
DiffusionSamplingParams(
prompt=T2V_PROMPT,
output_size="640x384",
num_frames=33,
),
),
# DiffusionTestCase(
# "helios_base_t2v",
# DiffusionServerArgs(
# model_path="BestWishYsh/Helios-Base",
# modality="video",
# ),
# DiffusionSamplingParams(
# prompt=T2V_PROMPT,
# output_size="640x384",
# num_frames=33,
# ),
# ),
# DiffusionTestCase(
# "helios_mid_t2v",
# DiffusionServerArgs(
# model_path="BestWishYsh/Helios-Mid",
# modality="video",
# ),
# DiffusionSamplingParams(
# prompt=T2V_PROMPT,
# output_size="640x384",
# num_frames=33,
# ),
# ),
# DiffusionTestCase(
# "helios_distilled_t2v",
# DiffusionServerArgs(
# model_path="BestWishYsh/Helios-Distilled",
# modality="video",
# ),
# DiffusionSamplingParams(
# prompt=T2V_PROMPT,
# output_size="640x384",
# num_frames=33,
# ),
# ),
]
# Skip hunyuan3d on AMD: marching_cubes surface extraction produces invalid SDF on ROCm.
@@ -800,6 +837,7 @@ TWO_GPU_CASES_A = [
DiffusionSamplingParams(
prompt="Nfj1nx with blue hair, a woman walking in a cyberpunk city at night",
),
run_lora_basic_api_check=True,
),
DiffusionTestCase(
"wan2_1_t2v_14b_2gpu",
@@ -861,6 +899,7 @@ TWO_GPU_CASES_B = [
lora_path="starsfriday/Wan2.1-Divine-Power-LoRA",
),
TI2V_sampling_params,
run_lora_basic_api_check=True,
),
DiffusionTestCase(
"wan2_1_i2v_14b_720P_2gpu",
@@ -937,34 +976,7 @@ if not current_platform.is_hip():
MULTI_IMAGE_TI2I_UPLOAD_sampling_params,
)
)
# Skip turbowan because Triton requires 81920 shared memory, but AMD only has 65536.
ONE_GPU_CASES_B.append(
DiffusionTestCase(
"turbo_wan2_1_t2v_1.3b",
DiffusionServerArgs(
model_path="IPostYellow/TurboWan2.1-T2V-1.3B-Diffusers",
modality="video",
custom_validator="video",
),
DiffusionSamplingParams(
prompt=T2V_PROMPT,
),
)
)
# Skip turbowan because Triton requires 81920 shared memory, but AMD only has 65536.
TWO_GPU_CASES_A.append(
DiffusionTestCase(
"turbo_wan2_2_i2v_a14b_2gpu",
DiffusionServerArgs(
model_path="IPostYellow/TurboWan2.2-I2V-A14B-Diffusers",
modality="video",
custom_validator="video",
num_gpus=2,
tp_size=2,
),
TURBOWAN_I2V_sampling_params,
)
)
# Load global configuration
BASELINE_CONFIG = BaselineConfig.load(