chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 11 Packages Projects Releases Wiki Activity

[diffusion] CI: improve validation method (#13627)

Browse Source
This commit is contained in:
Mick
2025-11-21 09:13:13 +08:00
committed by GitHub
parent 3ae664d786
commit c0a2513b07
4 changed files with 65 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
python/sglang/multimodal_gen/test/server/perf_baselines.json
View File

@@ -6,7 +6,8 @@
},
"tolerances": {
"e2e": 0.05,
"stage": 0.05,
"denoise_stage": 0.05,
"non_denoise_stage": 0.15,
"denoise_step": 0.15,
"denoise_agg": 0.08
},
@@ -40,7 +41,7 @@
"TimestepPreparationStage": 10.6,
"LatentPreparationStage": 11.8,
"DenoisingStage": 21202.6,
"DecodingStage": 476.12
"DecodingStage": 512.6
},
"denoise_step_ms": {
"0": 1077.77,
@@ -103,7 +104,7 @@
"TimestepPreparationStage": 1.91,
"LatentPreparationStage": 4.92,
"DenoisingStage": 8154.07,
"DecodingStage": 302.01
"DecodingStage": 386.20
},
"denoise_step_ms": {
"0": 56.03,
@@ -168,8 +169,8 @@
"expected_median_denoise_ms": 718.0,
"stages_ms": {
"InputValidationStage": 23,
"ImageEncodingStage": 1350.0,
"ImageVAEEncodingStage": 340.0,
"ImageEncodingStage": 1485.0,
"ImageVAEEncodingStage": 350.0,
"ConditioningStage": 0.13,
"TimestepPreparationStage": 13.78,
"LatentPreparationStage": 15.0,
@@ -232,7 +233,7 @@
"wan2_1_t2v_1.3b": {
"stages_ms": {
"InputValidationStage": 0.08,
"TextEncodingStage": 2140.81,
"TextEncodingStage": 2945.3,
"ConditioningStage": 0.01,
"TimestepPreparationStage": 1.97,
"LatentPreparationStage": 6.85,
@@ -241,7 +242,7 @@
"per_frame_generation": null
},
"denoise_step_ms": {
"0": 1998.19,
"0": 2831.7,
"1": 1356.14,
"2": 1368.21,
"3": 1369.72,
@@ -296,7 +297,6 @@
"expected_avg_denoise_ms": 1386.08,
"expected_median_denoise_ms": 1374.03
},
"wan2_2_i2v_a14b": {
"stages_ms": {
"InputValidationStage": 33.55,
@@ -304,7 +304,7 @@
"ConditioningStage": 0.01,
"TimestepPreparationStage": 1.52,
"LatentPreparationStage": 7.01,
"ImageVAEEncodingStage": 1904.33,
"ImageVAEEncodingStage": 2068,
"DenoisingStage": 231725.24,
"DecodingStage": 2655.39,
"per_frame_generation": null
@@ -351,7 +351,7 @@
"38": 4897.7,
"39": 4888.4
},
"expected_e2e_ms": 238587.95,
"expected_e2e_ms": 257132.95,
"expected_avg_denoise_ms": 5791.06,
"expected_median_denoise_ms": 4900.59
},
@@ -419,7 +419,7 @@
"stages_ms": {
"InputValidationStage": 53.67,
"TextEncodingStage": 2228.89,
"ImageEncodingStage": 1908.14,
"ImageEncodingStage": 2694.14,
"ConditioningStage": 0.01,
"TimestepPreparationStage": 2.25,
"LatentPreparationStage": 5.27,

25
python/sglang/multimodal_gen/test/server/test_server_performance.py
View File

@@ -406,7 +406,7 @@ Consider updating perf_baselines.json with the snippets below:
summary = validator.collect_metrics(perf_record, stage_metrics)
if is_baseline_generation_mode or missing_scenario:
self._dump_baseline_scenario(case, summary)
self._dump_baseline_for_testcase(case, summary)
if missing_scenario:
pytest.fail(f"Testcase '{case.id}' not found in perf_baselines.json")
return
@@ -415,8 +415,9 @@ Consider updating perf_baselines.json with the snippets below:
try:
validator.validate(perf_record, stage_metrics, case.num_frames)
except AssertionError:
self._dump_baseline_scenario(case, summary)
except AssertionError as e:
logger.error(f"Performance validation failed for {case.id}:\n{e}")
self._dump_baseline_for_testcase(case, summary)
raise
if case.modality == "video" and summary.frames_per_second:
@@ -504,10 +505,14 @@ Consider updating perf_baselines.json with the snippets below:
threshold = BASELINE_CONFIG.improvement_threshold
def is_sig_faster(actual, expected):
if expected == 0:
if expected == 0 or expected is None:
return False
return actual < expected * (1 - threshold)
def safe_get_metric(metric_dict, key):
val = metric_dict.get(key)
return val if val is not None else float("inf")
# Check for any significant improvement
if (
is_sig_faster(summary.e2e_ms, scenario.expected_e2e_ms)
@@ -521,15 +526,15 @@ Consider updating perf_baselines.json with the snippets below:
# Combine metrics, always taking the better (lower) value
new_stages = {
stage: min(
summary.stage_metrics.get(stage, float("inf")),
scenario.stages_ms.get(stage, float("inf")),
safe_get_metric(summary.stage_metrics, stage),
safe_get_metric(scenario.stages_ms, stage),
)
for stage in set(summary.stage_metrics) | set(scenario.stages_ms)
}
new_denoise_steps = {
step: min(
summary.all_denoise_steps.get(step, float("inf")),
scenario.denoise_step_ms.get(step, float("inf")),
safe_get_metric(summary.all_denoise_steps, step),
safe_get_metric(scenario.denoise_step_ms, step),
)
for step in set(summary.all_denoise_steps) | set(scenario.denoise_step_ms)
}
@@ -567,7 +572,7 @@ Consider updating perf_baselines.json with the snippets below:
}
self._improved_baselines.append({"id": case.id, "baseline": new_baseline})
def _dump_baseline_scenario(
def _dump_baseline_for_testcase(
self, case: DiffusionTestCase, summary: "PerformanceSummary"
) -> None:
"""Dump performance metrics as a JSON scenario for baselines."""
@@ -602,7 +607,7 @@ the "scenarios" section of perf_baselines.json:
"{case.id}": {json.dumps(baseline, indent=4)}
"""
print(output)
logger.error(output)
def test_diffusion_perf(
self,

37
python/sglang/multimodal_gen/test/server/test_server_utils.py
View File

@@ -306,14 +306,28 @@ class PerformanceValidator:
os.environ.get("SGLANG_GEN_BASELINE", "0") == "1"
)
def _assert_le(self, name: str, actual: float, expected: float, tolerance: float):
"""Assert that actual is less than or equal to expected within a tolerance."""
upper_bound = expected * (1 + tolerance)
def _assert_le(
self,
name: str,
actual: float,
expected: float,
tolerance: float,
min_abs_tolerance_ms: float = 20.0,
):
"""Assert that actual is less than or equal to expected within a tolerance.
Uses the larger of relative tolerance or absolute tolerance to prevent
flaky failures on very fast operations.
"""
rel_limit = expected * (1 + tolerance)
abs_limit = expected + min_abs_tolerance_ms
upper_bound = max(rel_limit, abs_limit)
assert actual <= upper_bound, (
f"Validation failed for '{name}'.\n"
f" - Actual: {actual:.4f}ms\n"
f" - Expected: {expected:.4f}ms\n"
f" - Limit: {upper_bound:.4f}ms (tolerance: {tolerance:.1%})"
f" Actual: {actual:.4f}ms\n"
f" Expected: {expected:.4f}ms\n"
f" Limit: {upper_bound:.4f}ms "
f"(rel_tol: {tolerance:.1%}, abs_pad: {min_abs_tolerance_ms}ms)"
)
def validate(
@@ -417,12 +431,17 @@ class PerformanceValidator:
continue
actual = summary.stage_metrics.get(stage)
assert actual is not None, f"Stage {stage} timing missing"
tolerance = (
self.tolerances.denoise_stage
if stage == "DenoisingStage"
else self.tolerances.non_denoise_stage
)
self._assert_le(
f"Stage '{stage}'",
actual,
expected,
self.tolerances.stage,
tolerance,
min_abs_tolerance_ms=120.0, # relax absolute tolerance for non-denoising stages
)
@@ -458,7 +477,7 @@ class VideoPerformanceValidator(PerformanceValidator):
"Average Frame Time",
summary.avg_frame_time_ms,
expected_frame_time,
self.tolerances.stage,
self.tolerances.denoise_stage,
)

13
python/sglang/multimodal_gen/test/server/testcase_configs.py
View File

@@ -30,7 +30,8 @@ class ToleranceConfig:
"""Tolerance ratios for performance validation."""
e2e: float
stage: float
denoise_stage: float
non_denoise_stage: float
denoise_step: float
denoise_agg: float
@@ -65,7 +66,15 @@ class BaselineConfig:
tol_data = data["tolerances"]
tolerances = ToleranceConfig(
e2e=float(os.getenv("SGLANG_E2E_TOLERANCE", tol_data["e2e"])),
stage=float(os.getenv("SGLANG_STAGE_TIME_TOLERANCE", tol_data["stage"])),
denoise_stage=float(
os.getenv("SGLANG_STAGE_TIME_TOLERANCE", tol_data["denoise_stage"])
),
non_denoise_stage=float(
os.getenv(
"SGLANG_NON_DENOISE_STAGE_TIME_TOLERANCE",
tol_data["non_denoise_stage"],
)
),
denoise_step=float(
os.getenv("SGLANG_DENOISE_STEP_TOLERANCE", tol_data["denoise_step"])
),