From c0a2513b0728b36316e609a86dceb02d444cd805 Mon Sep 17 00:00:00 2001 From: Mick Date: Fri, 21 Nov 2025 09:13:13 +0800 Subject: [PATCH] [diffusion] CI: improve validation method (#13627) --- .../test/server/perf_baselines.json | 22 +++++------ .../test/server/test_server_performance.py | 25 ++++++++----- .../test/server/test_server_utils.py | 37 ++++++++++++++----- .../test/server/testcase_configs.py | 13 ++++++- 4 files changed, 65 insertions(+), 32 deletions(-) diff --git a/python/sglang/multimodal_gen/test/server/perf_baselines.json b/python/sglang/multimodal_gen/test/server/perf_baselines.json index cc668582a..f2082f31c 100644 --- a/python/sglang/multimodal_gen/test/server/perf_baselines.json +++ b/python/sglang/multimodal_gen/test/server/perf_baselines.json @@ -6,7 +6,8 @@ }, "tolerances": { "e2e": 0.05, - "stage": 0.05, + "denoise_stage": 0.05, + "non_denoise_stage": 0.15, "denoise_step": 0.15, "denoise_agg": 0.08 }, @@ -40,7 +41,7 @@ "TimestepPreparationStage": 10.6, "LatentPreparationStage": 11.8, "DenoisingStage": 21202.6, - "DecodingStage": 476.12 + "DecodingStage": 512.6 }, "denoise_step_ms": { "0": 1077.77, @@ -103,7 +104,7 @@ "TimestepPreparationStage": 1.91, "LatentPreparationStage": 4.92, "DenoisingStage": 8154.07, - "DecodingStage": 302.01 + "DecodingStage": 386.20 }, "denoise_step_ms": { "0": 56.03, @@ -168,8 +169,8 @@ "expected_median_denoise_ms": 718.0, "stages_ms": { "InputValidationStage": 23, - "ImageEncodingStage": 1350.0, - "ImageVAEEncodingStage": 340.0, + "ImageEncodingStage": 1485.0, + "ImageVAEEncodingStage": 350.0, "ConditioningStage": 0.13, "TimestepPreparationStage": 13.78, "LatentPreparationStage": 15.0, @@ -232,7 +233,7 @@ "wan2_1_t2v_1.3b": { "stages_ms": { "InputValidationStage": 0.08, - "TextEncodingStage": 2140.81, + "TextEncodingStage": 2945.3, "ConditioningStage": 0.01, "TimestepPreparationStage": 1.97, "LatentPreparationStage": 6.85, @@ -241,7 +242,7 @@ "per_frame_generation": null }, "denoise_step_ms": { - "0": 1998.19, + "0": 2831.7, "1": 1356.14, "2": 1368.21, "3": 1369.72, @@ -296,7 +297,6 @@ "expected_avg_denoise_ms": 1386.08, "expected_median_denoise_ms": 1374.03 }, - "wan2_2_i2v_a14b": { "stages_ms": { "InputValidationStage": 33.55, @@ -304,7 +304,7 @@ "ConditioningStage": 0.01, "TimestepPreparationStage": 1.52, "LatentPreparationStage": 7.01, - "ImageVAEEncodingStage": 1904.33, + "ImageVAEEncodingStage": 2068, "DenoisingStage": 231725.24, "DecodingStage": 2655.39, "per_frame_generation": null @@ -351,7 +351,7 @@ "38": 4897.7, "39": 4888.4 }, - "expected_e2e_ms": 238587.95, + "expected_e2e_ms": 257132.95, "expected_avg_denoise_ms": 5791.06, "expected_median_denoise_ms": 4900.59 }, @@ -419,7 +419,7 @@ "stages_ms": { "InputValidationStage": 53.67, "TextEncodingStage": 2228.89, - "ImageEncodingStage": 1908.14, + "ImageEncodingStage": 2694.14, "ConditioningStage": 0.01, "TimestepPreparationStage": 2.25, "LatentPreparationStage": 5.27, diff --git a/python/sglang/multimodal_gen/test/server/test_server_performance.py b/python/sglang/multimodal_gen/test/server/test_server_performance.py index ce517403a..08b700da5 100644 --- a/python/sglang/multimodal_gen/test/server/test_server_performance.py +++ b/python/sglang/multimodal_gen/test/server/test_server_performance.py @@ -406,7 +406,7 @@ Consider updating perf_baselines.json with the snippets below: summary = validator.collect_metrics(perf_record, stage_metrics) if is_baseline_generation_mode or missing_scenario: - self._dump_baseline_scenario(case, summary) + self._dump_baseline_for_testcase(case, summary) if missing_scenario: pytest.fail(f"Testcase '{case.id}' not found in perf_baselines.json") return @@ -415,8 +415,9 @@ Consider updating perf_baselines.json with the snippets below: try: validator.validate(perf_record, stage_metrics, case.num_frames) - except AssertionError: - self._dump_baseline_scenario(case, summary) + except AssertionError as e: + logger.error(f"Performance validation failed for {case.id}:\n{e}") + self._dump_baseline_for_testcase(case, summary) raise if case.modality == "video" and summary.frames_per_second: @@ -504,10 +505,14 @@ Consider updating perf_baselines.json with the snippets below: threshold = BASELINE_CONFIG.improvement_threshold def is_sig_faster(actual, expected): - if expected == 0: + if expected == 0 or expected is None: return False return actual < expected * (1 - threshold) + def safe_get_metric(metric_dict, key): + val = metric_dict.get(key) + return val if val is not None else float("inf") + # Check for any significant improvement if ( is_sig_faster(summary.e2e_ms, scenario.expected_e2e_ms) @@ -521,15 +526,15 @@ Consider updating perf_baselines.json with the snippets below: # Combine metrics, always taking the better (lower) value new_stages = { stage: min( - summary.stage_metrics.get(stage, float("inf")), - scenario.stages_ms.get(stage, float("inf")), + safe_get_metric(summary.stage_metrics, stage), + safe_get_metric(scenario.stages_ms, stage), ) for stage in set(summary.stage_metrics) | set(scenario.stages_ms) } new_denoise_steps = { step: min( - summary.all_denoise_steps.get(step, float("inf")), - scenario.denoise_step_ms.get(step, float("inf")), + safe_get_metric(summary.all_denoise_steps, step), + safe_get_metric(scenario.denoise_step_ms, step), ) for step in set(summary.all_denoise_steps) | set(scenario.denoise_step_ms) } @@ -567,7 +572,7 @@ Consider updating perf_baselines.json with the snippets below: } self._improved_baselines.append({"id": case.id, "baseline": new_baseline}) - def _dump_baseline_scenario( + def _dump_baseline_for_testcase( self, case: DiffusionTestCase, summary: "PerformanceSummary" ) -> None: """Dump performance metrics as a JSON scenario for baselines.""" @@ -602,7 +607,7 @@ the "scenarios" section of perf_baselines.json: "{case.id}": {json.dumps(baseline, indent=4)} """ - print(output) + logger.error(output) def test_diffusion_perf( self, diff --git a/python/sglang/multimodal_gen/test/server/test_server_utils.py b/python/sglang/multimodal_gen/test/server/test_server_utils.py index 34f593096..da84e86be 100644 --- a/python/sglang/multimodal_gen/test/server/test_server_utils.py +++ b/python/sglang/multimodal_gen/test/server/test_server_utils.py @@ -306,14 +306,28 @@ class PerformanceValidator: os.environ.get("SGLANG_GEN_BASELINE", "0") == "1" ) - def _assert_le(self, name: str, actual: float, expected: float, tolerance: float): - """Assert that actual is less than or equal to expected within a tolerance.""" - upper_bound = expected * (1 + tolerance) + def _assert_le( + self, + name: str, + actual: float, + expected: float, + tolerance: float, + min_abs_tolerance_ms: float = 20.0, + ): + """Assert that actual is less than or equal to expected within a tolerance. + + Uses the larger of relative tolerance or absolute tolerance to prevent + flaky failures on very fast operations. + """ + rel_limit = expected * (1 + tolerance) + abs_limit = expected + min_abs_tolerance_ms + upper_bound = max(rel_limit, abs_limit) assert actual <= upper_bound, ( f"Validation failed for '{name}'.\n" - f" - Actual: {actual:.4f}ms\n" - f" - Expected: {expected:.4f}ms\n" - f" - Limit: {upper_bound:.4f}ms (tolerance: {tolerance:.1%})" + f" Actual: {actual:.4f}ms\n" + f" Expected: {expected:.4f}ms\n" + f" Limit: {upper_bound:.4f}ms " + f"(rel_tol: {tolerance:.1%}, abs_pad: {min_abs_tolerance_ms}ms)" ) def validate( @@ -417,12 +431,17 @@ class PerformanceValidator: continue actual = summary.stage_metrics.get(stage) assert actual is not None, f"Stage {stage} timing missing" - + tolerance = ( + self.tolerances.denoise_stage + if stage == "DenoisingStage" + else self.tolerances.non_denoise_stage + ) self._assert_le( f"Stage '{stage}'", actual, expected, - self.tolerances.stage, + tolerance, + min_abs_tolerance_ms=120.0, # relax absolute tolerance for non-denoising stages ) @@ -458,7 +477,7 @@ class VideoPerformanceValidator(PerformanceValidator): "Average Frame Time", summary.avg_frame_time_ms, expected_frame_time, - self.tolerances.stage, + self.tolerances.denoise_stage, ) diff --git a/python/sglang/multimodal_gen/test/server/testcase_configs.py b/python/sglang/multimodal_gen/test/server/testcase_configs.py index 10d30e2e8..ecfdb4f6a 100644 --- a/python/sglang/multimodal_gen/test/server/testcase_configs.py +++ b/python/sglang/multimodal_gen/test/server/testcase_configs.py @@ -30,7 +30,8 @@ class ToleranceConfig: """Tolerance ratios for performance validation.""" e2e: float - stage: float + denoise_stage: float + non_denoise_stage: float denoise_step: float denoise_agg: float @@ -65,7 +66,15 @@ class BaselineConfig: tol_data = data["tolerances"] tolerances = ToleranceConfig( e2e=float(os.getenv("SGLANG_E2E_TOLERANCE", tol_data["e2e"])), - stage=float(os.getenv("SGLANG_STAGE_TIME_TOLERANCE", tol_data["stage"])), + denoise_stage=float( + os.getenv("SGLANG_STAGE_TIME_TOLERANCE", tol_data["denoise_stage"]) + ), + non_denoise_stage=float( + os.getenv( + "SGLANG_NON_DENOISE_STAGE_TIME_TOLERANCE", + tol_data["non_denoise_stage"], + ) + ), denoise_step=float( os.getenv("SGLANG_DENOISE_STEP_TOLERANCE", tol_data["denoise_step"]) ),