diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index 12945ce54..c54a5bb7b 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -351,10 +351,15 @@ jobs: python3 run_suite.py --hw cpu --suite default - multimodal-gen-test: + multimodal-gen-test-1-gpu: needs: [check-changes, sgl-kernel-build-wheels] if: (always() && !failure() && !cancelled()) && needs.check-changes.outputs.multimodal_gen == 'true' runs-on: 1-gpu-runner + strategy: + fail-fast: false + max-parallel: 5 + matrix: + test_file: ["test_server_perf_a.py", "test_server_perf_b.py"] steps: - name: Checkout code uses: actions/checkout@v4 @@ -375,7 +380,34 @@ jobs: timeout-minutes: 60 run: | cd python - pytest -s -v --log-cli-level=INFO sglang/multimodal_gen/test/server/test_server_performance.py + pytest -s -v --log-cli-level=INFO sglang/multimodal_gen/test/server/${{ matrix.test_file }} + + + multimodal-gen-test-2-gpu: + needs: [check-changes, sgl-kernel-build-wheels] + if: (always() && !failure() && !cancelled()) && needs.check-changes.outputs.multimodal_gen == 'true' + runs-on: 2-gpu-runner + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Download artifacts + if: needs.check-changes.outputs.sgl_kernel == 'true' + uses: actions/download-artifact@v4 + with: + path: sgl-kernel/dist/ + merge-multiple: true + pattern: wheel-python3.10-cuda12.9 + + - name: Install dependencies + run: | + CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/ci_install_dependency.sh diffusion + + - name: Run diffusion server tests + timeout-minutes: 60 + run: | + cd python + pytest -s -v --log-cli-level=INFO sglang/multimodal_gen/test/server/test_server_perf_2_gpu.py unit-test-backend-1-gpu: needs: [check-changes, stage-a-test-1] @@ -963,7 +995,8 @@ jobs: sgl-kernel-benchmark-test, sgl-kernel-b200-test, - multimodal-gen-test, + multimodal-gen-test-1-gpu, + multimodal-gen-test-2-gpu, stage-a-test-1, unit-test-backend-1-gpu, diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/hunyuan.py b/python/sglang/multimodal_gen/configs/pipeline_configs/hunyuan.py index 2483063ea..d45dfadb2 100644 --- a/python/sglang/multimodal_gen/configs/pipeline_configs/hunyuan.py +++ b/python/sglang/multimodal_gen/configs/pipeline_configs/hunyuan.py @@ -15,7 +15,10 @@ from sglang.multimodal_gen.configs.models.encoders import ( LlamaConfig, ) from sglang.multimodal_gen.configs.models.vaes import HunyuanVAEConfig -from sglang.multimodal_gen.configs.pipeline_configs import PipelineConfig +from sglang.multimodal_gen.configs.pipeline_configs.base import ( + ModelTaskType, + PipelineConfig, +) PROMPT_TEMPLATE_ENCODE_VIDEO = ( "<|start_header_id|>system<|end_header_id|>\n\nDescribe the video by detailing the following aspects: " @@ -66,6 +69,8 @@ def clip_postprocess_text(outputs: BaseEncoderOutput, _text_inputs) -> torch.ten class HunyuanConfig(PipelineConfig): """Base configuration for HunYuan pipeline architecture.""" + task_type: ModelTaskType = ModelTaskType.T2V + # HunyuanConfig-specific parameters with defaults # DiT dit_config: DiTConfig = field(default_factory=HunyuanVideoConfig) diff --git a/python/sglang/multimodal_gen/configs/sample/base.py b/python/sglang/multimodal_gen/configs/sample/base.py index d15f2e427..3e2c1d302 100644 --- a/python/sglang/multimodal_gen/configs/sample/base.py +++ b/python/sglang/multimodal_gen/configs/sample/base.py @@ -414,6 +414,8 @@ class SamplingParams: if user_params is None: return + subclass_defined_fields = set(type(self).__annotations__.keys()) + # Compare against current instance to avoid constructing a default instance default_params = SamplingParams() @@ -430,7 +432,7 @@ class SamplingParams: if field_name != "output_file_name" else user_params.output_file_path is not None ) - if is_user_modified: + if is_user_modified and field_name not in subclass_defined_fields: if hasattr(self, field_name): setattr(self, field_name, user_value) diff --git a/python/sglang/multimodal_gen/runtime/layers/triton_ops.py b/python/sglang/multimodal_gen/runtime/layers/triton_ops.py index 8fa74994a..2a8d96af8 100644 --- a/python/sglang/multimodal_gen/runtime/layers/triton_ops.py +++ b/python/sglang/multimodal_gen/runtime/layers/triton_ops.py @@ -874,7 +874,7 @@ def norm_infer( out: Optional[Tensor] = None, ): M, N = x.shape - assert x.stride(-1) == 1 + x = x.contiguous() if weight is not None: assert weight.shape == (N,) assert weight.stride(-1) == 1 diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/denoising_dmd.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/denoising_dmd.py index 250af0135..1af44795d 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/denoising_dmd.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/denoising_dmd.py @@ -29,8 +29,12 @@ from sglang.multimodal_gen.runtime.pipelines_core.stages.denoising import ( vsa_available, ) from sglang.multimodal_gen.runtime.server_args import ServerArgs +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger +from sglang.multimodal_gen.runtime.utils.perf_logger import StageProfiler from sglang.multimodal_gen.utils import dict_to_3d_list +logger = init_logger(__name__) + # TODO: use base methods of DenoisingStage class DmdDenoisingStage(DenoisingStage): @@ -139,132 +143,140 @@ class DmdDenoisingStage(DenoisingStage): # Run denoising loop denoising_loop_start_time = time.time() + self.start_profile(batch=batch) with self.progress_bar(total=len(timesteps)) as progress_bar: for i, t in enumerate(timesteps): # Skip if interrupted if hasattr(self, "interrupt") and self.interrupt: continue - # Expand latents for I2V - noise_latents = latents.clone() - latent_model_input = latents.to(target_dtype) - if batch.image_latent is not None: - latent_model_input = torch.cat( - [ - latent_model_input, - batch.image_latent.permute(0, 2, 1, 3, 4), - ], - dim=2, - ).to(target_dtype) - assert not torch.isnan( - latent_model_input - ).any(), "latent_model_input contains nan" - - # Prepare inputs for transformer - t_expand = t.repeat(latent_model_input.shape[0]) - guidance_expand = ( - torch.tensor( - [server_args.pipeline_config.embedded_cfg_scale] - * latent_model_input.shape[0], - dtype=torch.float32, - device=get_local_torch_device(), - ).to(target_dtype) - * 1000.0 - if server_args.pipeline_config.embedded_cfg_scale is not None - else None - ) - - # Predict noise residual - with torch.autocast( - device_type="cuda", - dtype=target_dtype, - enabled=autocast_enabled, + with StageProfiler( + f"denoising_step_{i}", logger=logger, timings=batch.timings ): - if ( - vsa_available - and self.attn_backend == VideoSparseAttentionBackend - ): - self.attn_metadata_builder_cls = ( - self.attn_backend.get_builder_cls() - ) + # Expand latents for I2V + noise_latents = latents.clone() + latent_model_input = latents.to(target_dtype) - if self.attn_metadata_builder_cls is not None: - self.attn_metadata_builder = ( - self.attn_metadata_builder_cls() + if batch.image_latent is not None: + latent_model_input = torch.cat( + [ + latent_model_input, + batch.image_latent.permute(0, 2, 1, 3, 4), + ], + dim=2, + ).to(target_dtype) + assert not torch.isnan( + latent_model_input + ).any(), "latent_model_input contains nan" + + # Prepare inputs for transformer + t_expand = t.repeat(latent_model_input.shape[0]) + guidance_expand = ( + torch.tensor( + [server_args.pipeline_config.embedded_cfg_scale] + * latent_model_input.shape[0], + dtype=torch.float32, + device=get_local_torch_device(), + ).to(target_dtype) + * 1000.0 + if server_args.pipeline_config.embedded_cfg_scale is not None + else None + ) + + # Predict noise residual + with torch.autocast( + device_type="cuda", + dtype=target_dtype, + enabled=autocast_enabled, + ): + if ( + vsa_available + and self.attn_backend == VideoSparseAttentionBackend + ): + self.attn_metadata_builder_cls = ( + self.attn_backend.get_builder_cls() ) - # TODO(will): clean this up - attn_metadata = self.attn_metadata_builder.build( # type: ignore - current_timestep=i, # type: ignore - raw_latent_shape=batch.raw_latent_shape[2:5], # type: ignore - patch_size=server_args.pipeline_config.dit_config.patch_size, # type: ignore - STA_param=batch.STA_param, # type: ignore - VSA_sparsity=server_args.VSA_sparsity, # type: ignore - device=get_local_torch_device(), # type: ignore - ) # type: ignore - assert ( - attn_metadata is not None - ), "attn_metadata cannot be None" + + if self.attn_metadata_builder_cls is not None: + self.attn_metadata_builder = ( + self.attn_metadata_builder_cls() + ) + # TODO(will): clean this up + attn_metadata = self.attn_metadata_builder.build( # type: ignore + current_timestep=i, # type: ignore + raw_latent_shape=batch.raw_latent_shape[2:5], # type: ignore + patch_size=server_args.pipeline_config.dit_config.patch_size, # type: ignore + STA_param=batch.STA_param, # type: ignore + VSA_sparsity=server_args.VSA_sparsity, # type: ignore + device=get_local_torch_device(), # type: ignore + ) # type: ignore + assert ( + attn_metadata is not None + ), "attn_metadata cannot be None" + else: + attn_metadata = None else: attn_metadata = None - else: - attn_metadata = None - batch.is_cfg_negative = False - with set_forward_context( - current_timestep=i, - attn_metadata=attn_metadata, - forward_batch=batch, - # server_args=server_args - ): - # Run transformer - pred_noise = self.transformer( - latent_model_input.permute(0, 2, 1, 3, 4), - prompt_embeds, - t_expand, - guidance=guidance_expand, - **image_kwargs, - **pos_cond_kwargs, - ).permute(0, 2, 1, 3, 4) + batch.is_cfg_negative = False + with set_forward_context( + current_timestep=i, + attn_metadata=attn_metadata, + forward_batch=batch, + # server_args=server_args + ): + # Run transformer + pred_noise = self.transformer( + latent_model_input.permute(0, 2, 1, 3, 4), + prompt_embeds, + t_expand, + guidance=guidance_expand, + **image_kwargs, + **pos_cond_kwargs, + ).permute(0, 2, 1, 3, 4) - pred_video = pred_noise_to_pred_video( - pred_noise=pred_noise.flatten(0, 1), - noise_input_latent=noise_latents.flatten(0, 1), - timestep=t_expand, - scheduler=self.scheduler, - ).unflatten(0, pred_noise.shape[:2]) + pred_video = pred_noise_to_pred_video( + pred_noise=pred_noise.flatten(0, 1), + noise_input_latent=noise_latents.flatten(0, 1), + timestep=t_expand, + scheduler=self.scheduler, + ).unflatten(0, pred_noise.shape[:2]) - if i < len(timesteps) - 1: - next_timestep = timesteps[i + 1] * torch.ones( - [1], dtype=torch.long, device=pred_video.device - ) - noise = torch.randn( - video_raw_latent_shape, - dtype=pred_video.dtype, - generator=batch.generator[0], - ).to(self.device) - if sp_group: - noise = rearrange( - noise, - "b (n t) c h w -> b n t c h w", - n=sp_world_size, - ).contiguous() - noise = noise[:, rank_in_sp_group, :, :, :, :] - latents = self.scheduler.add_noise( - pred_video.flatten(0, 1), - noise.flatten(0, 1), - next_timestep, - ).unflatten(0, pred_video.shape[:2]) - else: - latents = pred_video + if i < len(timesteps) - 1: + next_timestep = timesteps[i + 1] * torch.ones( + [1], dtype=torch.long, device=pred_video.device + ) + noise = torch.randn( + video_raw_latent_shape, + dtype=pred_video.dtype, + generator=batch.generator[0], + ).to(self.device) + if sp_group: + noise = rearrange( + noise, + "b (n t) c h w -> b n t c h w", + n=sp_world_size, + ).contiguous() + noise = noise[:, rank_in_sp_group, :, :, :, :] + latents = self.scheduler.add_noise( + pred_video.flatten(0, 1), + noise.flatten(0, 1), + next_timestep, + ).unflatten(0, pred_video.shape[:2]) + else: + latents = pred_video - # Update progress bar - if i == len(timesteps) - 1 or ( - (i + 1) > num_warmup_steps - and (i + 1) % self.scheduler.order == 0 - and progress_bar is not None - ): - progress_bar.update() + # Update progress bar + if i == len(timesteps) - 1 or ( + (i + 1) > num_warmup_steps + and (i + 1) % self.scheduler.order == 0 + and progress_bar is not None + ): + progress_bar.update() + self.step_profile() + + self.stop_profile(batch) denoising_loop_end_time = time.time() if len(timesteps) > 0: self.log_info( diff --git a/python/sglang/multimodal_gen/runtime/utils/logging_utils.py b/python/sglang/multimodal_gen/runtime/utils/logging_utils.py index 7b752d5bb..01281405b 100644 --- a/python/sglang/multimodal_gen/runtime/utils/logging_utils.py +++ b/python/sglang/multimodal_gen/runtime/utils/logging_utils.py @@ -124,15 +124,20 @@ def _print_warning_once(logger: Logger, msg: str) -> None: logger.warning(msg, stacklevel=2) -def _get_rank_info(): - """Get rank and local rank from environment variables.""" +def get_is_main_process(): try: rank = int(os.environ["RANK"]) - local_rank = int(os.environ["LOCAL_RANK"]) except (KeyError, ValueError): rank = 0 - local_rank = 0 - return rank, local_rank + return rank == 0 + + +def get_is_local_main_process(): + try: + rank = int(os.environ["LOCAL_RANK"]) + except (KeyError, ValueError): + rank = 0 + return rank == 0 def _log_process_aware( @@ -145,9 +150,8 @@ def _log_process_aware( **kwargs: Any, ) -> None: """Helper function to log a message if the process rank matches the criteria.""" - rank, local_rank = _get_rank_info() - is_main_process = rank == 0 - is_local_main_process = local_rank == 0 + is_main_process = get_is_main_process() + is_local_main_process = get_is_local_main_process() should_log = ( not main_process_only @@ -393,8 +397,7 @@ def suppress_other_loggers(not_suppress_on_main_rank: bool = False): should_suppress = True if not_suppress_on_main_rank: - rank, _ = _get_rank_info() - if rank == 0: + if get_is_main_process() == 0: should_suppress = False loggers_to_suppress = ["urllib3"] diff --git a/python/sglang/multimodal_gen/runtime/utils/perf_logger.py b/python/sglang/multimodal_gen/runtime/utils/perf_logger.py index ae126c407..16f94255d 100644 --- a/python/sglang/multimodal_gen/runtime/utils/perf_logger.py +++ b/python/sglang/multimodal_gen/runtime/utils/perf_logger.py @@ -15,7 +15,10 @@ from dateutil.tz import UTC import sglang import sglang.multimodal_gen.envs as envs -from sglang.multimodal_gen.runtime.utils.logging_utils import _SGLDiffusionLogger +from sglang.multimodal_gen.runtime.utils.logging_utils import ( + _SGLDiffusionLogger, + get_is_main_process, +) @dataclasses.dataclass @@ -246,14 +249,15 @@ class PerformanceLogger: ) try: - log_dir = get_diffusion_perf_log_dir() - if not os.path.exists(log_dir): - os.makedirs(log_dir, exist_ok=True) + if get_is_main_process(): + log_dir = get_diffusion_perf_log_dir() + if not os.path.exists(log_dir): + os.makedirs(log_dir, exist_ok=True) - log_file = os.path.join(log_dir, "performance.log") + log_file = os.path.join(log_dir, "performance.log") - with open(log_file, "a", encoding="utf-8") as f: - f.write(json.dumps(dataclasses.asdict(record)) + "\n") + with open(log_file, "a", encoding="utf-8") as f: + f.write(json.dumps(dataclasses.asdict(record)) + "\n") except (OSError, PermissionError) as e: print(f"WARNING: Failed to log performance record: {e}", file=sys.stderr) diff --git a/python/sglang/multimodal_gen/test/server/perf_baselines.json b/python/sglang/multimodal_gen/test/server/perf_baselines.json index aab4f6bfc..2158b5b6a 100644 --- a/python/sglang/multimodal_gen/test/server/perf_baselines.json +++ b/python/sglang/multimodal_gen/test/server/perf_baselines.json @@ -297,259 +297,434 @@ "expected_avg_denoise_ms": 1820.02, "expected_median_denoise_ms": 1798.65 }, - "wan2_2_i2v_a14b": { + + "wan2_2_ti2v_5b": { "stages_ms": { - "InputValidationStage": 33.55, - "TextEncodingStage": 2291.94, - "ConditioningStage": 0.03, - "TimestepPreparationStage": 2.37, - "LatentPreparationStage": 8.87, - "ImageVAEEncodingStage": 2092.43, - "DenoisingStage": 255868.61, - "DecodingStage": 2747.78, + "InputValidationStage": 96.27, + "TextEncodingStage": 2238.81, + "ConditioningStage": 0.02, + "TimestepPreparationStage": 2.39, + "LatentPreparationStage": 27.62, + "DenoisingStage": 134069.79, + "DecodingStage": 13559.79, "per_frame_generation": null }, "denoise_step_ms": { - "0": 24266.38, - "1": 4987.24, - "2": 5023.9, - "3": 5033.86, - "4": 5039.19, - "5": 5049.18, - "6": 5036.97, - "7": 5037.92, - "8": 5040.91, - "9": 5051.29, - "10": 5039.41, - "11": 5049.76, - "12": 5044.17, - "13": 5049.54, - "14": 5050.61, - "15": 39939.62, - "16": 5037.41, - "17": 5055.67, - "18": 5059.63, - "19": 5070.04, - "20": 5078.54, - "21": 5079.15, - "22": 5065.42, - "23": 5064.19, - "24": 5065.35, - "25": 5059.83, - "26": 5055.47, - "27": 5048.03, - "28": 5044.28, - "29": 5044.19, - "30": 5040.91, - "31": 5037.98, - "32": 5026.15, - "33": 5027.36, - "34": 5018.56, - "35": 5007.23, - "36": 5012.65, - "37": 5013.68, - "38": 5005.83, - "39": 5000.26 + "0": 3181.0, + "1": 2561.67, + "2": 2578.49, + "3": 2582.1, + "4": 2572.24, + "5": 2577.72, + "6": 2581.35, + "7": 2578.79, + "8": 2584.98, + "9": 2588.49, + "10": 2594.37, + "11": 2591.19, + "12": 2591.32, + "13": 2595.35, + "14": 2594.35, + "15": 2595.62, + "16": 2596.35, + "17": 2596.11, + "18": 2597.24, + "19": 2603.13, + "20": 2599.9, + "21": 2601.48, + "22": 2603.58, + "23": 2601.13, + "24": 2600.47, + "25": 2604.13, + "26": 2606.04, + "27": 2605.3, + "28": 2602.02, + "29": 2601.83, + "30": 2603.57, + "31": 2606.63, + "32": 2606.1, + "33": 2602.24, + "34": 2603.29, + "35": 2602.34, + "36": 2602.16, + "37": 2608.14, + "38": 2603.48, + "39": 2601.7, + "40": 2603.96, + "41": 2604.58, + "42": 2606.67, + "43": 2603.52, + "44": 2599.88, + "45": 2598.66, + "46": 2600.74, + "47": 2602.31, + "48": 2608.4, + "49": 2606.02 }, - "expected_e2e_ms": 263069.24, - "expected_avg_denoise_ms": 6393.94, - "expected_median_denoise_ms": 5044.18 + "expected_e2e_ms": 150004.2, + "expected_avg_denoise_ms": 2608.84, + "expected_median_denoise_ms": 2601.59 }, - "wan2_1_i2v_14b_480P": { + "fastwan2_2_ti2v_5b": { "stages_ms": { + "InputValidationStage": 88.86, + "TextEncodingStage": 2327.87, + "ConditioningStage": 0.01, + "TimestepPreparationStage": 58.66, + "LatentPreparationStage": 28.55, + "DmdDenoisingStage": 4438.3, + "DecodingStage": 14177.77, "per_frame_generation": null }, "denoise_step_ms": { - "0": 5432.37, - "1": 4861.66, - "2": 4890.23, - "3": 4919.9, - "4": 4929.49, - "5": 4927.35, - "6": 4927.6, - "7": 4932.02, - "8": 4897.83, - "9": 4897.16, - "10": 4899.59, - "11": 4901.51, - "12": 4896.38, - "13": 4914.3, - "14": 4901.91, - "15": 4898.58, - "16": 4903.18, - "17": 4899.54, - "18": 4910.5, - "19": 4898.18, - "20": 4901.56, - "21": 4903.12, - "22": 4896.82, - "23": 4901.01, - "24": 4897.42, - "25": 4911.94, - "26": 4901.29, - "27": 4894.79, - "28": 4899.57, - "29": 4900.63, - "30": 4904.78, - "31": 4899.79, - "32": 4898.01, - "33": 4910.81, - "34": 4901.16, - "35": 4898.48, - "36": 4894.97, - "37": 4905.9, - "38": 4901.52, - "39": 4898.56, - "40": 4900.19, - "41": 4896.5, - "42": 4899.89, - "43": 4898.45, - "44": 4897.27, - "45": 4895.47, - "46": 4889.26, - "47": 4887.44, - "48": 4894.07, - "49": 4888.59 + "0": 2022.21, + "1": 1263.17, + "2": 1149.59 }, - "expected_e2e_ms": 254552.63, - "expected_avg_denoise_ms": 4912.17, - "expected_median_denoise_ms": 4899.69 + "expected_e2e_ms": 21133.36, + "expected_avg_denoise_ms": 1478.32, + "expected_median_denoise_ms": 1263.17 }, - "wan2_1_i2v_14b_720P": { + "fast_hunyuan_video": { + "stages_ms": { + "InputValidationStage": 0.07, + "TextEncodingStage": 598.66, + "ConditioningStage": 0.01, + "TimestepPreparationStage": 57.26, + "LatentPreparationStage": 15.38, + "DenoisingStage": 3367.01, + "DecodingStage": 2457.57, + "per_frame_generation": null + }, + "denoise_step_ms": { + "0": 1577.86, + "1": 161.04, + "2": 406.81, + "3": 408.67, + "4": 403.55, + "5": 405.31 + }, + "expected_e2e_ms": 6521.03, + "expected_avg_denoise_ms": 560.54, + "expected_median_denoise_ms": 406.06 + }, + "wan2_2_i2v_a14b_2gpu": { + "stages_ms": { + "InputValidationStage": 59.33, + "TextEncodingStage": 6062.41, + "ConditioningStage": 0.02, + "TimestepPreparationStage": 2.2, + "LatentPreparationStage": 8.93, + "ImageVAEEncodingStage": 2075.47, + "DenoisingStage": 382628.41, + "DecodingStage": 2820.89 + }, + "denoise_step_ms": { + "0": 31228.27, + "1": 7723.86, + "2": 7769.69, + "3": 7795.93, + "4": 7815.58, + "5": 7829.48, + "6": 7827.34, + "7": 7825.35, + "8": 7828.05, + "9": 7809.53, + "10": 7801.29, + "11": 7790.96, + "12": 7785.88, + "13": 7785.5, + "14": 7780.32, + "15": 55411.1, + "16": 7722.27, + "17": 7761.31, + "18": 7789.46, + "19": 7800.6, + "20": 7814.91, + "21": 7799.62, + "22": 7801.25, + "23": 7798.27, + "24": 7797.67, + "25": 7795.97, + "26": 7781.74, + "27": 7784.16, + "28": 7796.64, + "29": 7789.75, + "30": 7792.13, + "31": 7790.99, + "32": 7778.1, + "33": 7777.78, + "34": 7780.56, + "35": 7778.22, + "36": 7770.88, + "37": 7771.56, + "38": 7767.82, + "39": 7769.23 + }, + "expected_e2e_ms": 393606.77, + "expected_avg_denoise_ms": 9565.48, + "expected_median_denoise_ms": 7790.98 + }, + "wan2_1_i2v_14b_480P_2gpu": { + "stages_ms": { + "InputValidationStage": 32.94, + "TextEncodingStage": 2316.5, + "ImageEncodingStage": 3026.2, + "ConditioningStage": 0.01, + "TimestepPreparationStage": 2.69, + "LatentPreparationStage": 9.73, + "ImageVAEEncodingStage": 2290.98, + "DenoisingStage": 385080.09, + "DecodingStage": 2984.69, + "per_frame_generation": null + }, + "denoise_step_ms": { + "0": 8785.36, + "1": 7644.16, + "2": 7687.27, + "3": 7703.9, + "4": 7710.61, + "5": 7716.32, + "6": 7714.26, + "7": 7711.27, + "8": 7711.08, + "9": 7706.57, + "10": 7700.78, + "11": 7696.03, + "12": 7704.73, + "13": 7699.99, + "14": 7705.33, + "15": 7701.11, + "16": 7704.04, + "17": 7695.31, + "18": 7693.63, + "19": 7686.34, + "20": 7683.27, + "21": 7689.82, + "22": 7688.74, + "23": 7686.01, + "24": 7675.43, + "25": 7679.86, + "26": 7676.75, + "27": 7671.65, + "28": 7667.0, + "29": 7669.83, + "30": 7660.5, + "31": 7666.82, + "32": 7660.89, + "33": 7668.75, + "34": 7662.27, + "35": 7659.71, + "36": 7661.36, + "37": 7664.87, + "38": 7666.93, + "39": 7661.05, + "40": 7661.88, + "41": 7657.96, + "42": 7660.6, + "43": 7669.82, + "44": 7655.78, + "45": 7654.25, + "46": 7656.56, + "47": 7652.37, + "48": 7657.61, + "49": 7644.6 + }, + "expected_e2e_ms": 395758.23, + "expected_avg_denoise_ms": 7701.42, + "expected_median_denoise_ms": 7676.09 + }, + "wan2_1_i2v_14b_720P_2gpu": { "stages_ms": { "InputValidationStage": 53.67, "TextEncodingStage": 2838, - "ImageEncodingStage": 2694.14, - "ConditioningStage": 0.01, - "TimestepPreparationStage": 2.25, - "LatentPreparationStage": 5.27, - "ImageVAEEncodingStage": 1887.72, - "DenoisingStage": 245828.78, - "DecodingStage": 2872.0, - "per_frame_generation": null - }, - "denoise_step_ms": { - "0": 5432.36, - "1": 4867.45, - "2": 4903.52, - "3": 4916.85, - "4": 4928.58, - "5": 4927.72, - "6": 4916.65, - "7": 4910.61, - "8": 4900.32, - "9": 4901.14, - "10": 4907.02, - "11": 4908.95, - "12": 4903.16, - "13": 4910.68, - "14": 4907.48, - "15": 4901.52, - "16": 4909.76, - "17": 4905.68, - "18": 4904.09, - "19": 4908.87, - "20": 4897.14, - "21": 4905.94, - "22": 4916.29, - "23": 4904.12, - "24": 4912.86, - "25": 4904.16, - "26": 4916.7, - "27": 4918.63, - "28": 4903.83, - "29": 4909.2, - "30": 4898.29, - "31": 4901.82, - "32": 4902.38, - "33": 4910.32, - "34": 4903.55, - "35": 4902.21, - "36": 4907.05, - "37": 4911.2, - "38": 4898.33, - "39": 4902.63, - "40": 4895.15, - "41": 4898.9, - "42": 4901.14, - "43": 4896.49, - "44": 4899.01, - "45": 4896.71, - "46": 4887.24, - "47": 4892.91, - "48": 4886.71, - "49": 4885.16 - }, - "expected_e2e_ms": 254809.55, - "expected_avg_denoise_ms": 4914.77, - "expected_median_denoise_ms": 4903.96 - }, - "wan2_2_ti2v_5b": { - "stages_ms": { - "InputValidationStage": 94.18, - "TextEncodingStage": 3413.73, + "ImageEncodingStage": 3123.99, "ConditioningStage": 0.02, - "TimestepPreparationStage": 2.48, - "LatentPreparationStage": 22.52, - "DenoisingStage": 30000, - "DecodingStage": 12793.58, + "TimestepPreparationStage": 3.39, + "LatentPreparationStage": 6.68, + "ImageVAEEncodingStage": 2261.05, + "DenoisingStage": 386761.14, + "DecodingStage": 2968.35, "per_frame_generation": null }, "denoise_step_ms": { - "0": 1021.97, - "1": 407.7, - "2": 410.44, - "3": 411.69, - "4": 411.45, - "5": 410.84, - "6": 411.26, - "7": 412.68, - "8": 412.24, - "9": 410.15, - "10": 413.4, - "11": 410.18, - "12": 411.94, - "13": 411.54, - "14": 409.91, - "15": 412.89, - "16": 412.08, - "17": 411.64, - "18": 411.58, - "19": 410.54, - "20": 411.42, - "21": 412.88, - "22": 412.22, - "23": 412.97, - "24": 412.13, - "25": 413.21, - "26": 413.07, - "27": 410.89, - "28": 411.56, - "29": 414.19, - "30": 412.68, - "31": 411.13, - "32": 412.41, - "33": 412.79, - "34": 411.02, - "35": 410.2, - "36": 410.27, - "37": 411.63, - "38": 410.8, - "39": 411.52, - "40": 411.6, - "41": 411.75, - "42": 410.56, - "43": 411.65, - "44": 411.82, - "45": 410.83, - "46": 410.61, - "47": 411.5, - "48": 410.61, - "49": 411.6 + "0": 10021.98, + "1": 7633.62, + "2": 7676.46, + "3": 7704.68, + "4": 7725.09, + "5": 7732.86, + "6": 7735.42, + "7": 7739.05, + "8": 7740.89, + "9": 7724.35, + "10": 7730.2, + "11": 7713.23, + "12": 7715.93, + "13": 7710.93, + "14": 7699.95, + "15": 7704.72, + "16": 7704.03, + "17": 7700.47, + "18": 7702.0, + "19": 7705.92, + "20": 7704.35, + "21": 7705.11, + "22": 7693.85, + "23": 7696.91, + "24": 7689.6, + "25": 7681.2, + "26": 7675.63, + "27": 7678.95, + "28": 7683.82, + "29": 7681.07, + "30": 7671.07, + "31": 7674.65, + "32": 7679.56, + "33": 7674.59, + "34": 7672.16, + "35": 7679.68, + "36": 7670.81, + "37": 7661.84, + "38": 7668.58, + "39": 7667.1, + "40": 7670.22, + "41": 7664.97, + "42": 7667.3, + "43": 7668.87, + "44": 7663.43, + "45": 7656.34, + "46": 7662.81, + "47": 7662.05, + "48": 7654.13, + "49": 7648.62 }, - "expected_e2e_ms": 32954.73, - "expected_avg_denoise_ms": 423.75, - "expected_median_denoise_ms": 411.59 + "expected_e2e_ms": 397541.45, + "expected_avg_denoise_ms": 7735.02, + "expected_median_denoise_ms": 7681.14 + }, + "wan2_2_t2v_a14b_2gpu": { + "stages_ms": { + "InputValidationStage": 0.09, + "TextEncodingStage": 2322.57, + "ConditioningStage": 0.03, + "TimestepPreparationStage": 2.29, + "LatentPreparationStage": 3.08, + "DenoisingStage": 79913.08, + "DecodingStage": 1339.58 + }, + "denoise_step_ms": { + "0": 19269.37, + "1": 691.64, + "2": 699.28, + "3": 696.55, + "4": 698.6, + "5": 704.56, + "6": 699.26, + "7": 700.84, + "8": 700.27, + "9": 704.15, + "10": 699.04, + "11": 704.79, + "12": 701.48, + "13": 707.24, + "14": 697.54, + "15": 698.89, + "16": 697.97, + "17": 699.34, + "18": 697.68, + "19": 697.42, + "20": 697.14, + "21": 700.14, + "22": 696.75, + "23": 702.36, + "24": 697.3, + "25": 703.97, + "26": 33676.93, + "27": 700.4, + "28": 703.68, + "29": 691.86, + "30": 706.1, + "31": 704.18, + "32": 700.34, + "33": 698.62, + "34": 698.66, + "35": 699.77, + "36": 700.96, + "37": 701.02, + "38": 703.98, + "39": 702.18 + }, + "expected_e2e_ms": 83595.94, + "expected_avg_denoise_ms": 1988.81, + "expected_median_denoise_ms": 700.2 + }, + "wan2_1_t2v_14b_2gpu": { + "stages_ms": { + "InputValidationStage": 0.05, + "TextEncodingStage": 2310.34, + "ConditioningStage": 0.02, + "TimestepPreparationStage": 2.42, + "LatentPreparationStage": 27.7, + "DenoisingStage": 803631.52, + "DecodingStage": 8898.74, + "per_frame_generation": null + }, + "denoise_step_ms": { + "0": 17347.88, + "1": 15956.93, + "2": 16027.54, + "3": 16054.15, + "4": 16081.46, + "5": 16062.7, + "6": 16058.56, + "7": 16057.58, + "8": 16061.04, + "9": 16120.97, + "10": 16036.84, + "11": 16019.6, + "12": 16042.29, + "13": 16039.87, + "14": 16063.0, + "15": 16036.16, + "16": 16079.82, + "17": 16019.7, + "18": 16061.5, + "19": 16039.95, + "20": 16009.42, + "21": 16051.01, + "22": 16039.31, + "23": 16048.22, + "24": 16071.41, + "25": 16078.75, + "26": 16061.78, + "27": 16018.39, + "28": 16041.44, + "29": 16039.64, + "30": 16041.89, + "31": 16039.6, + "32": 16038.97, + "33": 15999.48, + "34": 16019.93, + "35": 16040.27, + "36": 16020.3, + "37": 16039.38, + "38": 15999.4, + "39": 16022.15, + "40": 16042.32, + "41": 16016.62, + "42": 15998.92, + "43": 16041.48, + "44": 15999.63, + "45": 16003.21, + "46": 15995.91, + "47": 16023.52, + "48": 16016.64, + "49": 16019.6 + }, + "expected_e2e_ms": 814884.71, + "expected_avg_denoise_ms": 16062.92, + "expected_median_denoise_ms": 16039.62 } } } diff --git a/python/sglang/multimodal_gen/test/server/test_server_perf_2_gpu.py b/python/sglang/multimodal_gen/test/server/test_server_perf_2_gpu.py new file mode 100644 index 000000000..1a85e9a87 --- /dev/null +++ b/python/sglang/multimodal_gen/test/server/test_server_perf_2_gpu.py @@ -0,0 +1,25 @@ +""" +2 GPU Performance tests (A14B models) with --num-gpus 2 --ulysses-degree 2. +""" + +from __future__ import annotations + +import pytest + +from sglang.multimodal_gen.test.server.test_server_perf_common import ( # noqa: F401 + DiffusionPerformanceBase, + diffusion_server, +) +from sglang.multimodal_gen.test.server.testcase_configs import ( + TWO_GPU_CASES, + DiffusionTestCase, +) + + +class TestDiffusionPerformanceTwoGpu(DiffusionPerformanceBase): + """Performance tests for 2-GPU diffusion cases.""" + + @pytest.fixture(params=TWO_GPU_CASES, ids=lambda c: c.id) + def case(self, request) -> DiffusionTestCase: + """Provide a DiffusionTestCase for each 2-GPU test.""" + return request.param diff --git a/python/sglang/multimodal_gen/test/server/test_server_perf_a.py b/python/sglang/multimodal_gen/test/server/test_server_perf_a.py new file mode 100644 index 000000000..f99b410bf --- /dev/null +++ b/python/sglang/multimodal_gen/test/server/test_server_perf_a.py @@ -0,0 +1,31 @@ +""" +Config-driven diffusion performance test with pytest parametrization. + + +If the actual run is significantly better than the baseline, the improved cases with their updated baseline will be printed +""" + +from __future__ import annotations + +import pytest + +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger +from sglang.multimodal_gen.test.server.test_server_perf_common import ( # noqa: F401 + DiffusionPerformanceBase, + diffusion_server, +) +from sglang.multimodal_gen.test.server.testcase_configs import ( + ONE_GPU_CASES_A, + DiffusionTestCase, +) + +logger = init_logger(__name__) + + +class TestDiffusionPerformanceOneGpu(DiffusionPerformanceBase): + """Performance tests for 1-GPU diffusion cases.""" + + @pytest.fixture(params=ONE_GPU_CASES_A, ids=lambda c: c.id) + def case(self, request) -> DiffusionTestCase: + """Provide a DiffusionTestCase for each 1-GPU test.""" + return request.param diff --git a/python/sglang/multimodal_gen/test/server/test_server_perf_b.py b/python/sglang/multimodal_gen/test/server/test_server_perf_b.py new file mode 100644 index 000000000..f99b410bf --- /dev/null +++ b/python/sglang/multimodal_gen/test/server/test_server_perf_b.py @@ -0,0 +1,31 @@ +""" +Config-driven diffusion performance test with pytest parametrization. + + +If the actual run is significantly better than the baseline, the improved cases with their updated baseline will be printed +""" + +from __future__ import annotations + +import pytest + +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger +from sglang.multimodal_gen.test.server.test_server_perf_common import ( # noqa: F401 + DiffusionPerformanceBase, + diffusion_server, +) +from sglang.multimodal_gen.test.server.testcase_configs import ( + ONE_GPU_CASES_A, + DiffusionTestCase, +) + +logger = init_logger(__name__) + + +class TestDiffusionPerformanceOneGpu(DiffusionPerformanceBase): + """Performance tests for 1-GPU diffusion cases.""" + + @pytest.fixture(params=ONE_GPU_CASES_A, ids=lambda c: c.id) + def case(self, request) -> DiffusionTestCase: + """Provide a DiffusionTestCase for each 1-GPU test.""" + return request.param diff --git a/python/sglang/multimodal_gen/test/server/test_server_performance.py b/python/sglang/multimodal_gen/test/server/test_server_perf_common.py similarity index 95% rename from python/sglang/multimodal_gen/test/server/test_server_performance.py rename to python/sglang/multimodal_gen/test/server/test_server_perf_common.py index 585e010a3..3dc4360c3 100644 --- a/python/sglang/multimodal_gen/test/server/test_server_performance.py +++ b/python/sglang/multimodal_gen/test/server/test_server_perf_common.py @@ -28,7 +28,6 @@ from sglang.multimodal_gen.test.server.test_server_utils import ( ) from sglang.multimodal_gen.test.server.testcase_configs import ( BASELINE_CONFIG, - DIFFUSION_CASES, DiffusionTestCase, PerformanceSummary, ScenarioConfig, @@ -44,24 +43,21 @@ from sglang.multimodal_gen.test.test_utils import ( logger = init_logger(__name__) -@pytest.fixture(params=DIFFUSION_CASES, ids=lambda c: c.id) -def case(request) -> DiffusionTestCase: - """Provide a DiffusionTestCase for each test.""" - return request.param - - @pytest.fixture def diffusion_server(case: DiffusionTestCase) -> ServerContext: """Start a diffusion server for a single case and tear it down afterwards.""" default_port = get_dynamic_server_port() port = int(os.environ.get("SGLANG_TEST_SERVER_PORT", default_port)) + extra_args = os.environ.get("SGLANG_TEST_SERVE_ARGS", "") + extra_args += f" --num-gpus {case.num_gpus} --ulysses-degree {case.num_gpus}" + # start server manager = ServerManager( model=case.model_path, port=port, wait_deadline=float(os.environ.get("SGLANG_TEST_WAIT_SECS", "1200")), - extra_args=os.environ.get("SGLANG_TEST_SERVE_ARGS", ""), + extra_args=extra_args, ) ctx = manager.start() @@ -98,10 +94,10 @@ def diffusion_server(case: DiffusionTestCase) -> ServerContext: ctx.cleanup() -class TestDiffusionPerformance: +class DiffusionPerformanceBase: """Performance tests for all diffusion models/scenarios. - This single test class runs against all cases defined in DIFFUSION_CASES. + This single test class runs against all cases defined in ONE_GPU_CASES. Each case gets its own server instance via the parametrized fixture. """ @@ -148,7 +144,7 @@ Consider updating perf_baselines.json with the snippets below: """Run generation and collect performance records.""" log_path = ctx.perf_log_path prev_len = len(read_perf_logs(log_path)) - log_wait_timeout = 1200 + log_wait_timeout = 30 rid = generate_fn() @@ -200,7 +196,7 @@ Consider updating perf_baselines.json with the snippets below: is_baseline_generation_mode = ( os.environ.get("SGLANG_GEN_BASELINE", "0") == "1" ) - timeout = 3600.0 if is_baseline_generation_mode else 600.0 + timeout = 3600.0 if is_baseline_generation_mode else 1200.0 deadline = time.time() + timeout while True: page = client.videos.list() # type: ignore[attr-defined] @@ -395,7 +391,7 @@ Consider updating perf_baselines.json with the snippets below: summary = validator.collect_metrics(perf_record) if is_baseline_generation_mode or missing_scenario: - self._dump_baseline_for_testcase(case, summary) + self._dump_baseline_for_testcase(case, summary, missing_scenario) if missing_scenario: pytest.fail(f"Testcase '{case.id}' not found in perf_baselines.json") return @@ -406,7 +402,7 @@ Consider updating perf_baselines.json with the snippets below: validator.validate(perf_record, case.num_frames) except AssertionError as e: logger.error(f"Performance validation failed for {case.id}:\n{e}") - self._dump_baseline_for_testcase(case, summary) + self._dump_baseline_for_testcase(case, summary, missing_scenario) raise result = { @@ -510,7 +506,10 @@ Consider updating perf_baselines.json with the snippets below: self._improved_baselines.append({"id": case.id, "baseline": new_baseline}) def _dump_baseline_for_testcase( - self, case: DiffusionTestCase, summary: "PerformanceSummary" + self, + case: DiffusionTestCase, + summary: "PerformanceSummary", + missing_scenario: bool = False, ) -> None: """Dump performance metrics as a JSON scenario for baselines.""" import json @@ -536,10 +535,9 @@ Consider updating perf_baselines.json with the snippets below: if summary.avg_frame_time_ms else None ) - + action = "add" if missing_scenario else "update" output = f""" -To add this baseline, copy the following JSON snippet into -the "scenarios" section of perf_baselines.json: +{action} this baseline in the "scenarios" section of perf_baselines.json: "{case.id}": {json.dumps(baseline, indent=4)} @@ -553,7 +551,7 @@ the "scenarios" section of perf_baselines.json: ): """Single parametrized test that runs for all cases. - Pytest will execute this test once per case in DIFFUSION_CASES, + Pytest will execute this test once per case in ONE_GPU_CASES, with test IDs like: - test_diffusion_perf[qwen_image_text] - test_diffusion_perf[qwen_image_edit] diff --git a/python/sglang/multimodal_gen/test/server/test_server_utils.py b/python/sglang/multimodal_gen/test/server/test_server_utils.py index c5d33dc3c..8817ec094 100644 --- a/python/sglang/multimodal_gen/test/server/test_server_utils.py +++ b/python/sglang/multimodal_gen/test/server/test_server_utils.py @@ -5,6 +5,7 @@ Server management and performance validation for diffusion tests. from __future__ import annotations import os +import shlex import subprocess import sys import tempfile @@ -130,7 +131,9 @@ class ServerManager: env["SGLANG_DIFFUSION_STAGE_LOGGING"] = "1" env["SGLANG_PERF_LOG_DIR"] = log_dir.as_posix() - stdout_fh = stdout_path.open("w", encoding="utf-8", buffering=1) + # TODO: unify with run_command + print(f"Running command: {shlex.join(command)}") + process = subprocess.Popen( command, stdout=subprocess.PIPE, @@ -141,6 +144,7 @@ class ServerManager: ) log_thread = None + stdout_fh = stdout_path.open("w", encoding="utf-8", buffering=1) if process.stdout: def _log_pipe(pipe: Any, file: Any) -> None: diff --git a/python/sglang/multimodal_gen/test/server/testcase_configs.py b/python/sglang/multimodal_gen/test/server/testcase_configs.py index fa8f281fd..c781ec838 100644 --- a/python/sglang/multimodal_gen/test/server/testcase_configs.py +++ b/python/sglang/multimodal_gen/test/server/testcase_configs.py @@ -122,7 +122,7 @@ class DiffusionTestCase: image_path: Path | str | None = None # input image/video for editing (Path or URL) # duration - seconds: int = 4 # for video: duration in seconds + seconds: int = 1 # for video: duration in seconds num_frames: int | None = None # for video: number of frames fps: int | None = None # for video: frames per second @@ -130,6 +130,9 @@ class DiffusionTestCase: warmup_edit: int = 0 # number of image/video-edit warmups custom_validator: str | None = None # optional custom validator name + # resources + num_gpus: int = 1 + def is_image_url(self) -> bool: """Check if image_edit_path is a URL.""" if self.image_path is None: @@ -206,26 +209,9 @@ class PerformanceSummary: ) -# Common paths -IMAGE_INPUT_FILE = Path(__file__).resolve().parents[1] / "test_files" / "girl.jpg" - # All test cases with clean default values # To test different models, simply add more DiffusionCase entries -DIFFUSION_CASES: list[DiffusionTestCase] = [ - # === Image to Video (I2V) === - DiffusionTestCase( - id="wan2_2_i2v_a14b", - model_path="Wan-AI/Wan2.2-I2V-A14B-Diffusers", - modality="video", - prompt="generate", # passing in something since failing if no prompt is passed - warmup_text=0, # warmups only for image gen models - warmup_edit=0, - output_size="832x1104", - edit_prompt="generate", - image_path="https://github.com/Wan-Video/Wan2.2/blob/990af50de458c19590c245151197326e208d7191/examples/i2v_input.JPG?raw=true", - custom_validator="video", - seconds=1, - ), +ONE_GPU_CASES_A: list[DiffusionTestCase] = [ # === Text to Image (T2I) === DiffusionTestCase( id="qwen_image_t2i", @@ -257,46 +243,43 @@ DIFFUSION_CASES: list[DiffusionTestCase] = [ edit_prompt="Convert 2D style to 3D style", image_path="https://github.com/lm-sys/lm-sys.github.io/releases/download/test/TI2I_Qwen_Image_Edit_Input.jpg", ), +] + + +ONE_GPU_CASES_B: list[DiffusionTestCase] = [ # === Text to Video (T2V) === - # TODO: FastWan2.1, FastWan2.2 DiffusionTestCase( id="wan2_1_t2v_1.3b", model_path="Wan-AI/Wan2.1-T2V-1.3B-Diffusers", modality="video", prompt="A curious raccoon", output_size="848x480", - seconds=4, - warmup_text=0, # warmups only for image gen models + warmup_text=0, + warmup_edit=0, + custom_validator="video", + ), + # NOTE(mick): flaky + # DiffusionTestCase( + # id="hunyuan_video", + # model_path="hunyuanvideo-community/HunyuanVideo", + # modality="video", + # prompt="A curious raccoon", + # output_size="720x480", + # warmup_text=0, + # warmup_edit=0, + # custom_validator="video", + # ), + DiffusionTestCase( + id="fast_hunyuan_video", + model_path="FastVideo/FastHunyuan-diffusers", + modality="video", + prompt="A curious raccoon", + output_size="720x480", + warmup_text=0, warmup_edit=0, custom_validator="video", ), # === Text and Image to Video (TI2V) === - DiffusionTestCase( - id="wan2_1_i2v_14b_480P", - model_path="Wan-AI/Wan2.1-I2V-14B-480P-Diffusers", - output_size="832x1104", - modality="video", - prompt="Animate this image", - edit_prompt="Add dynamic motion to the scene", - image_path="https://github.com/lm-sys/lm-sys.github.io/releases/download/test/TI2I_Qwen_Image_Edit_Input.jpg", - warmup_text=0, # warmups only for image gen models - warmup_edit=0, - custom_validator="video", - seconds=1, - ), - DiffusionTestCase( - id="wan2_1_i2v_14b_720P", - model_path="Wan-AI/Wan2.1-I2V-14B-720P-Diffusers", - modality="video", - prompt="Animate this image", - edit_prompt="Add dynamic motion to the scene", - image_path="https://github.com/lm-sys/lm-sys.github.io/releases/download/test/TI2I_Qwen_Image_Edit_Input.jpg", - output_size="832x1104", - warmup_text=0, # warmups only for image gen models - warmup_edit=0, - custom_validator="video", - seconds=1, - ), DiffusionTestCase( id="wan2_2_ti2v_5b", model_path="Wan-AI/Wan2.2-TI2V-5B-Diffusers", @@ -305,10 +288,86 @@ DIFFUSION_CASES: list[DiffusionTestCase] = [ prompt="Animate this image", edit_prompt="Add dynamic motion to the scene", image_path="https://github.com/lm-sys/lm-sys.github.io/releases/download/test/TI2I_Qwen_Image_Edit_Input.jpg", - warmup_text=0, # warmups only for image gen models + warmup_text=0, warmup_edit=0, custom_validator="video", - seconds=1, + ), + DiffusionTestCase( + id="fastwan2_2_ti2v_5b", + model_path="FastVideo/FastWan2.2-TI2V-5B-FullAttn-Diffusers", + modality="video", + output_size="832x1104", + prompt="Animate this image", + edit_prompt="Add dynamic motion to the scene", + image_path="https://github.com/lm-sys/lm-sys.github.io/releases/download/test/TI2I_Qwen_Image_Edit_Input.jpg", + warmup_text=0, + warmup_edit=0, + custom_validator="video", + ), +] + +TWO_GPU_CASES = [ + DiffusionTestCase( + id="wan2_2_i2v_a14b_2gpu", + model_path="Wan-AI/Wan2.2-I2V-A14B-Diffusers", + modality="video", + prompt="generate", + warmup_text=0, + warmup_edit=0, + output_size="832x1104", + edit_prompt="generate", + image_path="https://github.com/Wan-Video/Wan2.2/blob/990af50de458c19590c245151197326e208d7191/examples/i2v_input.JPG?raw=true", + custom_validator="video", + num_gpus=2, + num_frames=1, + ), + DiffusionTestCase( + id="wan2_2_t2v_a14b_2gpu", + model_path="Wan-AI/Wan2.2-T2V-A14B-Diffusers", + modality="video", + prompt="A curious raccoon", + output_size="720x480", + warmup_text=0, + warmup_edit=0, + custom_validator="video", + num_gpus=2, + ), + DiffusionTestCase( + id="wan2_1_t2v_14b_2gpu", + model_path="Wan-AI/Wan2.1-T2V-14B-Diffusers", + modality="video", + prompt="A curious raccoon", + output_size="720x480", + warmup_text=0, + warmup_edit=0, + custom_validator="video", + num_gpus=2, + ), + DiffusionTestCase( + id="wan2_1_i2v_14b_480P_2gpu", + model_path="Wan-AI/Wan2.1-I2V-14B-480P-Diffusers", + output_size="832x1104", + modality="video", + prompt="Animate this image", + edit_prompt="Add dynamic motion to the scene", + image_path="https://github.com/lm-sys/lm-sys.github.io/releases/download/test/TI2I_Qwen_Image_Edit_Input.jpg", + warmup_text=0, + warmup_edit=0, + custom_validator="video", + num_gpus=2, + ), + DiffusionTestCase( + id="wan2_1_i2v_14b_720P_2gpu", + model_path="Wan-AI/Wan2.1-I2V-14B-720P-Diffusers", + modality="video", + prompt="Animate this image", + edit_prompt="Add dynamic motion to the scene", + image_path="https://github.com/lm-sys/lm-sys.github.io/releases/download/test/TI2I_Qwen_Image_Edit_Input.jpg", + output_size="832x1104", + warmup_text=0, + warmup_edit=0, + custom_validator="video", + num_gpus=2, ), ] diff --git a/python/sglang/multimodal_gen/test/test_files/girl.jpg b/python/sglang/multimodal_gen/test/test_files/girl.jpg deleted file mode 100644 index 63ba28d39..000000000 Binary files a/python/sglang/multimodal_gen/test/test_files/girl.jpg and /dev/null differ diff --git a/python/sglang/multimodal_gen/test/test_utils.py b/python/sglang/multimodal_gen/test/test_utils.py index 457406892..33e60a919 100644 --- a/python/sglang/multimodal_gen/test/test_utils.py +++ b/python/sglang/multimodal_gen/test/test_utils.py @@ -164,7 +164,7 @@ def wait_for_req_perf_record( request_id: str, prev_len: int, log_path: Path, - timeout: float = 300.0, + timeout: float = 30.0, ) -> tuple[RequestPerfRecord | None, int]: """ the stage metrics of this request should be in the performance_log file with {request-id} @@ -173,7 +173,7 @@ def wait_for_req_perf_record( deadline = time.time() + timeout while time.time() < deadline: records = read_perf_logs(log_path) - if len(records) == prev_len + 1: + if len(records) >= prev_len + 1: # FIXME: unable to get rid from openai apis, this is a hack. we should compare rid # potential error when there are multiple servers return records[-1], len(records) @@ -183,6 +183,8 @@ def wait_for_req_perf_record( if os.environ.get("SGLANG_GEN_BASELINE", "0") == "1": records = read_perf_logs(log_path) return None, len(records) + + logger.error(f"record: {records}") raise AssertionError(f"Timeout waiting for stage metrics for request {request_id} ")