[diffusion] CI: minor refactor CI for less code duplication (#13905)

2025-11-25 18:44:11 +08:00
parent 59464dbf15
commit 6f094deff0
11 changed files with 516 additions and 597 deletions
@@ -108,17 +108,27 @@ class BaselineConfig:


@dataclass(frozen=True)
-class DiffusionTestCase:
+class DiffusionServerArgs:
    """Configuration for a single model/scenario test case."""

-    id: str  # pytest test id and scenario name
    model_path: str  # HF repo or local path
    modality: str = "image"  # "image" or "video" or "3d"
+
+    warmup_text: int = 1  # number of text-to-image/video warmups
+    warmup_edit: int = 0  # number of image/video-edit warmups
+    custom_validator: str | None = None  # optional custom validator name
+    # resources
+    num_gpus: int = 1
+
+
+@dataclass(frozen=True)
+class DiffusionSamplingParams:
+    """Configuration for a single model/scenario test case."""
+
    output_size: str = "1024x1024"  # output image dimensions (or video resolution)

    # inputs and conditioning
    prompt: str | None = None  # text prompt for generation
-    edit_prompt: str | None = None  # prompt for editing
    image_path: Path | str | None = None  # input image/video for editing (Path or URL)

    # duration
@@ -126,21 +136,14 @@ class DiffusionTestCase:
    num_frames: int | None = None  # for video: number of frames
    fps: int | None = None  # for video: frames per second

-    warmup_text: int = 1  # number of text-to-image/video warmups
-    warmup_edit: int = 0  # number of image/video-edit warmups
-    custom_validator: str | None = None  # optional custom validator name

-    # resources
-    num_gpus: int = 1
+@dataclass(frozen=True)
+class DiffusionTestCase:
+    """Configuration for a single model/scenario test case."""

-    def is_image_url(self) -> bool:
-        """Check if image_edit_path is a URL."""
-        if self.image_path is None:
-            return False
-        return isinstance(self.image_path, str) and (
-            self.image_path.startswith("http://")
-            or self.image_path.startswith("https://")
-        )
+    id: str  # pytest test id and scenario name
+    server_args: DiffusionServerArgs
+    sampling_params: DiffusionSamplingParams


 def sample_step_indices(
@@ -214,49 +217,63 @@ class PerformanceSummary:
 ONE_GPU_CASES_A: list[DiffusionTestCase] = [
    # === Text to Image (T2I) ===
    DiffusionTestCase(
-        id="qwen_image_t2i",
-        model_path="Qwen/Qwen-Image",
-        modality="image",
-        prompt="A futuristic cityscape at sunset with flying cars",
-        output_size="1024x1024",
-        warmup_text=1,
-        warmup_edit=0,
+        "qwen_image_t2i",
+        DiffusionServerArgs(
+            model_path="Qwen/Qwen-Image",
+            modality="image",
+            warmup_text=1,
+            warmup_edit=0,
+        ),
+        DiffusionSamplingParams(
+            prompt="A futuristic cityscape at sunset with flying cars",
+            output_size="1024x1024",
+        ),
    ),
    DiffusionTestCase(
-        id="flux_image_t2i",
-        model_path="black-forest-labs/FLUX.1-dev",
-        modality="image",
-        prompt="A futuristic cityscape at sunset with flying cars",
-        output_size="1024x1024",
-        warmup_text=1,
-        warmup_edit=0,
+        "flux_image_t2i",
+        DiffusionServerArgs(
+            model_path="black-forest-labs/FLUX.1-dev",
+            modality="image",
+            warmup_text=1,
+            warmup_edit=0,
+        ),
+        DiffusionSamplingParams(
+            prompt="A futuristic cityscape at sunset with flying cars",
+            output_size="1024x1024",
+        ),
    ),
    # === Text and Image to Image (TI2I) ===
    DiffusionTestCase(
-        id="qwen_image_edit_ti2i",
-        model_path="Qwen/Qwen-Image-Edit",
-        modality="image",
-        prompt=None,  # not used for editing
-        output_size="1024x1536",
-        warmup_text=0,
-        warmup_edit=1,
-        edit_prompt="Convert 2D style to 3D style",
-        image_path="https://github.com/lm-sys/lm-sys.github.io/releases/download/test/TI2I_Qwen_Image_Edit_Input.jpg",
+        "qwen_image_edit_ti2i",
+        DiffusionServerArgs(
+            model_path="Qwen/Qwen-Image-Edit",
+            warmup_text=0,
+            warmup_edit=1,
+            modality="image",
+        ),
+        DiffusionSamplingParams(
+            prompt="Convert 2D style to 3D style",
+            output_size="1024x1536",
+            image_path="https://github.com/lm-sys/lm-sys.github.io/releases/download/test/TI2I_Qwen_Image_Edit_Input.jpg",
+        ),
    ),
 ]

-
 ONE_GPU_CASES_B: list[DiffusionTestCase] = [
    # === Text to Video (T2V) ===
    DiffusionTestCase(
-        id="wan2_1_t2v_1.3b",
-        model_path="Wan-AI/Wan2.1-T2V-1.3B-Diffusers",
-        modality="video",
-        prompt="A curious raccoon",
-        output_size="848x480",
-        warmup_text=0,
-        warmup_edit=0,
-        custom_validator="video",
+        "wan2_1_t2v_1.3b",
+        DiffusionServerArgs(
+            model_path="Wan-AI/Wan2.1-T2V-1.3B-Diffusers",
+            modality="video",
+            warmup_text=0,
+            warmup_edit=0,
+            custom_validator="video",
+        ),
+        DiffusionSamplingParams(
+            prompt="A curious raccoon",
+            output_size="848x480",
+        ),
    ),
    # NOTE(mick): flaky
    # DiffusionTestCase(
@@ -270,126 +287,161 @@ ONE_GPU_CASES_B: list[DiffusionTestCase] = [
    #     custom_validator="video",
    # ),
    DiffusionTestCase(
-        id="fast_hunyuan_video",
-        model_path="FastVideo/FastHunyuan-diffusers",
-        modality="video",
-        prompt="A curious raccoon",
-        output_size="720x480",
-        warmup_text=0,
-        warmup_edit=0,
-        custom_validator="video",
+        "fast_hunyuan_video",
+        DiffusionServerArgs(
+            model_path="FastVideo/FastHunyuan-diffusers",
+            modality="video",
+            warmup_text=0,
+            warmup_edit=0,
+            custom_validator="video",
+        ),
+        DiffusionSamplingParams(
+            prompt="A curious raccoon",
+            output_size="720x480",
+        ),
    ),
    # === Text and Image to Video (TI2V) ===
    DiffusionTestCase(
-        id="wan2_2_ti2v_5b",
-        model_path="Wan-AI/Wan2.2-TI2V-5B-Diffusers",
-        modality="video",
-        output_size="832x1104",
-        prompt="Animate this image",
-        edit_prompt="Add dynamic motion to the scene",
-        image_path="https://github.com/lm-sys/lm-sys.github.io/releases/download/test/TI2I_Qwen_Image_Edit_Input.jpg",
-        warmup_text=0,
-        warmup_edit=0,
-        custom_validator="video",
+        "wan2_2_ti2v_5b",
+        DiffusionServerArgs(
+            model_path="Wan-AI/Wan2.2-TI2V-5B-Diffusers",
+            modality="video",
+            warmup_text=0,
+            warmup_edit=0,
+            custom_validator="video",
+        ),
+        DiffusionSamplingParams(
+            output_size="832x1104",
+            prompt="Add dynamic motion to the scene",
+            image_path="https://github.com/lm-sys/lm-sys.github.io/releases/download/test/TI2I_Qwen_Image_Edit_Input.jpg",
+        ),
    ),
    DiffusionTestCase(
-        id="fastwan2_2_ti2v_5b",
-        model_path="FastVideo/FastWan2.2-TI2V-5B-FullAttn-Diffusers",
-        modality="video",
-        output_size="832x1104",
-        prompt="Animate this image",
-        edit_prompt="Add dynamic motion to the scene",
-        image_path="https://github.com/lm-sys/lm-sys.github.io/releases/download/test/TI2I_Qwen_Image_Edit_Input.jpg",
-        warmup_text=0,
-        warmup_edit=0,
-        custom_validator="video",
+        "fastwan2_2_ti2v_5b",
+        DiffusionServerArgs(
+            model_path="FastVideo/FastWan2.2-TI2V-5B-FullAttn-Diffusers",
+            modality="video",
+            warmup_text=0,
+            warmup_edit=0,
+            custom_validator="video",
+        ),
+        DiffusionSamplingParams(
+            output_size="832x1104",
+            prompt="Add dynamic motion to the scene",
+            image_path="https://github.com/lm-sys/lm-sys.github.io/releases/download/test/TI2I_Qwen_Image_Edit_Input.jpg",
+        ),
    ),
 ]

 TWO_GPU_CASES_A = [
    DiffusionTestCase(
-        id="wan2_2_i2v_a14b_2gpu",
-        model_path="Wan-AI/Wan2.2-I2V-A14B-Diffusers",
-        modality="video",
-        prompt="generate",
-        warmup_text=0,
-        warmup_edit=0,
-        output_size="832x1104",
-        edit_prompt="generate",
-        image_path="https://github.com/Wan-Video/Wan2.2/blob/990af50de458c19590c245151197326e208d7191/examples/i2v_input.JPG?raw=true",
-        custom_validator="video",
-        num_gpus=2,
-        num_frames=1,
+        "wan2_2_i2v_a14b_2gpu",
+        DiffusionServerArgs(
+            model_path="Wan-AI/Wan2.2-I2V-A14B-Diffusers",
+            modality="video",
+            warmup_text=0,
+            warmup_edit=0,
+            custom_validator="video",
+            num_gpus=2,
+        ),
+        DiffusionSamplingParams(
+            prompt="generate",
+            output_size="832x1104",
+            image_path="https://github.com/Wan-Video/Wan2.2/blob/990af50de458c19590c245151197326e208d7191/examples/i2v_input.JPG?raw=true",
+            num_frames=1,
+        ),
    ),
    DiffusionTestCase(
-        id="wan2_2_t2v_a14b_2gpu",
-        model_path="Wan-AI/Wan2.2-T2V-A14B-Diffusers",
-        modality="video",
-        prompt="A curious raccoon",
-        output_size="720x480",
-        warmup_text=0,
-        warmup_edit=0,
-        custom_validator="video",
-        num_gpus=2,
+        "wan2_2_t2v_a14b_2gpu",
+        DiffusionServerArgs(
+            model_path="Wan-AI/Wan2.2-T2V-A14B-Diffusers",
+            modality="video",
+            warmup_text=0,
+            warmup_edit=0,
+            custom_validator="video",
+            num_gpus=2,
+        ),
+        DiffusionSamplingParams(
+            prompt="A curious raccoon",
+            output_size="720x480",
+        ),
    ),
    DiffusionTestCase(
-        id="wan2_1_t2v_14b_2gpu",
-        model_path="Wan-AI/Wan2.1-T2V-14B-Diffusers",
-        modality="video",
-        prompt="A curious raccoon",
-        output_size="720x480",
-        warmup_text=0,
-        warmup_edit=0,
-        custom_validator="video",
-        num_gpus=2,
+        "wan2_1_t2v_14b_2gpu",
+        DiffusionServerArgs(
+            model_path="Wan-AI/Wan2.1-T2V-14B-Diffusers",
+            warmup_text=0,
+            warmup_edit=0,
+            modality="video",
+            num_gpus=2,
+            custom_validator="video",
+        ),
+        DiffusionSamplingParams(
+            prompt="A curious raccoon",
+            output_size="720x480",
+        ),
    ),
 ]

 TWO_GPU_CASES_B = [
    DiffusionTestCase(
-        id="wan2_1_i2v_14b_480P_2gpu",
-        model_path="Wan-AI/Wan2.1-I2V-14B-480P-Diffusers",
-        output_size="832x1104",
-        modality="video",
-        prompt="Animate this image",
-        edit_prompt="Add dynamic motion to the scene",
-        image_path="https://github.com/lm-sys/lm-sys.github.io/releases/download/test/TI2I_Qwen_Image_Edit_Input.jpg",
-        warmup_text=0,
-        warmup_edit=0,
-        custom_validator="video",
-        num_gpus=2,
+        "wan2_1_i2v_14b_480P_2gpu",
+        DiffusionServerArgs(
+            model_path="Wan-AI/Wan2.1-I2V-14B-480P-Diffusers",
+            warmup_text=0,
+            warmup_edit=0,
+            modality="video",
+            custom_validator="video",
+            num_gpus=2,
+        ),
+        DiffusionSamplingParams(
+            output_size="832x1104",
+            prompt="Add dynamic motion to the scene",
+            image_path="https://github.com/lm-sys/lm-sys.github.io/releases/download/test/TI2I_Qwen_Image_Edit_Input.jpg",
+        ),
    ),
    DiffusionTestCase(
-        id="wan2_1_i2v_14b_720P_2gpu",
-        model_path="Wan-AI/Wan2.1-I2V-14B-720P-Diffusers",
-        modality="video",
-        prompt="Animate this image",
-        edit_prompt="Add dynamic motion to the scene",
-        image_path="https://github.com/lm-sys/lm-sys.github.io/releases/download/test/TI2I_Qwen_Image_Edit_Input.jpg",
-        output_size="832x1104",
-        warmup_text=0,
-        warmup_edit=0,
-        custom_validator="video",
-        num_gpus=2,
+        "wan2_1_i2v_14b_720P_2gpu",
+        DiffusionServerArgs(
+            model_path="Wan-AI/Wan2.1-I2V-14B-720P-Diffusers",
+            modality="video",
+            warmup_text=0,
+            warmup_edit=0,
+            custom_validator="video",
+            num_gpus=2,
+        ),
+        DiffusionSamplingParams(
+            prompt="Add dynamic motion to the scene",
+            image_path="https://github.com/lm-sys/lm-sys.github.io/releases/download/test/TI2I_Qwen_Image_Edit_Input.jpg",
+            output_size="832x1104",
+        ),
    ),
    DiffusionTestCase(
-        id="qwen_image_t2i_2_gpus",
-        model_path="Qwen/Qwen-Image",
-        modality="image",
-        prompt="A futuristic cityscape at sunset with flying cars",
-        output_size="1024x1024",
-        warmup_text=1,
-        warmup_edit=0,
-        num_gpus=2,
+        "qwen_image_t2i_2_gpus",
+        DiffusionServerArgs(
+            model_path="Qwen/Qwen-Image",
+            modality="image",
+            warmup_text=1,
+            warmup_edit=0,
+            num_gpus=2,
+        ),
+        DiffusionSamplingParams(
+            prompt="A futuristic cityscape at sunset with flying cars",
+            output_size="1024x1024",
+        ),
    ),
    DiffusionTestCase(
-        id="flux_image_t2i_2_gpus",
-        model_path="black-forest-labs/FLUX.1-dev",
-        modality="image",
-        prompt="A futuristic cityscape at sunset with flying cars",
-        output_size="1024x1024",
-        warmup_text=1,
-        warmup_edit=0,
+        "flux_image_t2i_2_gpus",
+        DiffusionServerArgs(
+            model_path="black-forest-labs/FLUX.1-dev",
+            modality="image",
+            warmup_text=1,
+            warmup_edit=0,
+        ),
+        DiffusionSamplingParams(
+            prompt="A futuristic cityscape at sunset with flying cars",
+            output_size="1024x1024",
+        ),
    ),
 ]