[diffusion] feat: support diffusers backend - run any model supported by diffusers (#14112)

python/sglang/multimodal_gen/configs/pipeline_configs/__init__.py

@@ -4,6 +4,9 @@ from sglang.multimodal_gen.configs.pipeline_configs.base import (
     PipelineConfig,
     SlidingTileAttnConfig,
 )
+from sglang.multimodal_gen.configs.pipeline_configs.diffusers_generic import (
+    DiffusersGenericPipelineConfig,
+)
 from sglang.multimodal_gen.configs.pipeline_configs.flux import FluxPipelineConfig
 from sglang.multimodal_gen.configs.pipeline_configs.flux_finetuned import (
     Flux2FinetunedPipelineConfig,
@@ -22,6 +25,7 @@ from sglang.multimodal_gen.configs.pipeline_configs.wan import (
 from sglang.multimodal_gen.configs.pipeline_configs.zimage import ZImagePipelineConfig
 
 __all__ = [
+    "DiffusersGenericPipelineConfig",
     "HunyuanConfig",
     "FastHunyuanConfig",
     "FluxPipelineConfig",

python/sglang/multimodal_gen/configs/pipeline_configs/diffusers_generic.py

@@ -0,0 +1,84 @@
+# SPDX-License-Identifier: Apache-2.0
+"""
+Generic pipeline configuration for diffusers backend.
+
+This module provides a minimal pipeline configuration that works with the diffusers backend.
+Since diffusers handles its own model loading and configuration, this config is intentionally minimal.
+"""
+
+from dataclasses import dataclass, field
+from typing import Any
+
+from sglang.multimodal_gen.configs.models import DiTConfig, EncoderConfig, VAEConfig
+from sglang.multimodal_gen.configs.pipeline_configs.base import (
+    ModelTaskType,
+    PipelineConfig,
+)
+
+
+@dataclass
+class DiffusersGenericPipelineConfig(PipelineConfig):
+    """
+    Generic pipeline configuration for diffusers backend.
+
+    This is a minimal configuration since the diffusers backend handles most
+    configuration internally. It provides sensible defaults for the required fields.
+    """
+
+    # default to T2I since it's the most common
+    task_type: ModelTaskType = ModelTaskType.T2I
+
+    dit_precision: str = "bf16"
+    vae_precision: str = "bf16"
+
+    should_use_guidance: bool = True
+    embedded_cfg_scale: float = 1.0
+    flow_shift: float | None = None
+    disable_autocast: bool = True  # let diffusers handle dtype
+
+    # diffusers handles its own loading
+    dit_config: DiTConfig = field(default_factory=DiTConfig)
+    vae_config: VAEConfig = field(default_factory=VAEConfig)
+    image_encoder_config: EncoderConfig = field(default_factory=EncoderConfig)
+    text_encoder_configs: tuple[EncoderConfig, ...] = field(
+        default_factory=lambda: (EncoderConfig(),)
+    )
+    text_encoder_precisions: tuple[str, ...] = field(default_factory=lambda: ("fp16",))
+
+    # VAE settings
+    vae_tiling: bool = False  # diffusers handles this
+    vae_slicing: bool = False  # slice VAE decode for lower memory usage
+    vae_sp: bool = False
+
+    # Attention backend for diffusers models (e.g., "flash", "_flash_3_hub", "sage", "xformers")
+    # See: https://huggingface.co/docs/diffusers/main/en/optimization/attention_backends
+    diffusers_attention_backend: str | None = None
+
+    # Quantization config for pipeline-level quantization
+    # See: https://huggingface.co/docs/diffusers/main/en/quantization/overview
+    # Use PipelineQuantizationConfig for component-level control:
+    #   from diffusers.quantizers import PipelineQuantizationConfig
+    #   quantization_config = PipelineQuantizationConfig(
+    #       quant_backend="bitsandbytes_4bit",
+    #       quant_kwargs={"load_in_4bit": True, "bnb_4bit_compute_dtype": torch.bfloat16},
+    #       components_to_quantize=["transformer", "text_encoder_2"],
+    #   )
+    quantization_config: Any = None
+
+    def check_pipeline_config(self) -> None:
+        """
+        Override to skip most validation since diffusers handles its own config.
+        """
+        pass
+
+    def adjust_size(self, width, height, image):
+        """
+        Pass through - diffusers handles size adjustments.
+        """
+        return width, height
+
+    def adjust_num_frames(self, num_frames):
+        """
+        Pass through - diffusers handles frame count.
+        """
+        return num_frames

python/sglang/multimodal_gen/configs/sample/__init__.py

@@ -1,5 +1,8 @@
 # Copied and adapted from: https://github.com/hao-ai-lab/FastVideo
 
+from sglang.multimodal_gen.configs.sample.diffusers_generic import (
+    DiffusersGenericSamplingParams,
+)
 from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams
 
-__all__ = ["SamplingParams"]
+__all__ = ["SamplingParams", "DiffusersGenericSamplingParams"]

python/sglang/multimodal_gen/configs/sample/diffusers_generic.py

@@ -0,0 +1,52 @@
+# SPDX-License-Identifier: Apache-2.0
+"""
+Generic sampling parameters for diffusers backend.
+
+This module provides generic sampling parameters that work with any diffusers pipeline.
+"""
+
+from dataclasses import dataclass, field
+from typing import Any
+
+from sglang.multimodal_gen.configs.sample.sampling_params import (
+    DataType,
+    SamplingParams,
+)
+
+
+@dataclass
+class DiffusersGenericSamplingParams(SamplingParams):
+    """
+    Generic sampling parameters for diffusers backend.
+
+    These parameters cover the most common options across different diffusers pipelines.
+    The diffusers pipeline will use whichever parameters it supports.
+
+    For pipeline-specific parameters, use `diffusers_kwargs` dict which will be
+    passed directly to the diffusers pipeline call.
+    """
+
+    # Override defaults with more conservative values that work across pipelines
+    num_frames: int = 1  # default to image generation
+    height: int = 1024
+    width: int = 1024
+    num_inference_steps: int = 30
+    guidance_scale: float = 7.5
+    negative_prompt: str = ""
+
+    # extra kwargs to pass directly to the diffusers pipeline
+    # example: {"output_type": "latent", "return_dict": False}
+    diffusers_kwargs: dict[str, Any] = field(default_factory=dict)
+
+    def __post_init__(self) -> None:
+        if self.num_frames > 1:
+            self.data_type = DataType.VIDEO
+        else:
+            self.data_type = DataType.IMAGE
+
+        if self.width is None:
+            self.width_not_provided = True
+            self.width = 1024
+        if self.height is None:
+            self.height_not_provided = True
+            self.height = 1024

python/sglang/multimodal_gen/configs/sample/sampling_params.py

@@ -124,6 +124,7 @@ class SamplingParams:
     num_inference_steps: int = None
     guidance_scale: float = None
     guidance_scale_2: float = None
+    true_cfg_scale: float = None  # for CFG vs guidance distillation (e.g., QwenImage)
     guidance_rescale: float = 0.0
     boundary_ratio: float | None = None
 
@@ -580,6 +581,13 @@ class SamplingParams:
             default=SamplingParams.return_trajectory_decoded,
             help="Whether to return the decoded trajectory",
         )
+        parser.add_argument(
+            "--diffusers-kwargs",
+            type=str,
+            default=None,
+            help="JSON string of extra kwargs to pass to diffusers pipeline. "
+            'Example: \'{"output_type": "latent", "clip_skip": 2}\'',
+        )
         parser.add_argument(
             "--no-override-protected-fields",
             action="store_true",