867 lines
31 KiB
Python
867 lines
31 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
"""
|
|
Central registry for multimodal models.
|
|
|
|
This module provides a centralized registry for multimodal models, including pipelines
|
|
and sampling parameters. It allows for easy registration and retrieval of model
|
|
information based on model paths or other identifiers.
|
|
"""
|
|
|
|
import dataclasses
|
|
import importlib
|
|
import os
|
|
import pkgutil
|
|
import sys
|
|
from functools import lru_cache
|
|
from typing import (
|
|
TYPE_CHECKING,
|
|
Any,
|
|
Callable,
|
|
Dict,
|
|
List,
|
|
Optional,
|
|
Tuple,
|
|
Type,
|
|
Union,
|
|
)
|
|
|
|
if TYPE_CHECKING:
|
|
from sglang.multimodal_gen.runtime.server_args import Backend
|
|
|
|
from sglang.multimodal_gen.configs.pipeline_configs import (
|
|
FastHunyuanConfig,
|
|
FluxPipelineConfig,
|
|
HeliosDistilledConfig,
|
|
HeliosMidConfig,
|
|
HeliosT2VConfig,
|
|
HunyuanConfig,
|
|
WanI2V480PConfig,
|
|
WanI2V720PConfig,
|
|
WanT2V480PConfig,
|
|
WanT2V720PConfig,
|
|
ZImagePipelineConfig,
|
|
)
|
|
from sglang.multimodal_gen.configs.pipeline_configs.base import PipelineConfig
|
|
from sglang.multimodal_gen.configs.pipeline_configs.flux import (
|
|
Flux2KleinPipelineConfig,
|
|
Flux2PipelineConfig,
|
|
)
|
|
from sglang.multimodal_gen.configs.pipeline_configs.glm_image import (
|
|
GlmImagePipelineConfig,
|
|
)
|
|
from sglang.multimodal_gen.configs.pipeline_configs.hunyuan3d import (
|
|
Hunyuan3D2PipelineConfig,
|
|
)
|
|
from sglang.multimodal_gen.configs.pipeline_configs.ltx_2 import LTX2PipelineConfig
|
|
from sglang.multimodal_gen.configs.pipeline_configs.mova import (
|
|
MOVA360PConfig,
|
|
MOVA720PConfig,
|
|
)
|
|
from sglang.multimodal_gen.configs.pipeline_configs.qwen_image import (
|
|
QwenImageEditPipelineConfig,
|
|
QwenImageEditPlus_2511_PipelineConfig,
|
|
QwenImageEditPlusPipelineConfig,
|
|
QwenImageLayeredPipelineConfig,
|
|
QwenImagePipelineConfig,
|
|
)
|
|
from sglang.multimodal_gen.configs.pipeline_configs.sana import SanaPipelineConfig
|
|
from sglang.multimodal_gen.configs.pipeline_configs.wan import (
|
|
FastWan2_1_T2V_480P_Config,
|
|
FastWan2_2_TI2V_5B_Config,
|
|
TurboWanI2V720Config,
|
|
TurboWanT2V480PConfig,
|
|
Wan2_2_I2V_A14B_Config,
|
|
Wan2_2_T2V_A14B_Config,
|
|
Wan2_2_TI2V_5B_Config,
|
|
)
|
|
from sglang.multimodal_gen.configs.sample.flux import (
|
|
Flux2KleinSamplingParams,
|
|
FluxSamplingParams,
|
|
)
|
|
from sglang.multimodal_gen.configs.sample.glmimage import GlmImageSamplingParams
|
|
from sglang.multimodal_gen.configs.sample.helios import (
|
|
HeliosDistilledSamplingParams,
|
|
HeliosMidSamplingParams,
|
|
HeliosT2VSamplingParams,
|
|
)
|
|
from sglang.multimodal_gen.configs.sample.hunyuan import (
|
|
FastHunyuanSamplingParam,
|
|
HunyuanSamplingParams,
|
|
)
|
|
from sglang.multimodal_gen.configs.sample.hunyuan3d import Hunyuan3DSamplingParams
|
|
from sglang.multimodal_gen.configs.sample.ltx_2 import LTX2SamplingParams
|
|
from sglang.multimodal_gen.configs.sample.mova import (
|
|
MOVA_360P_SamplingParams,
|
|
MOVA_720P_SamplingParams,
|
|
)
|
|
from sglang.multimodal_gen.configs.sample.qwenimage import (
|
|
QwenImage2512SamplingParams,
|
|
QwenImageEditPlusSamplingParams,
|
|
QwenImageLayeredSamplingParams,
|
|
QwenImageSamplingParams,
|
|
)
|
|
from sglang.multimodal_gen.configs.sample.sana import SanaSamplingParams
|
|
from sglang.multimodal_gen.configs.sample.wan import (
|
|
FastWanT2V480PConfig,
|
|
Turbo_Wan2_2_I2V_A14B_SamplingParam,
|
|
Wan2_1_Fun_1_3B_InP_SamplingParams,
|
|
Wan2_2_I2V_A14B_SamplingParam,
|
|
Wan2_2_T2V_A14B_SamplingParam,
|
|
Wan2_2_TI2V_5B_SamplingParam,
|
|
WanI2V_14B_480P_SamplingParam,
|
|
WanI2V_14B_720P_SamplingParam,
|
|
WanT2V_1_3B_SamplingParams,
|
|
WanT2V_14B_SamplingParams,
|
|
)
|
|
from sglang.multimodal_gen.configs.sample.zimage import (
|
|
ZImageSamplingParams,
|
|
ZImageTurboSamplingParams,
|
|
)
|
|
from sglang.multimodal_gen.runtime.pipelines_core.composed_pipeline_base import (
|
|
ComposedPipelineBase,
|
|
)
|
|
from sglang.multimodal_gen.runtime.utils.hf_diffusers_utils import (
|
|
maybe_download_model_index,
|
|
verify_model_config_and_directory,
|
|
)
|
|
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
|
|
|
|
logger = init_logger(__name__)
|
|
|
|
# --- Part 1: Pipeline Discovery ---
|
|
|
|
_PIPELINE_REGISTRY: Dict[str, Type[ComposedPipelineBase]] = {}
|
|
|
|
# Registry for pipeline configuration classes (for safetensors files without model_index.json)
|
|
# Maps pipeline_class_name -> (PipelineConfig class, SamplingParams class)
|
|
_PIPELINE_CONFIG_REGISTRY: Dict[str, Tuple[Type[PipelineConfig], Type[Any]]] = {}
|
|
|
|
|
|
def _discover_and_register_pipelines():
|
|
"""
|
|
Automatically discover and register all ComposedPipelineBase subclasses.
|
|
This function scans the 'sglang.multimodal_gen.runtime.pipelines' package,
|
|
finds modules with an 'EntryClass' attribute, and maps the class's 'pipeline_name'
|
|
to the class itself in a global registry.
|
|
"""
|
|
if _PIPELINE_REGISTRY: # run only once
|
|
return
|
|
|
|
package_name = "sglang.multimodal_gen.runtime.pipelines"
|
|
package = importlib.import_module(package_name)
|
|
|
|
for _, module_name, ispkg in pkgutil.walk_packages(
|
|
package.__path__, package.__name__ + "."
|
|
):
|
|
if not ispkg:
|
|
pipeline_module = importlib.import_module(module_name)
|
|
if hasattr(pipeline_module, "EntryClass"):
|
|
entry_cls = pipeline_module.EntryClass
|
|
entry_cls_list = (
|
|
[entry_cls] if not isinstance(entry_cls, list) else entry_cls
|
|
)
|
|
|
|
for cls in entry_cls_list:
|
|
if not issubclass(cls, ComposedPipelineBase):
|
|
continue
|
|
if cls.pipeline_name in _PIPELINE_REGISTRY:
|
|
logger.warning(
|
|
f"Duplicate pipeline name '{cls.pipeline_name}' found. Overwriting."
|
|
)
|
|
_PIPELINE_REGISTRY[cls.pipeline_name] = cls
|
|
|
|
# Special handling for ComfyUI Pipelines:
|
|
# Auto-register config classes if Pipeline class has them defined
|
|
# since comfyui get model from a single weight file, so we need to register the config classes here
|
|
if hasattr(cls, "pipeline_config_cls") and hasattr(
|
|
cls, "sampling_params_cls"
|
|
):
|
|
_PIPELINE_CONFIG_REGISTRY[cls.pipeline_name] = (
|
|
cls.pipeline_config_cls,
|
|
cls.sampling_params_cls,
|
|
)
|
|
logger.debug(
|
|
f"Auto-registered config classes for pipeline '{cls.pipeline_name}': "
|
|
f"PipelineConfig={cls.pipeline_config_cls.__name__}, "
|
|
f"SamplingParams={cls.sampling_params_cls.__name__}"
|
|
)
|
|
logger.debug(
|
|
f"Registering pipelines complete, {len(_PIPELINE_REGISTRY)} pipelines registered"
|
|
)
|
|
|
|
|
|
def get_pipeline_config_classes(
|
|
pipeline_class_name: str,
|
|
) -> Tuple[Type[PipelineConfig], Type[Any]] | None:
|
|
"""
|
|
Get the configuration classes for a pipeline.
|
|
"""
|
|
# Ensure pipelines are discovered first
|
|
_discover_and_register_pipelines()
|
|
return _PIPELINE_CONFIG_REGISTRY.get(pipeline_class_name)
|
|
|
|
|
|
# --- Part 2: Config Registration ---
|
|
@dataclasses.dataclass
|
|
class ConfigInfo:
|
|
"""Encapsulates all configuration information required to register a
|
|
diffusers model within this framework."""
|
|
|
|
sampling_param_cls: Any
|
|
pipeline_config_cls: Type[PipelineConfig]
|
|
|
|
|
|
# The central registry mapping a model name to its configuration information
|
|
_CONFIG_REGISTRY: Dict[str, ConfigInfo] = {}
|
|
|
|
# Mappings from Hugging Face model paths to our internal model names
|
|
_MODEL_HF_PATH_TO_NAME: Dict[str, str] = {}
|
|
|
|
# Detectors to identify model families from paths or class names
|
|
_MODEL_NAME_DETECTORS: List[Tuple[str, Callable[[str], bool]]] = []
|
|
|
|
|
|
def register_configs(
|
|
sampling_param_cls: Any,
|
|
pipeline_config_cls: Type[PipelineConfig],
|
|
hf_model_paths: Optional[List[str]] = None,
|
|
model_detectors: Optional[List[Callable[[str], bool]]] = None,
|
|
):
|
|
"""
|
|
Registers configuration classes for a new model family.
|
|
"""
|
|
model_id = str(len(_CONFIG_REGISTRY))
|
|
|
|
_CONFIG_REGISTRY[model_id] = ConfigInfo(
|
|
sampling_param_cls=sampling_param_cls,
|
|
pipeline_config_cls=pipeline_config_cls,
|
|
)
|
|
if hf_model_paths:
|
|
for path in hf_model_paths:
|
|
if path in _MODEL_HF_PATH_TO_NAME:
|
|
logger.warning(
|
|
f"Model path '{path}' is already mapped to '{_MODEL_HF_PATH_TO_NAME[path]}' and will be overwritten by '{model_id}'."
|
|
)
|
|
_MODEL_HF_PATH_TO_NAME[path] = model_id
|
|
|
|
if model_detectors:
|
|
for detector in model_detectors:
|
|
_MODEL_NAME_DETECTORS.append((model_id, detector))
|
|
|
|
|
|
def get_model_short_name(model_id: str) -> str:
|
|
if "/" in model_id:
|
|
return model_id.rstrip("/").split("/")[-1]
|
|
else:
|
|
return model_id
|
|
|
|
|
|
@lru_cache(maxsize=1)
|
|
def _get_config_info(
|
|
model_path: str, model_id: Optional[str] = None
|
|
) -> Optional[ConfigInfo]:
|
|
"""
|
|
Gets the ConfigInfo for a given model path using mappings and detectors.
|
|
"""
|
|
all_model_hf_paths = sorted(_MODEL_HF_PATH_TO_NAME.keys(), key=len, reverse=True)
|
|
|
|
# 0. Explicit model_id override: match by short name
|
|
if model_id is not None:
|
|
model_id_lower = model_id.lower()
|
|
for registered_hf_id in all_model_hf_paths:
|
|
if get_model_short_name(registered_hf_id).lower() == model_id_lower:
|
|
logger.debug(
|
|
f"Resolved model via explicit --model-id '{model_id}' → '{registered_hf_id}'."
|
|
)
|
|
return _CONFIG_REGISTRY.get(_MODEL_HF_PATH_TO_NAME[registered_hf_id])
|
|
logger.warning(
|
|
f"--model-id '{model_id}' did not match any registered model; "
|
|
"falling back to automatic detection."
|
|
)
|
|
|
|
# 1. Exact match
|
|
if model_path in _MODEL_HF_PATH_TO_NAME:
|
|
model_id = _MODEL_HF_PATH_TO_NAME[model_path]
|
|
logger.debug(f"Resolved model path '{model_path}' from exact path match.")
|
|
return _CONFIG_REGISTRY.get(model_id)
|
|
|
|
# 2. Partial match: find the best (longest) match against all registered model hf paths.
|
|
model_short_name = get_model_short_name(model_path.lower())
|
|
for registered_model_hf_id in all_model_hf_paths:
|
|
registered_model_name = get_model_short_name(registered_model_hf_id.lower())
|
|
|
|
if registered_model_name in model_short_name:
|
|
logger.debug(
|
|
f"Resolved model name '{registered_model_hf_id}' from partial path match."
|
|
)
|
|
model_id = _MODEL_HF_PATH_TO_NAME[registered_model_hf_id]
|
|
return _CONFIG_REGISTRY.get(model_id)
|
|
|
|
# 3. Use detectors
|
|
if os.path.exists(model_path):
|
|
config = verify_model_config_and_directory(model_path)
|
|
else:
|
|
config = maybe_download_model_index(model_path)
|
|
|
|
pipeline_name = config.get("_class_name", "").lower()
|
|
|
|
matched_model_names = []
|
|
for model_id, detector in _MODEL_NAME_DETECTORS:
|
|
if detector(model_path.lower()) or detector(pipeline_name):
|
|
logger.debug(
|
|
f"Matched model name '{model_id}' using a registered detector."
|
|
)
|
|
matched_model_names += [model_id]
|
|
|
|
if len(matched_model_names) >= 1:
|
|
if len(matched_model_names) > 1:
|
|
logger.warning(
|
|
f"More than one model name is matched, using the first matched"
|
|
)
|
|
model_id = matched_model_names[0]
|
|
return _CONFIG_REGISTRY.get(model_id)
|
|
else:
|
|
raise RuntimeError(f"No model info found for model path: {model_path}")
|
|
|
|
|
|
# --- Part 3: Main Resolver ---
|
|
|
|
|
|
@dataclasses.dataclass
|
|
class ModelInfo:
|
|
"""
|
|
Encapsulates all configuration information required to register a
|
|
diffusers model within this framework.
|
|
"""
|
|
|
|
pipeline_cls: Type[ComposedPipelineBase]
|
|
sampling_param_cls: Any
|
|
pipeline_config_cls: Type[PipelineConfig]
|
|
|
|
|
|
def _get_diffusers_model_info(
|
|
model_path: Optional[str] = None,
|
|
model_id: Optional[str] = None,
|
|
) -> ModelInfo:
|
|
"""
|
|
Get model info for diffusers backend.
|
|
|
|
Returns a ModelInfo with DiffusersPipeline and generic configs.
|
|
When model_path is provided and has a registered native config,
|
|
inherits task_type from it so that validation (e.g. accepts_image_input)
|
|
works correctly even under the diffusers backend.
|
|
"""
|
|
from sglang.multimodal_gen.configs.pipeline_configs.diffusers_generic import (
|
|
DiffusersGenericPipelineConfig,
|
|
)
|
|
from sglang.multimodal_gen.configs.sample.diffusers_generic import (
|
|
DiffusersGenericSamplingParams,
|
|
)
|
|
from sglang.multimodal_gen.runtime.pipelines.diffusers_pipeline import (
|
|
DiffusersPipeline,
|
|
)
|
|
|
|
sampling_param_cls = DiffusersGenericSamplingParams
|
|
pipeline_config_cls = DiffusersGenericPipelineConfig
|
|
|
|
# If there is a registered native config for this model, inherit its task_type
|
|
if model_path is not None:
|
|
config_info = _get_config_info(model_path, model_id=model_id)
|
|
if config_info is not None:
|
|
sampling_param_cls = config_info.sampling_param_cls
|
|
native_task_type = config_info.pipeline_config_cls.task_type
|
|
if native_task_type != DiffusersGenericPipelineConfig.task_type:
|
|
pipeline_config_cls = dataclasses.make_dataclass(
|
|
"DiffusersGenericPipelineConfig",
|
|
[
|
|
(
|
|
"task_type",
|
|
type(native_task_type),
|
|
dataclasses.field(default=native_task_type),
|
|
)
|
|
],
|
|
bases=(DiffusersGenericPipelineConfig,),
|
|
)
|
|
# make_dataclass sets __module__="types"; fix for pickle.
|
|
pipeline_config_cls.__module__ = (
|
|
DiffusersGenericPipelineConfig.__module__
|
|
)
|
|
pipeline_config_cls.__qualname__ = (
|
|
DiffusersGenericPipelineConfig.__qualname__
|
|
)
|
|
parent_module = sys.modules[DiffusersGenericPipelineConfig.__module__]
|
|
setattr(
|
|
parent_module,
|
|
DiffusersGenericPipelineConfig.__name__,
|
|
pipeline_config_cls,
|
|
)
|
|
logger.debug(
|
|
"Inherited task_type=%s from native config for diffusers backend",
|
|
native_task_type.name,
|
|
)
|
|
|
|
return ModelInfo(
|
|
pipeline_cls=DiffusersPipeline,
|
|
sampling_param_cls=sampling_param_cls,
|
|
pipeline_config_cls=pipeline_config_cls,
|
|
)
|
|
|
|
|
|
@lru_cache(maxsize=1)
|
|
def get_model_info(
|
|
model_path: str,
|
|
backend: Optional[Union[str, "Backend"]] = None,
|
|
model_id: Optional[str] = None,
|
|
) -> Optional[ModelInfo]:
|
|
"""
|
|
Resolves all necessary classes (pipeline, sampling, config) for a given model path.
|
|
|
|
This function serves as the main entry point for model resolution. It performs two main tasks:
|
|
1. Dynamically resolves the pipeline class by reading 'model_index.json' and matching
|
|
'_class_name' against an auto-discovered registry of pipeline implementations.
|
|
2. Resolves the associated configuration classes (for sampling and pipeline) using a
|
|
manually registered mapping based on the model path.
|
|
|
|
Args:
|
|
backend: Backend to use ('auto', 'sglang', 'diffusers'). If None, uses 'auto'.
|
|
|
|
"""
|
|
# import Backend enum here to avoid circular imports
|
|
from sglang.multimodal_gen.runtime.server_args import Backend
|
|
|
|
# Normalize backend
|
|
if backend is None:
|
|
backend = Backend.AUTO
|
|
elif isinstance(backend, str):
|
|
backend = Backend.from_string(backend)
|
|
|
|
# Handle explicit diffusers backend
|
|
if backend == Backend.DIFFUSERS:
|
|
logger.info(
|
|
"Using diffusers backend for model '%s' (explicitly requested)", model_path
|
|
)
|
|
return _get_diffusers_model_info(model_path=model_path, model_id=model_id)
|
|
|
|
# For AUTO or SGLANG backend, try native implementation first
|
|
# 1. Discover all available pipeline classes and cache them
|
|
_discover_and_register_pipelines()
|
|
|
|
# Detect quantized models and fallback to diffusers
|
|
is_quantized = any(q in model_path.lower() for q in ["-4bit", "-awq", "-gptq"])
|
|
if is_quantized and backend != Backend.DIFFUSERS:
|
|
logger.info(
|
|
"Detected a quantized model format ('%s'). "
|
|
"The native sglang-diffusion engine currently only supports BF16/FP16. "
|
|
"Falling back to diffusers backend.",
|
|
model_path,
|
|
)
|
|
return _get_diffusers_model_info(model_path=model_path, model_id=model_id)
|
|
|
|
# 2. Get pipeline class - check non-diffusers models first
|
|
pipeline_class_name = get_non_diffusers_pipeline_name(model_path)
|
|
if pipeline_class_name:
|
|
# Known non-diffusers model, skip model_index.json download
|
|
logger.debug(
|
|
f"Using registered pipeline '{pipeline_class_name}' for non-diffusers model '{model_path}'"
|
|
)
|
|
else:
|
|
# Try to get from model_index.json
|
|
try:
|
|
if os.path.exists(model_path):
|
|
config = verify_model_config_and_directory(model_path)
|
|
else:
|
|
config = maybe_download_model_index(model_path)
|
|
except Exception as e:
|
|
logger.error(f"Could not read model config for '{model_path}': {e}")
|
|
if backend == Backend.AUTO:
|
|
logger.info("Falling back to diffusers backend")
|
|
return _get_diffusers_model_info(
|
|
model_path=model_path, model_id=model_id
|
|
)
|
|
return None
|
|
|
|
pipeline_class_name = config.get("_class_name")
|
|
if not pipeline_class_name:
|
|
logger.error(
|
|
f"'_class_name' not found in model_index.json for '{model_path}'"
|
|
)
|
|
if backend == Backend.AUTO:
|
|
logger.info("Falling back to diffusers backend")
|
|
return _get_diffusers_model_info(
|
|
model_path=model_path, model_id=model_id
|
|
)
|
|
return None
|
|
|
|
pipeline_cls = _PIPELINE_REGISTRY.get(pipeline_class_name)
|
|
if not pipeline_cls:
|
|
if backend == Backend.AUTO:
|
|
logger.warning(
|
|
f"Pipeline class '{pipeline_class_name}' specified in '{model_path}' has no native sglang support. "
|
|
f"Falling back to diffusers backend."
|
|
)
|
|
return _get_diffusers_model_info(model_path=model_path, model_id=model_id)
|
|
else:
|
|
logger.error(
|
|
f"Pipeline class '{pipeline_class_name}' specified in '{model_path}' is not a registered EntryClass in the framework. "
|
|
f"Available pipelines: {list(_PIPELINE_REGISTRY.keys())}. "
|
|
f"Consider using --backend diffusers to use vanilla diffusers pipeline."
|
|
)
|
|
return None
|
|
|
|
# 3. Get configuration classes (sampling, pipeline config)
|
|
config_info = _get_config_info(model_path, model_id=model_id)
|
|
if not config_info:
|
|
if backend == Backend.AUTO:
|
|
logger.warning(
|
|
f"Could not resolve native configuration for model '{model_path}'. "
|
|
f"Falling back to diffusers backend."
|
|
)
|
|
return _get_diffusers_model_info(model_path=model_path, model_id=model_id)
|
|
else:
|
|
logger.error(
|
|
f"Could not resolve configuration for model '{model_path}'. "
|
|
"It is not a registered model path or detected by any registered model family detectors. "
|
|
f"Known model paths: {list(_MODEL_HF_PATH_TO_NAME.keys())}. "
|
|
f"Consider using --backend diffusers to use vanilla diffusers pipeline."
|
|
)
|
|
return None
|
|
|
|
# 4. Combine and return the complete model info
|
|
logger.debug("Using native sglang backend for model '%s'", model_path)
|
|
model_info = ModelInfo(
|
|
pipeline_cls=pipeline_cls,
|
|
sampling_param_cls=config_info.sampling_param_cls,
|
|
pipeline_config_cls=config_info.pipeline_config_cls,
|
|
)
|
|
logger.debug(f"Found model info: {model_info}")
|
|
|
|
return model_info
|
|
|
|
|
|
# Registration of model configs
|
|
def _register_configs():
|
|
# LTX-2
|
|
register_configs(
|
|
sampling_param_cls=LTX2SamplingParams,
|
|
pipeline_config_cls=LTX2PipelineConfig,
|
|
model_detectors=[
|
|
lambda path: "ltx" in path.lower() and "video" in path.lower(),
|
|
lambda path: "ltx-2" in path.lower(),
|
|
],
|
|
)
|
|
|
|
# Hunyuan
|
|
register_configs(
|
|
sampling_param_cls=HunyuanSamplingParams,
|
|
pipeline_config_cls=HunyuanConfig,
|
|
hf_model_paths=[
|
|
"hunyuanvideo-community/HunyuanVideo",
|
|
],
|
|
model_detectors=[lambda hf_id: "hunyuanvideo" in hf_id.lower()],
|
|
)
|
|
register_configs(
|
|
sampling_param_cls=FastHunyuanSamplingParam,
|
|
pipeline_config_cls=FastHunyuanConfig,
|
|
hf_model_paths=[
|
|
"FastVideo/FastHunyuan-diffusers",
|
|
],
|
|
)
|
|
# Wan
|
|
register_configs(
|
|
sampling_param_cls=WanT2V_1_3B_SamplingParams,
|
|
pipeline_config_cls=WanT2V480PConfig,
|
|
hf_model_paths=[
|
|
"Wan-AI/Wan2.1-T2V-1.3B-Diffusers",
|
|
],
|
|
model_detectors=[lambda hf_id: "wanpipeline" in hf_id.lower()],
|
|
)
|
|
register_configs(
|
|
sampling_param_cls=WanT2V_1_3B_SamplingParams,
|
|
pipeline_config_cls=TurboWanT2V480PConfig,
|
|
hf_model_paths=[
|
|
"IPostYellow/TurboWan2.1-T2V-1.3B-Diffusers",
|
|
],
|
|
)
|
|
register_configs(
|
|
sampling_param_cls=WanT2V_14B_SamplingParams,
|
|
pipeline_config_cls=WanT2V720PConfig,
|
|
hf_model_paths=[
|
|
"Wan-AI/Wan2.1-T2V-14B-Diffusers",
|
|
],
|
|
)
|
|
register_configs(
|
|
sampling_param_cls=WanT2V_14B_SamplingParams,
|
|
pipeline_config_cls=TurboWanT2V480PConfig,
|
|
hf_model_paths=[
|
|
"IPostYellow/TurboWan2.1-T2V-14B-Diffusers",
|
|
"IPostYellow/TurboWan2.1-T2V-14B-720P-Diffusers",
|
|
],
|
|
)
|
|
register_configs(
|
|
sampling_param_cls=WanI2V_14B_480P_SamplingParam,
|
|
pipeline_config_cls=WanI2V480PConfig,
|
|
hf_model_paths=[
|
|
"Wan-AI/Wan2.1-I2V-14B-480P-Diffusers",
|
|
],
|
|
model_detectors=[lambda hf_id: "wanimagetovideo" in hf_id.lower()],
|
|
)
|
|
register_configs(
|
|
sampling_param_cls=WanI2V_14B_720P_SamplingParam,
|
|
pipeline_config_cls=WanI2V720PConfig,
|
|
hf_model_paths=[
|
|
"Wan-AI/Wan2.1-I2V-14B-720P-Diffusers",
|
|
],
|
|
)
|
|
register_configs(
|
|
sampling_param_cls=Turbo_Wan2_2_I2V_A14B_SamplingParam,
|
|
pipeline_config_cls=TurboWanI2V720Config,
|
|
hf_model_paths=[
|
|
"IPostYellow/TurboWan2.2-I2V-A14B-Diffusers",
|
|
],
|
|
)
|
|
register_configs(
|
|
sampling_param_cls=Wan2_1_Fun_1_3B_InP_SamplingParams,
|
|
pipeline_config_cls=WanI2V480PConfig,
|
|
hf_model_paths=[
|
|
"weizhou03/Wan2.1-Fun-1.3B-InP-Diffusers",
|
|
],
|
|
)
|
|
register_configs(
|
|
sampling_param_cls=Wan2_2_TI2V_5B_SamplingParam,
|
|
pipeline_config_cls=Wan2_2_TI2V_5B_Config,
|
|
hf_model_paths=[
|
|
"Wan-AI/Wan2.2-TI2V-5B-Diffusers",
|
|
],
|
|
)
|
|
register_configs(
|
|
sampling_param_cls=Wan2_2_TI2V_5B_SamplingParam,
|
|
pipeline_config_cls=FastWan2_2_TI2V_5B_Config,
|
|
hf_model_paths=[
|
|
"FastVideo/FastWan2.2-TI2V-5B-FullAttn-Diffusers",
|
|
"FastVideo/FastWan2.2-TI2V-5B-Diffusers",
|
|
],
|
|
)
|
|
register_configs(
|
|
sampling_param_cls=Wan2_2_T2V_A14B_SamplingParam,
|
|
pipeline_config_cls=Wan2_2_T2V_A14B_Config,
|
|
hf_model_paths=["Wan-AI/Wan2.2-T2V-A14B-Diffusers"],
|
|
)
|
|
register_configs(
|
|
sampling_param_cls=Wan2_2_I2V_A14B_SamplingParam,
|
|
pipeline_config_cls=Wan2_2_I2V_A14B_Config,
|
|
hf_model_paths=["Wan-AI/Wan2.2-I2V-A14B-Diffusers"],
|
|
)
|
|
register_configs(
|
|
sampling_param_cls=FastWanT2V480PConfig,
|
|
pipeline_config_cls=FastWan2_1_T2V_480P_Config,
|
|
hf_model_paths=[
|
|
"FastVideo/FastWan2.1-T2V-1.3B-Diffusers",
|
|
],
|
|
)
|
|
# MOVA
|
|
register_configs(
|
|
sampling_param_cls=MOVA_360P_SamplingParams,
|
|
pipeline_config_cls=MOVA360PConfig,
|
|
model_detectors=[
|
|
lambda hf_id: "mova" in hf_id.lower() and "360p" in hf_id.lower()
|
|
],
|
|
)
|
|
register_configs(
|
|
sampling_param_cls=MOVA_720P_SamplingParams,
|
|
pipeline_config_cls=MOVA720PConfig,
|
|
model_detectors=[
|
|
lambda hf_id: "mova" in hf_id.lower() and "720p" in hf_id.lower()
|
|
],
|
|
)
|
|
# FLUX
|
|
register_configs(
|
|
sampling_param_cls=FluxSamplingParams,
|
|
pipeline_config_cls=FluxPipelineConfig,
|
|
hf_model_paths=[
|
|
"black-forest-labs/FLUX.1-dev",
|
|
],
|
|
model_detectors=[lambda hf_id: "flux.1" in hf_id.lower()],
|
|
)
|
|
register_configs(
|
|
sampling_param_cls=Flux2KleinSamplingParams,
|
|
pipeline_config_cls=Flux2KleinPipelineConfig,
|
|
hf_model_paths=[
|
|
"black-forest-labs/FLUX.2-klein-4B",
|
|
"black-forest-labs/FLUX.2-klein-9B",
|
|
],
|
|
model_detectors=[
|
|
lambda hf_id: "flux.2-klein" in hf_id.lower()
|
|
or "flux2-klein" in hf_id.lower()
|
|
],
|
|
)
|
|
register_configs(
|
|
sampling_param_cls=FluxSamplingParams,
|
|
pipeline_config_cls=Flux2PipelineConfig,
|
|
hf_model_paths=[
|
|
"black-forest-labs/FLUX.2-dev",
|
|
],
|
|
model_detectors=[
|
|
lambda hf_id: "flux.2" in hf_id.lower() and "klein" not in hf_id.lower()
|
|
],
|
|
)
|
|
register_configs(
|
|
sampling_param_cls=ZImageTurboSamplingParams,
|
|
pipeline_config_cls=ZImagePipelineConfig,
|
|
hf_model_paths=[
|
|
"Tongyi-MAI/Z-Image-Turbo",
|
|
],
|
|
model_detectors=[lambda hf_id: "z-image-turbo" in hf_id.lower()],
|
|
)
|
|
register_configs(
|
|
sampling_param_cls=ZImageSamplingParams,
|
|
pipeline_config_cls=ZImagePipelineConfig,
|
|
hf_model_paths=[
|
|
"Tongyi-MAI/Z-Image",
|
|
],
|
|
model_detectors=[
|
|
lambda hf_id: "z-image" in hf_id.lower() and "turbo" not in hf_id.lower()
|
|
],
|
|
)
|
|
# Qwen-Image
|
|
register_configs(
|
|
sampling_param_cls=QwenImageSamplingParams,
|
|
pipeline_config_cls=QwenImagePipelineConfig,
|
|
hf_model_paths=["Qwen/Qwen-Image"],
|
|
model_detectors=[
|
|
lambda hf_id: "qwen-image" in hf_id.lower()
|
|
and "edit" not in hf_id.lower()
|
|
and "layered" not in hf_id.lower()
|
|
and "2512" not in hf_id.lower()
|
|
],
|
|
)
|
|
register_configs(
|
|
sampling_param_cls=QwenImage2512SamplingParams,
|
|
pipeline_config_cls=QwenImagePipelineConfig,
|
|
hf_model_paths=["Qwen/Qwen-Image-2512"],
|
|
model_detectors=[lambda hf_id: "qwen-image-2512" in hf_id.lower()],
|
|
)
|
|
register_configs(
|
|
sampling_param_cls=QwenImageSamplingParams,
|
|
pipeline_config_cls=QwenImageEditPipelineConfig,
|
|
hf_model_paths=["Qwen/Qwen-Image-Edit"],
|
|
model_detectors=[
|
|
lambda hf_id: "qwen-image-edit" in hf_id.lower()
|
|
and "2509" not in hf_id.lower()
|
|
and "2511" not in hf_id.lower()
|
|
],
|
|
)
|
|
|
|
register_configs(
|
|
sampling_param_cls=QwenImageEditPlusSamplingParams,
|
|
pipeline_config_cls=QwenImageEditPlusPipelineConfig,
|
|
hf_model_paths=["Qwen/Qwen-Image-Edit-2509"],
|
|
model_detectors=[lambda hf_id: "qwen-image-edit-2509" in hf_id.lower()],
|
|
)
|
|
|
|
register_configs(
|
|
sampling_param_cls=QwenImageEditPlusSamplingParams,
|
|
pipeline_config_cls=QwenImageEditPlus_2511_PipelineConfig,
|
|
hf_model_paths=["Qwen/Qwen-Image-Edit-2511"],
|
|
model_detectors=[lambda hf_id: "qwen-image-edit-2511" in hf_id.lower()],
|
|
)
|
|
|
|
register_configs(
|
|
sampling_param_cls=QwenImageLayeredSamplingParams,
|
|
pipeline_config_cls=QwenImageLayeredPipelineConfig,
|
|
hf_model_paths=["Qwen/Qwen-Image-Layered"],
|
|
model_detectors=[lambda hf_id: "qwen-image-layered" in hf_id.lower()],
|
|
)
|
|
|
|
register_configs(
|
|
sampling_param_cls=GlmImageSamplingParams,
|
|
pipeline_config_cls=GlmImagePipelineConfig,
|
|
model_detectors=[lambda hf_id: "glm-image" in hf_id.lower()],
|
|
)
|
|
register_configs(
|
|
sampling_param_cls=Hunyuan3DSamplingParams,
|
|
pipeline_config_cls=Hunyuan3D2PipelineConfig,
|
|
hf_model_paths=[
|
|
"tencent/Hunyuan3D-2",
|
|
],
|
|
model_detectors=[lambda hf_id: "hunyuan3d" in hf_id.lower()],
|
|
)
|
|
|
|
# Helios
|
|
register_configs(
|
|
sampling_param_cls=HeliosT2VSamplingParams,
|
|
pipeline_config_cls=HeliosT2VConfig,
|
|
hf_model_paths=[
|
|
"BestWishYsh/Helios-Base",
|
|
],
|
|
model_detectors=[
|
|
lambda hf_id: "helios" in hf_id.lower()
|
|
and "mid" not in hf_id.lower()
|
|
and "distill" not in hf_id.lower()
|
|
],
|
|
)
|
|
register_configs(
|
|
sampling_param_cls=HeliosMidSamplingParams,
|
|
pipeline_config_cls=HeliosMidConfig,
|
|
hf_model_paths=[
|
|
"BestWishYsh/Helios-Mid",
|
|
],
|
|
)
|
|
register_configs(
|
|
sampling_param_cls=HeliosDistilledSamplingParams,
|
|
pipeline_config_cls=HeliosDistilledConfig,
|
|
hf_model_paths=[
|
|
"BestWishYsh/Helios-Distilled",
|
|
],
|
|
)
|
|
|
|
# SANA
|
|
register_configs(
|
|
sampling_param_cls=SanaSamplingParams,
|
|
pipeline_config_cls=SanaPipelineConfig,
|
|
hf_model_paths=[
|
|
"Efficient-Large-Model/SANA1.5_1.6B_1024px_diffusers",
|
|
"Efficient-Large-Model/SANA1.5_4.8B_1024px_diffusers",
|
|
"Efficient-Large-Model/Sana_1600M_1024px_diffusers",
|
|
"Efficient-Large-Model/Sana_600M_1024px_diffusers",
|
|
"Efficient-Large-Model/Sana_1600M_512px_diffusers",
|
|
"Efficient-Large-Model/Sana_600M_512px_diffusers",
|
|
],
|
|
model_detectors=[lambda hf_id: "sana" in hf_id.lower()],
|
|
)
|
|
|
|
# FireRed-Image-Edit
|
|
register_configs(
|
|
sampling_param_cls=QwenImageEditPlusSamplingParams,
|
|
pipeline_config_cls=QwenImageEditPlusPipelineConfig,
|
|
hf_model_paths=[
|
|
"FireRedTeam/FireRed-Image-Edit-1.0",
|
|
"FireRedTeam/FireRed-Image-Edit-1.1",
|
|
],
|
|
)
|
|
|
|
|
|
_register_configs()
|
|
|
|
|
|
# Known non-diffusers multimodal model patterns
|
|
# Maps pattern -> pipeline_name for models that don't have model_index.json
|
|
_NON_DIFFUSERS_MULTIMODAL_PATTERNS: Dict[str, str] = {
|
|
"hunyuan3d": "Hunyuan3D2Pipeline",
|
|
}
|
|
|
|
|
|
def is_known_non_diffusers_multimodal_model(model_path: str) -> bool:
|
|
model_path_lower = model_path.lower()
|
|
return any(
|
|
pattern in model_path_lower for pattern in _NON_DIFFUSERS_MULTIMODAL_PATTERNS
|
|
)
|
|
|
|
|
|
def get_non_diffusers_pipeline_name(model_path: str) -> Optional[str]:
|
|
"""Get the pipeline name for a known non-diffusers model."""
|
|
model_path_lower = model_path.lower()
|
|
for pattern, pipeline_name in _NON_DIFFUSERS_MULTIMODAL_PATTERNS.items():
|
|
if pattern in model_path_lower:
|
|
return pipeline_name
|
|
return None
|