chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[diffusion] CI: improve diffusion CI (#13562)

Browse Source 
Co-authored-by: Adarsh Shirawalmath <114558126+adarshxs@users.noreply.github.com>
This commit is contained in:
Mick
2025-11-20 10:54:13 +08:00
committed by GitHub
co-authored by Adarsh Shirawalmath
parent af6bcadcf7
commit 127d59cd2c
19 changed files with 782 additions and 414 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/pr-test.yml
-2
View File
@@ -244,8 +244,6 @@ jobs:
echo "All benchmark tests completed!" echo "All benchmark tests completed!"
# =============================================== multimodal_gen ====================================================
# Adding a single CUDA13 smoke test to verify that the kernel builds and runs # Adding a single CUDA13 smoke test to verify that the kernel builds and runs
# TODO: Add back this test when it can pass on CI # TODO: Add back this test when it can pass on CI
# cuda13-kernel-smoke-test: # cuda13-kernel-smoke-test:
python/sglang/multimodal_gen/envs.py
+55 -55
View File
@@ -15,26 +15,26 @@ from packaging import version
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
if TYPE_CHECKING: if TYPE_CHECKING:
SGL_DIFFUSION_RINGBUFFER_WARNING_INTERVAL: int = 60 SGLANG_DIFFUSION_RINGBUFFER_WARNING_INTERVAL: int = 60
SGL_DIFFUSION_NCCL_SO_PATH: str | None = None SGLANG_DIFFUSION_NCCL_SO_PATH: str | None = None
LD_LIBRARY_PATH: str | None = None LD_LIBRARY_PATH: str | None = None
LOCAL_RANK: int = 0 LOCAL_RANK: int = 0
CUDA_VISIBLE_DEVICES: str | None = None CUDA_VISIBLE_DEVICES: str | None = None
SGL_DIFFUSION_CACHE_ROOT: str = os.path.expanduser("~/.cache/sgl_diffusion") SGLANG_DIFFUSION_CACHE_ROOT: str = os.path.expanduser("~/.cache/sgl_diffusion")
SGL_DIFFUSION_CONFIG_ROOT: str = os.path.expanduser("~/.config/sgl_diffusion") SGLANG_DIFFUSION_CONFIG_ROOT: str = os.path.expanduser("~/.config/sgl_diffusion")
SGL_DIFFUSION_CONFIGURE_LOGGING: int = 1 SGLANG_DIFFUSION_CONFIGURE_LOGGING: int = 1
SGL_DIFFUSION_LOGGING_LEVEL: str = "INFO" SGLANG_DIFFUSION_LOGGING_LEVEL: str = "INFO"
SGL_DIFFUSION_LOGGING_PREFIX: str = "" SGLANG_DIFFUSION_LOGGING_PREFIX: str = ""
SGL_DIFFUSION_LOGGING_CONFIG_PATH: str | None = None SGLANG_DIFFUSION_LOGGING_CONFIG_PATH: str | None = None
SGL_DIFFUSION_TRACE_FUNCTION: int = 0 SGLANG_DIFFUSION_TRACE_FUNCTION: int = 0
SGL_DIFFUSION_WORKER_MULTIPROC_METHOD: str = "fork" SGLANG_DIFFUSION_WORKER_MULTIPROC_METHOD: str = "fork"
SGL_DIFFUSION_TARGET_DEVICE: str = "cuda" SGLANG_DIFFUSION_TARGET_DEVICE: str = "cuda"
MAX_JOBS: str | None = None MAX_JOBS: str | None = None
NVCC_THREADS: str | None = None NVCC_THREADS: str | None = None
CMAKE_BUILD_TYPE: str | None = None CMAKE_BUILD_TYPE: str | None = None
VERBOSE: bool = False VERBOSE: bool = False
SGL_DIFFUSION_SERVER_DEV_MODE: bool = False SGLANG_DIFFUSION_SERVER_DEV_MODE: bool = False
SGL_DIFFUSION_STAGE_LOGGING: bool = False SGLANG_DIFFUSION_STAGE_LOGGING: bool = False
def _is_hip(): def _is_hip():
@@ -165,8 +165,8 @@ environment_variables: dict[str, Callable[[], Any]] = {
# ================== Installation Time Env Vars ================== # ================== Installation Time Env Vars ==================
# Target device of sgl-diffusion, supporting [cuda (by default), # Target device of sgl-diffusion, supporting [cuda (by default),
# rocm, neuron, cpu, openvino] # rocm, neuron, cpu, openvino]
"SGL_DIFFUSION_TARGET_DEVICE": lambda: os.getenv( "SGLANG_DIFFUSION_TARGET_DEVICE": lambda: os.getenv(
"SGL_DIFFUSION_TARGET_DEVICE", "cuda" "SGLANG_DIFFUSION_TARGET_DEVICE", "cuda"
), ),
# Maximum number of compilation jobs to run in parallel. # Maximum number of compilation jobs to run in parallel.
# By default this is the number of CPUs # By default this is the number of CPUs
@@ -176,10 +176,10 @@ environment_variables: dict[str, Callable[[], Any]] = {
# If set, `MAX_JOBS` will be reduced to avoid oversubscribing the CPU. # If set, `MAX_JOBS` will be reduced to avoid oversubscribing the CPU.
"NVCC_THREADS": lambda: os.getenv("NVCC_THREADS", None), "NVCC_THREADS": lambda: os.getenv("NVCC_THREADS", None),
# If set, sgl_diffusion will use precompiled binaries (*.so) # If set, sgl_diffusion will use precompiled binaries (*.so)
"SGL_DIFFUSION_USE_PRECOMPILED": lambda: bool( "SGLANG_DIFFUSION_USE_PRECOMPILED": lambda: bool(
os.environ.get("SGL_DIFFUSION_USE_PRECOMPILED") os.environ.get("SGLANG_DIFFUSION_USE_PRECOMPILED")
) )
or bool(os.environ.get("SGL_DIFFUSION_PRECOMPILED_WHEEL_LOCATION")), or bool(os.environ.get("SGLANG_DIFFUSION_PRECOMPILED_WHEEL_LOCATION")),
# CMake build type # CMake build type
# If not set, defaults to "Debug" or "RelWithDebInfo" # If not set, defaults to "Debug" or "RelWithDebInfo"
# Available options: "Debug", "Release", "RelWithDebInfo" # Available options: "Debug", "Release", "RelWithDebInfo"
@@ -191,36 +191,36 @@ environment_variables: dict[str, Callable[[], Any]] = {
# Note that this not only affects how sgl_diffusion finds its configuration files # Note that this not only affects how sgl_diffusion finds its configuration files
# during runtime, but also affects how sgl_diffusion installs its configuration # during runtime, but also affects how sgl_diffusion installs its configuration
# files during **installation**. # files during **installation**.
"SGL_DIFFUSION_CONFIG_ROOT": lambda: os.path.expanduser( "SGLANG_DIFFUSION_CONFIG_ROOT": lambda: os.path.expanduser(
os.getenv( os.getenv(
"SGL_DIFFUSION_CONFIG_ROOT", "SGLANG_DIFFUSION_CONFIG_ROOT",
os.path.join(get_default_config_root(), "sgl_diffusion"), os.path.join(get_default_config_root(), "sgl_diffusion"),
) )
), ),
# ================== Runtime Env Vars ================== # ================== Runtime Env Vars ==================
# Root directory for FASTVIDEO cache files # Root directory for FASTVIDEO cache files
# Defaults to `~/.cache/sgl_diffusion` unless `XDG_CACHE_HOME` is set # Defaults to `~/.cache/sgl_diffusion` unless `XDG_CACHE_HOME` is set
"SGL_DIFFUSION_CACHE_ROOT": lambda: os.path.expanduser( "SGLANG_DIFFUSION_CACHE_ROOT": lambda: os.path.expanduser(
os.getenv( os.getenv(
"SGL_DIFFUSION_CACHE_ROOT", "SGLANG_DIFFUSION_CACHE_ROOT",
os.path.join(get_default_cache_root(), "sgl_diffusion"), os.path.join(get_default_cache_root(), "sgl_diffusion"),
) )
), ),
# Interval in seconds to log a warning message when the ring buffer is full # Interval in seconds to log a warning message when the ring buffer is full
"SGL_DIFFUSION_RINGBUFFER_WARNING_INTERVAL": lambda: int( "SGLANG_DIFFUSION_RINGBUFFER_WARNING_INTERVAL": lambda: int(
os.environ.get("SGL_DIFFUSION_RINGBUFFER_WARNING_INTERVAL", "60") os.environ.get("SGLANG_DIFFUSION_RINGBUFFER_WARNING_INTERVAL", "60")
), ),
# Path to the NCCL library file. It is needed because nccl>=2.19 brought # Path to the NCCL library file. It is needed because nccl>=2.19 brought
# by PyTorch contains a bug: https://github.com/NVIDIA/nccl/issues/1234 # by PyTorch contains a bug: https://github.com/NVIDIA/nccl/issues/1234
"SGL_DIFFUSION_NCCL_SO_PATH": lambda: os.environ.get( "SGLANG_DIFFUSION_NCCL_SO_PATH": lambda: os.environ.get(
"SGL_DIFFUSION_NCCL_SO_PATH", None "SGLANG_DIFFUSION_NCCL_SO_PATH", None
), ),
# when `SGL_DIFFUSION_NCCL_SO_PATH` is not set, sgl_diffusion will try to find the nccl # when `SGLANG_DIFFUSION_NCCL_SO_PATH` is not set, sgl_diffusion will try to find the nccl
# library file in the locations specified by `LD_LIBRARY_PATH` # library file in the locations specified by `LD_LIBRARY_PATH`
"LD_LIBRARY_PATH": lambda: os.environ.get("LD_LIBRARY_PATH", None), "LD_LIBRARY_PATH": lambda: os.environ.get("LD_LIBRARY_PATH", None),
# Internal flag to enable Dynamo fullgraph capture # Internal flag to enable Dynamo fullgraph capture
"SGL_DIFFUSION_TEST_DYNAMO_FULLGRAPH_CAPTURE": lambda: bool( "SGLANG_DIFFUSION_TEST_DYNAMO_FULLGRAPH_CAPTURE": lambda: bool(
os.environ.get("SGL_DIFFUSION_TEST_DYNAMO_FULLGRAPH_CAPTURE", "1") != "0" os.environ.get("SGLANG_DIFFUSION_TEST_DYNAMO_FULLGRAPH_CAPTURE", "1") != "0"
), ),
# local rank of the process in the distributed setting, used to determine # local rank of the process in the distributed setting, used to determine
# the GPU device id # the GPU device id
@@ -228,62 +228,62 @@ environment_variables: dict[str, Callable[[], Any]] = {
# used to control the visible devices in the distributed setting # used to control the visible devices in the distributed setting
"CUDA_VISIBLE_DEVICES": lambda: os.environ.get("CUDA_VISIBLE_DEVICES", None), "CUDA_VISIBLE_DEVICES": lambda: os.environ.get("CUDA_VISIBLE_DEVICES", None),
# timeout for each iteration in the engine # timeout for each iteration in the engine
"SGL_DIFFUSION_ENGINE_ITERATION_TIMEOUT_S": lambda: int( "SGLANG_DIFFUSION_ENGINE_ITERATION_TIMEOUT_S": lambda: int(
os.environ.get("SGL_DIFFUSION_ENGINE_ITERATION_TIMEOUT_S", "60") os.environ.get("SGLANG_DIFFUSION_ENGINE_ITERATION_TIMEOUT_S", "60")
), ),
# Logging configuration # Logging configuration
# If set to 0, sgl_diffusion will not configure logging # If set to 0, sgl_diffusion will not configure logging
# If set to 1, sgl_diffusion will configure logging using the default configuration # If set to 1, sgl_diffusion will configure logging using the default configuration
# or the configuration file specified by SGL_DIFFUSION_LOGGING_CONFIG_PATH # or the configuration file specified by SGLANG_DIFFUSION_LOGGING_CONFIG_PATH
"SGL_DIFFUSION_CONFIGURE_LOGGING": lambda: int( "SGLANG_DIFFUSION_CONFIGURE_LOGGING": lambda: int(
os.getenv("SGL_DIFFUSION_CONFIGURE_LOGGING", "1") os.getenv("SGLANG_DIFFUSION_CONFIGURE_LOGGING", "1")
), ),
"SGL_DIFFUSION_LOGGING_CONFIG_PATH": lambda: os.getenv( "SGLANG_DIFFUSION_LOGGING_CONFIG_PATH": lambda: os.getenv(
"SGL_DIFFUSION_LOGGING_CONFIG_PATH" "SGLANG_DIFFUSION_LOGGING_CONFIG_PATH"
), ),
# this is used for configuring the default logging level # this is used for configuring the default logging level
"SGL_DIFFUSION_LOGGING_LEVEL": lambda: os.getenv( "SGLANG_DIFFUSION_LOGGING_LEVEL": lambda: os.getenv(
"SGL_DIFFUSION_LOGGING_LEVEL", "INFO" "SGLANG_DIFFUSION_LOGGING_LEVEL", "INFO"
), ),
# if set, SGL_DIFFUSION_LOGGING_PREFIX will be prepended to all log messages # if set, SGLANG_DIFFUSION_LOGGING_PREFIX will be prepended to all log messages
"SGL_DIFFUSION_LOGGING_PREFIX": lambda: os.getenv( "SGLANG_DIFFUSION_LOGGING_PREFIX": lambda: os.getenv(
"SGL_DIFFUSION_LOGGING_PREFIX", "" "SGLANG_DIFFUSION_LOGGING_PREFIX", ""
), ),
# Trace function calls # Trace function calls
# If set to 1, sgl_diffusion will trace function calls # If set to 1, sgl_diffusion will trace function calls
# Useful for debugging # Useful for debugging
"SGL_DIFFUSION_TRACE_FUNCTION": lambda: int( "SGLANG_DIFFUSION_TRACE_FUNCTION": lambda: int(
os.getenv("SGL_DIFFUSION_TRACE_FUNCTION", "0") os.getenv("SGLANG_DIFFUSION_TRACE_FUNCTION", "0")
), ),
# Path to the attention configuration file. Only used for sliding tile # Path to the attention configuration file. Only used for sliding tile
# attention for now. # attention for now.
"SGL_DIFFUSION_ATTENTION_CONFIG": lambda: ( "SGLANG_DIFFUSION_ATTENTION_CONFIG": lambda: (
None None
if os.getenv("SGL_DIFFUSION_ATTENTION_CONFIG", None) is None if os.getenv("SGLANG_DIFFUSION_ATTENTION_CONFIG", None) is None
else os.path.expanduser(os.getenv("SGL_DIFFUSION_ATTENTION_CONFIG", ".")) else os.path.expanduser(os.getenv("SGLANG_DIFFUSION_ATTENTION_CONFIG", "."))
), ),
# Use dedicated multiprocess context for workers. # Use dedicated multiprocess context for workers.
# Both spawn and fork work # Both spawn and fork work
"SGL_DIFFUSION_WORKER_MULTIPROC_METHOD": lambda: os.getenv( "SGLANG_DIFFUSION_WORKER_MULTIPROC_METHOD": lambda: os.getenv(
"SGL_DIFFUSION_WORKER_MULTIPROC_METHOD", "fork" "SGLANG_DIFFUSION_WORKER_MULTIPROC_METHOD", "fork"
), ),
# Enables torch profiler if set. Path to the directory where torch profiler # Enables torch profiler if set. Path to the directory where torch profiler
# traces are saved. Note that it must be an absolute path. # traces are saved. Note that it must be an absolute path.
"SGL_DIFFUSION_TORCH_PROFILER_DIR": lambda: ( "SGLANG_DIFFUSION_TORCH_PROFILER_DIR": lambda: (
None None
if os.getenv("SGL_DIFFUSION_TORCH_PROFILER_DIR", None) is None if os.getenv("SGLANG_DIFFUSION_TORCH_PROFILER_DIR", None) is None
else os.path.expanduser(os.getenv("SGL_DIFFUSION_TORCH_PROFILER_DIR", ".")) else os.path.expanduser(os.getenv("SGLANG_DIFFUSION_TORCH_PROFILER_DIR", "."))
), ),
# If set, sgl_diffusion will run in development mode, which will enable # If set, sgl_diffusion will run in development mode, which will enable
# some additional endpoints for developing and debugging, # some additional endpoints for developing and debugging,
# e.g. `/reset_prefix_cache` # e.g. `/reset_prefix_cache`
"SGL_DIFFUSION_SERVER_DEV_MODE": lambda: bool( "SGLANG_DIFFUSION_SERVER_DEV_MODE": lambda: bool(
int(os.getenv("SGL_DIFFUSION_SERVER_DEV_MODE", "0")) int(os.getenv("SGLANG_DIFFUSION_SERVER_DEV_MODE", "0"))
), ),
# If set, sgl_diffusion will enable stage logging, which will print the time # If set, sgl_diffusion will enable stage logging, which will print the time
# taken for each stage # taken for each stage
"SGL_DIFFUSION_STAGE_LOGGING": lambda: bool( "SGLANG_DIFFUSION_STAGE_LOGGING": lambda: bool(
int(os.getenv("SGL_DIFFUSION_STAGE_LOGGING", "0")) int(os.getenv("SGLANG_DIFFUSION_STAGE_LOGGING", "0"))
), ),
} }
python/sglang/multimodal_gen/runtime/distributed/device_communicators/pynccl_wrapper.py
+3 -3
View File
@@ -21,10 +21,10 @@
# recompilation of the code every time we want to switch between different # recompilation of the code every time we want to switch between different
# versions. This current implementation, with a **pure** Python wrapper, is # versions. This current implementation, with a **pure** Python wrapper, is
# more flexible. We can easily switch between different versions of NCCL by # more flexible. We can easily switch between different versions of NCCL by
# changing the environment variable `SGL_DIFFUSION_NCCL_SO_PATH`, or the `so_file` # changing the environment variable `SGLANG_DIFFUSION_NCCL_SO_PATH`, or the `so_file`
# variable in the code. # variable in the code.
# TODO(will): support SGL_DIFFUSION_NCCL_SO_PATH # TODO(will): support SGLANG_DIFFUSION_NCCL_SO_PATH
import ctypes import ctypes
import platform import platform
@@ -281,7 +281,7 @@ class NCCLLibrary:
"Otherwise, the nccl library might not exist, be corrupted " "Otherwise, the nccl library might not exist, be corrupted "
"or it does not support the current platform %s." "or it does not support the current platform %s."
"If you already have the library, please set the " "If you already have the library, please set the "
"environment variable SGL_DIFFUSION_NCCL_SO_PATH" "environment variable SGLANG_DIFFUSION_NCCL_SO_PATH"
" to point to the correct nccl library path.", " to point to the correct nccl library path.",
so_file, so_file,
platform.platform(), platform.platform(),
python/sglang/multimodal_gen/runtime/layers/attention/backends/sliding_tile_attn.py
+2 -2
View File
@@ -119,9 +119,9 @@ class SlidingTileAttentionImpl(AttentionImpl):
raise ValueError("st attn not supported") raise ValueError("st attn not supported")
# TODO(will-refactor): for now this is the mask strategy, but maybe we should # TODO(will-refactor): for now this is the mask strategy, but maybe we should
# have a more general config for STA? # have a more general config for STA?
config_file = envs.SGL_DIFFUSION_ATTENTION_CONFIG config_file = envs.SGLANG_DIFFUSION_ATTENTION_CONFIG
if config_file is None: if config_file is None:
raise ValueError("SGL_DIFFUSION_ATTENTION_CONFIG is not set") raise ValueError("SGLANG_DIFFUSION_ATTENTION_CONFIG is not set")
# TODO(kevin): get mask strategy for different STA modes # TODO(kevin): get mask strategy for different STA modes
with open(config_file) as f: with open(config_file) as f:
python/sglang/multimodal_gen/runtime/layers/attention/selector.py
+1 -1
View File
@@ -110,7 +110,7 @@ def _cached_get_attn_backend(
# Check whether a particular choice of backend was # Check whether a particular choice of backend was
# previously forced. # previously forced.
# #
# THIS SELECTION OVERRIDES THE SGL_DIFFUSION_ATTENTION_BACKEND # THIS SELECTION OVERRIDES THE SGLANG_DIFFUSION_ATTENTION_BACKEND
# ENVIRONMENT VARIABLE. # ENVIRONMENT VARIABLE.
from sglang.multimodal_gen.runtime.platforms import current_platform from sglang.multimodal_gen.runtime.platforms import current_platform
python/sglang/multimodal_gen/runtime/managers/forward_context.py
+2 -2
View File
@@ -19,11 +19,11 @@ if TYPE_CHECKING:
logger = init_logger(__name__) logger = init_logger(__name__)
# TODO(will): check if this is needed # TODO(will): check if this is needed
# track_batchsize: bool = envs.SGL_DIFFUSION_LOG_BATCHSIZE_INTERVAL >= 0 # track_batchsize: bool = envs.SGLANG_DIFFUSION_LOG_BATCHSIZE_INTERVAL >= 0
track_batchsize: bool = False track_batchsize: bool = False
last_logging_time: float = 0 last_logging_time: float = 0
forward_start_time: float = 0 forward_start_time: float = 0
# batchsize_logging_interval: float = envs.SGL_DIFFUSION_LOG_BATCHSIZE_INTERVAL # batchsize_logging_interval: float = envs.SGLANG_DIFFUSION_LOG_BATCHSIZE_INTERVAL
batchsize_logging_interval: float = 1000 batchsize_logging_interval: float = 1000
batchsize_forward_time: defaultdict = defaultdict(list) batchsize_forward_time: defaultdict = defaultdict(list)
python/sglang/multimodal_gen/runtime/models/registry.py
+2 -2
View File
@@ -102,7 +102,7 @@ def _discover_and_register_models() -> dict[str, tuple[str, str, str]]:
return discovered_models return discovered_models
_SGL_DIFFUSION_MODELS = _discover_and_register_models() _SGLANG_DIFFUSION_MODELS = _discover_and_register_models()
_SUBPROCESS_COMMAND = [ _SUBPROCESS_COMMAND = [
sys.executable, sys.executable,
@@ -361,6 +361,6 @@ ModelRegistry = _ModelRegistry(
component_name, component_name,
mod_relname, mod_relname,
cls_name, cls_name,
) in _SGL_DIFFUSION_MODELS.items() ) in _SGLANG_DIFFUSION_MODELS.items()
} }
) )
python/sglang/multimodal_gen/runtime/pipelines/stages/base.py
+1 -1
View File
@@ -187,7 +187,7 @@ class PipelineStage(ABC):
# Execute the actual stage logic # Execute the actual stage logic
logging_info = getattr(batch, "logging_info", None) logging_info = getattr(batch, "logging_info", None)
if envs.SGL_DIFFUSION_STAGE_LOGGING: if envs.SGLANG_DIFFUSION_STAGE_LOGGING:
logger.info("[%s] Starting execution", stage_name) logger.info("[%s] Starting execution", stage_name)
start_time = time.perf_counter() start_time = time.perf_counter()
python/sglang/multimodal_gen/runtime/pipelines/stages/denoising.py
+2 -2
View File
@@ -1284,9 +1284,9 @@ class DenoisingStage(PipelineStage):
elif STA_mode == STA_Mode.STA_INFERENCE: elif STA_mode == STA_Mode.STA_INFERENCE:
import sglang.multimodal_gen.envs as envs import sglang.multimodal_gen.envs as envs
config_file = envs.SGL_DIFFUSION_ATTENTION_CONFIG config_file = envs.SGLANG_DIFFUSION_ATTENTION_CONFIG
if config_file is None: if config_file is None:
raise ValueError("SGL_DIFFUSION_ATTENTION_CONFIG is not set") raise ValueError("SGLANG_DIFFUSION_ATTENTION_CONFIG is not set")
STA_param = configure_sta( STA_param = configure_sta(
mode=STA_Mode.STA_INFERENCE, mode=STA_Mode.STA_INFERENCE,
layer_num=layer_num, layer_num=layer_num,
python/sglang/multimodal_gen/runtime/platforms/rocm.py
+2 -2
View File
@@ -69,8 +69,8 @@ class RocmPlatform(Platform):
dtype: torch.dtype, dtype: torch.dtype,
) -> str: ) -> str:
logger.info( logger.info(
"Trying SGL_DIFFUSION_ATTENTION_BACKEND=%s", "Trying SGLANG_DIFFUSION_ATTENTION_BACKEND=%s",
envs.SGL_DIFFUSION_ATTENTION_BACKEND, envs.SGLANG_DIFFUSION_ATTENTION_BACKEND,
) )
if selected_backend == AttentionBackendEnum.TORCH_SDPA: if selected_backend == AttentionBackendEnum.TORCH_SDPA:
python/sglang/multimodal_gen/runtime/utils/logging_utils.py
+7 -7
View File
@@ -17,10 +17,10 @@ from typing import Any, cast
import sglang.multimodal_gen.envs as envs import sglang.multimodal_gen.envs as envs
SGL_DIFFUSION_CONFIGURE_LOGGING = envs.SGL_DIFFUSION_CONFIGURE_LOGGING SGLANG_DIFFUSION_CONFIGURE_LOGGING = envs.SGLANG_DIFFUSION_CONFIGURE_LOGGING
SGL_DIFFUSION_LOGGING_CONFIG_PATH = envs.SGL_DIFFUSION_LOGGING_CONFIG_PATH SGLANG_DIFFUSION_LOGGING_CONFIG_PATH = envs.SGLANG_DIFFUSION_LOGGING_CONFIG_PATH
SGL_DIFFUSION_LOGGING_LEVEL = envs.SGL_DIFFUSION_LOGGING_LEVEL SGLANG_DIFFUSION_LOGGING_LEVEL = envs.SGLANG_DIFFUSION_LOGGING_LEVEL
SGL_DIFFUSION_LOGGING_PREFIX = envs.SGL_DIFFUSION_LOGGING_PREFIX SGLANG_DIFFUSION_LOGGING_PREFIX = envs.SGLANG_DIFFUSION_LOGGING_PREFIX
RED = "\033[91m" RED = "\033[91m"
GREEN = "\033[92m" GREEN = "\033[92m"
@@ -28,7 +28,7 @@ YELLOW = "\033[93m"
RESET = "\033[0;0m" RESET = "\033[0;0m"
_FORMAT = ( _FORMAT = (
f"{SGL_DIFFUSION_LOGGING_PREFIX}%(levelname)s %(asctime)s " f"{SGLANG_DIFFUSION_LOGGING_PREFIX}%(levelname)s %(asctime)s "
"[%(filename)s: %(lineno)d] %(message)s" "[%(filename)s: %(lineno)d] %(message)s"
) )
@@ -47,7 +47,7 @@ DEFAULT_LOGGING_CONFIG = {
"sgl_diffusion": { "sgl_diffusion": {
"class": "logging.StreamHandler", "class": "logging.StreamHandler",
"formatter": "sgl_diffusion", "formatter": "sgl_diffusion",
"level": SGL_DIFFUSION_LOGGING_LEVEL, "level": SGLANG_DIFFUSION_LOGGING_LEVEL,
"stream": "ext://sys.stdout", "stream": "ext://sys.stdout",
}, },
}, },
@@ -340,7 +340,7 @@ def enable_trace_function_call(log_file_path: str, root_dir: str | None = None):
will have the trace enabled. Other threads will not be affected. will have the trace enabled. Other threads will not be affected.
""" """
logger.warning( logger.warning(
"SGL_DIFFUSION_TRACE_FUNCTION is enabled. It will record every" "SGLANG_DIFFUSION_TRACE_FUNCTION is enabled. It will record every"
" function executed by Python. This will slow down the code. It " " function executed by Python. This will slow down the code. It "
"is suggested to be used for debugging hang or crashes only." "is suggested to be used for debugging hang or crashes only."
) )
python/sglang/multimodal_gen/runtime/utils/performance_logger.py
+7 -1
View File
@@ -6,10 +6,13 @@ import os
import subprocess import subprocess
import time import time
from datetime import datetime from datetime import datetime
from pathlib import Path
from typing import Any from typing import Any
from dateutil.tz import UTC from dateutil.tz import UTC
import sglang
def get_diffusion_perf_log_dir() -> str: def get_diffusion_perf_log_dir() -> str:
""" """
@@ -26,7 +29,10 @@ def get_diffusion_perf_log_dir() -> str:
return os.path.abspath(log_dir) return os.path.abspath(log_dir)
if log_dir is None: if log_dir is None:
# Not set, use default # Not set, use default
return os.path.join(os.path.expanduser("~/.cache/sglang"), "logs") sglang_path = Path(sglang.__file__).resolve()
# .gitignore
target_path = (sglang_path.parent / "../../.cache/logs").resolve()
return str(target_path)
# Is set, but is an empty string # Is set, but is an empty string
return "" return ""
python/sglang/multimodal_gen/test/server/perf_baselines.json
+343 -147
View File
@@ -1,155 +1,351 @@
{ {
"metadata": { "metadata": {
"model": "Diffusion Server", "model": "Diffusion Server",
"hardware": "CI H100 80GB pool", "hardware": "CI H100 80GB pool",
"description": "Reference numbers captured from the CI diffusion server baseline run" "description": "Reference numbers captured from the CI diffusion server baseline run"
},
"tolerances": {
"e2e": 0.25,
"stage": 0.3,
"denoise_step": 0.2,
"denoise_agg": 0.1
},
"sampling": {
"step_fractions": [
0.0,
0.2,
0.4,
0.6,
0.8,
1.0
],
"warmup_requests": {
"text": 1,
"image_edit": 0
}
},
"scenarios": {
"text_to_image": {
"notes": "Single-image generation using the default prompt",
"expected_e2e_ms": 74500.0,
"expected_avg_denoise_ms": 422.42,
"expected_median_denoise_ms": 410.62,
"stages_ms": {
"InputValidationStage": 0.1,
"TextEncodingStage": 834.2,
"ConditioningStage": 0.1,
"TimestepPreparationStage": 10.6,
"LatentPreparationStage": 9.0,
"DenoisingStage": 21202.6,
"DecodingStage": 476.12
},
"denoise_step_ms": {
"0": 1077.77, "1": 345.13, "2": 413.8, "3": 405.49, "4": 408.14, "5": 409.06,
"6": 408.85, "7": 410.53, "8": 407.51, "9": 409.44, "10": 408.65, "11": 410.14,
"12": 411.74, "13": 409.59, "14": 409.17, "15": 410.78, "16": 410.66, "17": 410.58,
"18": 411.27, "19": 410.51, "20": 409.03, "21": 410.16, "22": 409.42, "23": 411.03,
"24": 410.18, "25": 409.72, "26": 410.26, "27": 410.21, "28": 410.71, "29": 410.76,
"30": 411.06, "31": 410.1, "32": 410.55, "33": 410.77, "34": 410.74, "35": 411.75,
"36": 410.78, "37": 411.56, "38": 410.85, "39": 411.08, "40": 411.12, "41": 411.1,
"42": 411.09, "43": 410.87, "44": 411.37, "45": 411.68, "46": 411.0, "47": 410.09,
"48": 412.72, "49": 410.42
}
}, },
"image_edit": { "tolerances": {
"notes": "single uploaded reference image, Qwen/Qwen-Image-Edit", "e2e": 0.05,
"expected_e2e_ms": 138500.0, "stage": 0.05,
"expected_avg_denoise_ms": 720.0, "denoise_step": 0.15,
"expected_median_denoise_ms": 718.0, "denoise_agg": 0.08
"stages_ms": {
"InputValidationStage": 23,
"ImageEncodingStage": 1350.0,
"ImageVAEEncodingStage": 340.0,
"ConditioningStage": 0.13,
"TimestepPreparationStage": 13.78,
"LatentPreparationStage": 10.0,
"DenoisingStage": 36000.0,
"DecodingStage": 850.0
},
"denoise_step_ms": {
"0": 720.0, "1": 720.0, "2": 720.0, "3": 720.0, "4": 720.0, "5": 720.0,
"6": 720.0, "7": 720.0, "8": 720.0, "9": 720.0, "10": 720.0, "11": 720.0,
"12": 720.0, "13": 720.0, "14": 720.0, "15": 720.0, "16": 720.0, "17": 720.0,
"18": 720.0, "19": 720.0, "20": 720.0, "21": 720.0, "22": 720.0, "23": 720.0,
"24": 720.0, "25": 720.0, "26": 720.0, "27": 720.0, "28": 720.0, "29": 720.0,
"30": 720.0, "31": 720.0, "32": 720.0, "33": 720.0, "34": 720.0, "35": 720.0,
"36": 720.0, "37": 720.0, "38": 720.0, "39": 720.0, "40": 720.0, "41": 720.0,
"42": 720.0, "43": 720.0, "44": 720.0, "45": 720.0, "46": 720.0, "47": 720.0,
"48": 720.0, "49": 720.0
}
}, },
"text_to_video": { "improvement_reporting": {
"notes": "Single-video generation using the default prompt", "threshold": 0.2
"expected_e2e_ms": 95616.59,
"expected_avg_denoise_ms": 1798.77,
"expected_median_denoise_ms": 1786.78,
"stages_ms": {
"InputValidationStage": 1.03,
"TextEncodingStage": 3450.0,
"ConditioningStage": 1.0,
"TimestepPreparationStage": 6.0,
"LatentPreparationStage": 15.0,
"DenoisingStage": 90100.0,
"DecodingStage": 3650.0
},
"denoise_step_ms": {
"0": 3500.0, "10": 1800.0, "20": 1800.0, "29": 1800.0, "39": 1800.0, "49": 1800.0
},
"frames_per_second": 0.51,
"total_frames": 49,
"avg_frame_time_ms": 1951.36
}, },
"image_to_video": { "sampling": {
"notes": "Wan-AI/Wan2.2-I2V-A14B", "step_fractions": [
"expected_e2e_ms": 282500.0, 0.0,
"expected_avg_denoise_ms": 7000.0, 0.2,
"expected_median_denoise_ms": 7000.19, 0.4,
"stages_ms": { 0.6,
"InputValidationStage": 20.0, 0.8,
"TextEncodingStage": 2100.0, 1.0
"ConditioningStage": 2.0, ],
"TimestepPreparationStage": 2.0, "warmup_requests": {
"LatentPreparationStage": 10.0, "text": 1,
"ImageVAEEncodingStage": 1800.0, "image_edit": 0
"DenoisingStage": 278000.0,
"DecodingStage": 2700.0
},
"denoise_step_ms": {
"0": 24000.0,
"8": 7000.0,
"16": 7000.0,
"23": 7000.0,
"31": 7000.0,
"39": 7000.0
} }
}, },
"text_image_to_video": { "scenarios": {
"notes": "Text-and-Image-to-Video generation baseline for Wan2.2-TI2V-5B", "qwen_image_t2i": {
"expected_e2e_ms": 178300.0, "notes": "Single-image generation using the default prompt",
"expected_avg_denoise_ms": 3250.0, "expected_e2e_ms": 74500.0,
"expected_median_denoise_ms": 3260.0, "expected_avg_denoise_ms": 422.42,
"stages_ms": { "expected_median_denoise_ms": 410.62,
"InputValidationStage": 80.0, "stages_ms": {
"TextEncodingStage": 3000.0, "InputValidationStage": 0.1,
"ConditioningStage": 1.0, "TextEncodingStage": 834.2,
"TimestepPreparationStage": 6.0, "ConditioningStage": 0.1,
"LatentPreparationStage": 30.0, "TimestepPreparationStage": 10.6,
"DenoisingStage": 162900.0, "LatentPreparationStage": 11.8,
"DecodingStage": 13500.0 "DenoisingStage": 21202.6,
}, "DecodingStage": 476.12
"denoise_step_ms": { },
"0": 3700.0, "denoise_step_ms": {
"10": 3300.0, "0": 1077.77,
"20": 3300.0, "1": 345.13,
"29": 3300.0, "2": 413.8,
"39": 3300.0, "3": 405.49,
"49": 3300.0 "4": 408.14,
}, "5": 409.06,
"frames_per_second": null, "6": 408.85,
"total_frames": null, "7": 410.53,
"avg_frame_time_ms": null "8": 407.51,
"9": 409.44,
"10": 408.65,
"11": 410.14,
"12": 411.74,
"13": 409.59,
"14": 409.17,
"15": 410.78,
"16": 410.66,
"17": 410.58,
"18": 411.27,
"19": 410.51,
"20": 409.03,
"21": 410.16,
"22": 409.42,
"23": 411.03,
"24": 410.18,
"25": 409.72,
"26": 410.26,
"27": 410.21,
"28": 410.71,
"29": 410.76,
"30": 411.06,
"31": 410.1,
"32": 410.55,
"33": 410.77,
"34": 410.74,
"35": 411.75,
"36": 410.78,
"37": 411.56,
"38": 410.85,
"39": 411.08,
"40": 411.12,
"41": 411.1,
"42": 411.09,
"43": 410.87,
"44": 411.37,
"45": 411.68,
"46": 411.0,
"47": 410.09,
"48": 412.72,
"49": 410.42
}
},
"flux_image_t2i": {
"stages_ms": {
"InputValidationStage": 0.08,
"TextEncodingStage": 87.36,
"ConditioningStage": 0.024,
"TimestepPreparationStage": 2.53,
"LatentPreparationStage": 6.21,
"DenoisingStage": 8161.97,
"DecodingStage": 378.21
},
"denoise_step_ms": {
"0": 56.16,
"10": 163.54,
"20": 165.94,
"29": 165.26,
"39": 165.16,
"49": 165.36
},
"expected_e2e_ms": 8764.73,
"expected_avg_denoise_ms": 160.79,
"expected_median_denoise_ms": 165.14
},
"qwen_image_edit_ti2i": {
"notes": "single uploaded reference image, Qwen/Qwen-Image-Edit",
"expected_e2e_ms": 138500.0,
"expected_avg_denoise_ms": 720.0,
"expected_median_denoise_ms": 718.0,
"stages_ms": {
"InputValidationStage": 23,
"ImageEncodingStage": 1350.0,
"ImageVAEEncodingStage": 340.0,
"ConditioningStage": 0.13,
"TimestepPreparationStage": 13.78,
"LatentPreparationStage": 15.0,
"DenoisingStage": 36000.0,
"DecodingStage": 850.0
},
"denoise_step_ms": {
"0": 720.0,
"1": 720.0,
"2": 720.0,
"3": 720.0,
"4": 720.0,
"5": 720.0,
"6": 720.0,
"7": 720.0,
"8": 720.0,
"9": 720.0,
"10": 720.0,
"11": 720.0,
"12": 720.0,
"13": 720.0,
"14": 720.0,
"15": 720.0,
"16": 720.0,
"17": 720.0,
"18": 720.0,
"19": 720.0,
"20": 720.0,
"21": 720.0,
"22": 720.0,
"23": 720.0,
"24": 720.0,
"25": 720.0,
"26": 720.0,
"27": 720.0,
"28": 720.0,
"29": 720.0,
"30": 720.0,
"31": 720.0,
"32": 720.0,
"33": 720.0,
"34": 720.0,
"35": 720.0,
"36": 720.0,
"37": 720.0,
"38": 720.0,
"39": 720.0,
"40": 720.0,
"41": 720.0,
"42": 720.0,
"43": 720.0,
"44": 720.0,
"45": 720.0,
"46": 720.0,
"47": 720.0,
"48": 720.0,
"49": 720.0
}
},
"fastwan2_1_t2v": {
"notes": "Single-video generation using the default prompt",
"expected_e2e_ms": 95616.59,
"expected_avg_denoise_ms": 1798.77,
"expected_median_denoise_ms": 1786.78,
"stages_ms": {
"InputValidationStage": 1.03,
"TextEncodingStage": 3450.0,
"ConditioningStage": 1.0,
"TimestepPreparationStage": 6.0,
"LatentPreparationStage": 15.0,
"DenoisingStage": 90100.0,
"DecodingStage": 3650.0
},
"denoise_step_ms": {
"0": 3500.0,
"10": 1800.0,
"20": 1800.0,
"29": 1800.0,
"39": 1800.0,
"49": 1800.0
},
"frames_per_second": 0.51,
"total_frames": 49,
"avg_frame_time_ms": 1951.36
},
"wan2_2_i2v_a14b": {
"notes": "Wan-AI/Wan2.2-I2V-A14B",
"expected_e2e_ms": 282500.0,
"expected_avg_denoise_ms": 7000.0,
"expected_median_denoise_ms": 7000.19,
"stages_ms": {
"InputValidationStage": 26.26,
"TextEncodingStage": 2749.6,
"ConditioningStage": 2.0,
"TimestepPreparationStage": 2.0,
"LatentPreparationStage": 10.0,
"ImageVAEEncodingStage": 2031.0,
"DenoisingStage": 278000.0,
"DecodingStage": 2849.6
},
"denoise_step_ms": {
"0": 24000.0,
"8": 7000.0,
"16": 7000.0,
"23": 7000.0,
"31": 7000.0,
"39": 7000.0
}
},
"wan2_1_i2v_14b_480P": {
"stages_ms": {
"per_frame_generation": null
},
"denoise_step_ms": {
"0": 5432.37,
"1": 4861.66,
"2": 4890.23,
"3": 4919.9,
"4": 4929.49,
"5": 4927.35,
"6": 4927.6,
"7": 4932.02,
"8": 4897.83,
"9": 4897.16,
"10": 4899.59,
"11": 4901.51,
"12": 4896.38,
"13": 4914.3,
"14": 4901.91,
"15": 4898.58,
"16": 4903.18,
"17": 4899.54,
"18": 4910.5,
"19": 4898.18,
"20": 4901.56,
"21": 4903.12,
"22": 4896.82,
"23": 4901.01,
"24": 4897.42,
"25": 4911.94,
"26": 4901.29,
"27": 4894.79,
"28": 4899.57,
"29": 4900.63,
"30": 4904.78,
"31": 4899.79,
"32": 4898.01,
"33": 4910.81,
"34": 4901.16,
"35": 4898.48,
"36": 4894.97,
"37": 4905.9,
"38": 4901.52,
"39": 4898.56,
"40": 4900.19,
"41": 4896.5,
"42": 4899.89,
"43": 4898.45,
"44": 4897.27,
"45": 4895.47,
"46": 4889.26,
"47": 4887.44,
"48": 4894.07,
"49": 4888.59
},
"expected_e2e_ms": 254552.63,
"expected_avg_denoise_ms": 4912.17,
"expected_median_denoise_ms": 4899.69
},
"wan2_2_i2v_14b_720P": {
"stages_ms": {
"InputValidationStage": 47.04,
"TextEncodingStage": 2321.27,
"ImageEncodingStage": 3244.34,
"ConditioningStage": 0.0234,
"TimestepPreparationStage": 2.88,
"LatentPreparationStage": 5.24,
"ImageVAEEncodingStage": 1887.64,
"DenoisingStage": 245826.78,
"DecodingStage": 2882.45,
"per_frame_generation": null
},
"denoise_step_ms": {
"0": 5429.38,
"10": 4901.39,
"20": 4912.89,
"29": 4900.75,
"39": 4906.23,
"49": 4892.55
},
"expected_e2e_ms": 254850.94,
"expected_avg_denoise_ms": 4914.73,
"expected_median_denoise_ms": 4903.27
},
"wan2_2_ti2v_5b": {
"notes": "Text-and-Image-to-Video generation baseline for Wan2.2-TI2V-5B",
"expected_e2e_ms": 178300.0,
"expected_avg_denoise_ms": 3250.0,
"expected_median_denoise_ms": 3260.0,
"stages_ms": {
"InputValidationStage": 150.0,
"TextEncodingStage": 3000.0,
"ConditioningStage": 1.0,
"TimestepPreparationStage": 6.0,
"LatentPreparationStage": 30.0,
"DenoisingStage": 162900.0,
"DecodingStage": 14767.0
},
"denoise_step_ms": {
"0": 3700.0,
"10": 3300.0,
"20": 3300.0,
"29": 3300.0,
"39": 3300.0,
"49": 3300.0
},
"frames_per_second": null,
"total_frames": null,
"avg_frame_time_ms": null
}
} }
}
} }
python/sglang/multimodal_gen/test/server/test_server_performance.py
+118 -51
View File
@@ -1,6 +1,5 @@
""" """
Config-driven diffusion performance test with pytest parametrization. Config-driven diffusion performance test with pytest parametrization.
Adding a new model/scenario = adding one DiffusionCase entry in diffusion_config.py.
""" """
from __future__ import annotations from __future__ import annotations
@@ -15,20 +14,20 @@ from openai import OpenAI
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
from sglang.multimodal_gen.test.server.conftest import _GLOBAL_PERF_RESULTS from sglang.multimodal_gen.test.server.conftest import _GLOBAL_PERF_RESULTS
from sglang.multimodal_gen.test.server.diffusion_config import ( from sglang.multimodal_gen.test.server.test_server_utils import (
BASELINE_CONFIG,
DIFFUSION_CASES,
DiffusionCase,
)
from sglang.multimodal_gen.test.server.diffusion_server import (
VALIDATOR_REGISTRY, VALIDATOR_REGISTRY,
PerformanceValidator, PerformanceValidator,
ServerContext, ServerContext,
ServerManager, ServerManager,
VideoPerformanceValidator,
WarmupRunner, WarmupRunner,
download_image_from_url, download_image_from_url,
) )
from sglang.multimodal_gen.test.server.testcase_configs import (
BASELINE_CONFIG,
DIFFUSION_CASES,
DiffusionTestCase,
PerformanceSummary,
)
from sglang.multimodal_gen.test.test_utils import ( from sglang.multimodal_gen.test.test_utils import (
get_dynamic_server_port, get_dynamic_server_port,
read_perf_records, read_perf_records,
@@ -42,13 +41,13 @@ logger = init_logger(__name__)
@pytest.fixture(params=DIFFUSION_CASES, ids=lambda c: c.id) @pytest.fixture(params=DIFFUSION_CASES, ids=lambda c: c.id)
def case(request) -> DiffusionCase: def case(request) -> DiffusionTestCase:
"""Provide a DiffusionCase for each test.""" """Provide a DiffusionTestCase for each test."""
return request.param return request.param
@pytest.fixture @pytest.fixture
def diffusion_server(case: DiffusionCase) -> ServerContext: def diffusion_server(case: DiffusionTestCase) -> ServerContext:
"""Start a diffusion server for a single case and tear it down afterwards.""" """Start a diffusion server for a single case and tear it down afterwards."""
default_port = get_dynamic_server_port() default_port = get_dynamic_server_port()
port = int(os.environ.get("SGLANG_TEST_SERVER_PORT", default_port)) port = int(os.environ.get("SGLANG_TEST_SERVER_PORT", default_port))
@@ -79,17 +78,17 @@ def diffusion_server(case: DiffusionCase) -> ServerContext:
) )
warmup.run_text_warmups(case.warmup_text) warmup.run_text_warmups(case.warmup_text)
if case.warmup_edit > 0 and case.image_edit_prompt and case.image_edit_path: if case.warmup_edit > 0 and case.edit_prompt and case.image_path:
# Handle URL or local path # Handle URL or local path
image_path = case.image_edit_path image_path = case.image_path
if case.is_image_url(): if case.is_image_url():
image_path = download_image_from_url(str(case.image_edit_path)) image_path = download_image_from_url(str(case.image_path))
else: else:
image_path = Path(case.image_edit_path) image_path = Path(case.image_path)
warmup.run_edit_warmups( warmup.run_edit_warmups(
count=case.warmup_edit, count=case.warmup_edit,
edit_prompt=case.image_edit_prompt, edit_prompt=case.edit_prompt,
image_path=image_path, image_path=image_path,
) )
except Exception as exc: except Exception as exc:
@@ -132,12 +131,13 @@ class TestDiffusionPerformance:
def _run_and_collect( def _run_and_collect(
self, self,
ctx: ServerContext, ctx: ServerContext,
case: DiffusionCase, case: DiffusionTestCase,
generate_fn: Callable[[], None], generate_fn: Callable[[], None],
) -> tuple[dict, dict]: ) -> tuple[dict, dict]:
"""Run generation and collect performance records.""" """Run generation and collect performance records."""
log_path = ctx.perf_log_path log_path = ctx.perf_log_path
prev_len = len(read_perf_records(log_path)) prev_len = len(read_perf_records(log_path))
log_wait_timeout = 1200
generate_fn() generate_fn()
@@ -145,22 +145,25 @@ class TestDiffusionPerformance:
"total_inference_time", "total_inference_time",
prev_len, prev_len,
log_path, log_path,
timeout=log_wait_timeout,
) )
scenario = BASELINE_CONFIG.scenarios[case.scenario_name] stage_metrics = {}
stage_metrics, _ = wait_for_stage_metrics( if perf_record:
perf_record.get("request_id", ""),
prev_len, stage_metrics, _ = wait_for_stage_metrics(
len(scenario.stages_ms), perf_record.get("request_id", ""),
log_path, prev_len,
) log_path,
timeout=log_wait_timeout,
)
return perf_record, stage_metrics return perf_record, stage_metrics
def _generate_for_case( def _generate_for_case(
self, self,
ctx: ServerContext, ctx: ServerContext,
case: DiffusionCase, case: DiffusionTestCase,
) -> Callable[[], None]: ) -> Callable[[], None]:
"""Return appropriate generation function for the case.""" """Return appropriate generation function for the case."""
client = self._client(ctx) client = self._client(ctx)
@@ -192,21 +195,35 @@ class TestDiffusionPerformance:
job = client.videos.create(**create_kwargs) # type: ignore[attr-defined] job = client.videos.create(**create_kwargs) # type: ignore[attr-defined]
video_id = job.id video_id = job.id
deadline = time.time() + 600 job_completed = False
is_baseline_generation_mode = (
os.environ.get("SGLANG_GEN_BASELINE", "0") == "1"
)
timeout = 3600.0 if is_baseline_generation_mode else 600.0
deadline = time.time() + timeout
while True: while True:
page = client.videos.list() # type: ignore[attr-defined] page = client.videos.list() # type: ignore[attr-defined]
item = next((v for v in page.data if v.id == video_id), None) item = next((v for v in page.data if v.id == video_id), None)
if item and getattr(item, "status", None) == "completed": if item and getattr(item, "status", None) == "completed":
job_completed = True
break break
if time.time() > deadline: if time.time() > deadline:
pytest.fail( break
f"{case.id}: video job {video_id} did not complete in time"
)
time.sleep(5) time.sleep(5)
if not job_completed:
if is_baseline_generation_mode:
logger.warning(
f"{case.id}: video job {video_id} timed out during baseline generation. "
"Attempting to collect performance data anyway."
)
return b""
pytest.fail(f"{case.id}: video job {video_id} did not complete in time")
# download video # download video
resp = client.videos.download_content(video_id=video_id) # type: ignore[attr-defined] resp = client.videos.download_content(video_id=video_id) # type: ignore[attr-defined]
content = resp.read() content = resp.read()
@@ -235,14 +252,14 @@ class TestDiffusionPerformance:
def generate_image_edit(): def generate_image_edit():
"""TI2I: Text + Image ? Image edit.""" """TI2I: Text + Image ? Image edit."""
if not case.image_edit_prompt or not case.image_edit_path: if not case.edit_prompt or not case.image_path:
pytest.skip(f"{case.id}: no edit config") pytest.skip(f"{case.id}: no edit config")
# Handle URL or local path # Handle URL or local path
if case.is_image_url(): if case.is_image_url():
image_path = download_image_from_url(str(case.image_edit_path)) image_path = download_image_from_url(str(case.image_path))
else: else:
image_path = Path(case.image_edit_path) image_path = Path(case.image_path)
if not image_path.exists(): if not image_path.exists():
pytest.skip(f"{case.id}: file missing: {image_path}") pytest.skip(f"{case.id}: file missing: {image_path}")
@@ -250,7 +267,7 @@ class TestDiffusionPerformance:
result = client.images.edit( result = client.images.edit(
model=case.model_path, model=case.model_path,
image=fh, image=fh,
prompt=case.image_edit_prompt, prompt=case.edit_prompt,
n=1, n=1,
size=case.output_size, size=case.output_size,
response_format="b64_json", response_format="b64_json",
@@ -275,21 +292,21 @@ class TestDiffusionPerformance:
def generate_image_to_video(): def generate_image_to_video():
"""I2V: Image ? Video (optional prompt).""" """I2V: Image ? Video (optional prompt)."""
if not case.image_edit_path: if not case.image_path:
pytest.skip(f"{case.id}: no input image configured") pytest.skip(f"{case.id}: no input image configured")
# Handle URL or local path # Handle URL or local path
if case.is_image_url(): if case.is_image_url():
image_path = download_image_from_url(str(case.image_edit_path)) image_path = download_image_from_url(str(case.image_path))
else: else:
image_path = Path(case.image_edit_path) image_path = Path(case.image_path)
if not image_path.exists(): if not image_path.exists():
pytest.skip(f"{case.id}: file missing: {image_path}") pytest.skip(f"{case.id}: file missing: {image_path}")
with image_path.open("rb") as fh: with image_path.open("rb") as fh:
_create_and_download_video( _create_and_download_video(
model=case.model_path, model=case.model_path,
prompt=case.image_edit_prompt, prompt=case.edit_prompt,
size=case.output_size, size=case.output_size,
seconds=video_seconds, seconds=video_seconds,
input_reference=fh, input_reference=fh,
@@ -297,49 +314,55 @@ class TestDiffusionPerformance:
def generate_text_image_to_video(): def generate_text_image_to_video():
"""TI2V: Text + Image ? Video.""" """TI2V: Text + Image ? Video."""
if not case.image_edit_prompt or not case.image_edit_path: if not case.edit_prompt or not case.image_path:
pytest.skip(f"{case.id}: no edit config") pytest.skip(f"{case.id}: no edit config")
# Handle URL or local path # Handle URL or local path
if case.is_image_url(): if case.is_image_url():
image_path = download_image_from_url(str(case.image_edit_path)) image_path = download_image_from_url(str(case.image_path))
else: else:
image_path = Path(case.image_edit_path) image_path = Path(case.image_path)
if not image_path.exists(): if not image_path.exists():
pytest.skip(f"{case.id}: file missing: {image_path}") pytest.skip(f"{case.id}: file missing: {image_path}")
with image_path.open("rb") as fh: with image_path.open("rb") as fh:
_create_and_download_video( _create_and_download_video(
model=case.model_path, model=case.model_path,
prompt=case.image_edit_prompt, prompt=case.edit_prompt,
size=case.output_size, size=case.output_size,
seconds=video_seconds, seconds=video_seconds,
input_reference=fh, input_reference=fh,
) )
if case.modality == "video": if case.modality == "video":
if case.image_edit_path and case.image_edit_prompt: if case.image_path and case.edit_prompt:
return generate_text_image_to_video return generate_text_image_to_video
elif case.image_edit_path: elif case.image_path:
return generate_image_to_video return generate_image_to_video
else: else:
return generate_video return generate_video
# Image modality # Image modality
if case.image_edit_prompt and case.image_edit_path: if case.edit_prompt and case.image_path:
return generate_image_edit return generate_image_edit
return generate_image return generate_image
def _validate_and_record( def _validate_and_record(
self, self,
case: DiffusionCase, case: DiffusionTestCase,
perf_record: dict, perf_record: dict,
stage_metrics: dict, stage_metrics: dict,
) -> None: ) -> None:
"""Validate metrics and record results.""" """Validate metrics and record results."""
scenario = BASELINE_CONFIG.scenarios[case.scenario_name] is_baseline_generation_mode = os.environ.get("SGLANG_GEN_BASELINE", "0") == "1"
scenario = BASELINE_CONFIG.scenarios.get(case.id)
if scenario is None:
if is_baseline_generation_mode:
scenario = {} # Dummy scenario
else:
pytest.fail(f"Testcase '{case.id}' not in perf_baselines.json")
validator_name = case.custom_validator or "default" validator_name = case.custom_validator or "default"
validator_class = VALIDATOR_REGISTRY.get(validator_name, PerformanceValidator) validator_class = VALIDATOR_REGISTRY.get(validator_name, PerformanceValidator)
@@ -349,10 +372,17 @@ class TestDiffusionPerformance:
step_fractions=BASELINE_CONFIG.step_fractions, step_fractions=BASELINE_CONFIG.step_fractions,
) )
if isinstance(validator, VideoPerformanceValidator): summary = validator.collect_metrics(perf_record, stage_metrics)
summary = validator.validate(perf_record, stage_metrics, case.num_frames)
else: if is_baseline_generation_mode:
summary = validator.validate(perf_record, stage_metrics) self._dump_baseline_scenario(case, summary)
return
try:
validator.validate(perf_record, stage_metrics, case.num_frames)
except AssertionError:
self._dump_baseline_scenario(case, summary)
raise
if case.modality == "video" and summary.frames_per_second: if case.modality == "video" and summary.frames_per_second:
logger.info( logger.info(
@@ -428,9 +458,46 @@ class TestDiffusionPerformance:
summary.avg_frame_time_ms, summary.avg_frame_time_ms,
) )
def _dump_baseline_scenario(
self, case: DiffusionTestCase, summary: "PerformanceSummary"
) -> None:
"""Dump performance metrics as a JSON scenario for baselines."""
import json
denoise_steps_formatted = {
str(k): round(v, 2) for k, v in summary.sampled_steps.items()
}
stages_formatted = {k: round(v, 2) for k, v in summary.stage_metrics.items()}
baseline = {
"stages_ms": stages_formatted,
"denoise_step_ms": denoise_steps_formatted,
"expected_e2e_ms": round(summary.e2e_ms, 2),
"expected_avg_denoise_ms": round(summary.avg_denoise_ms, 2),
"expected_median_denoise_ms": round(summary.median_denoise_ms, 2),
}
# Video-specific metrics
if case.modality == "video":
if "per_frame_generation" not in baseline["stages_ms"]:
baseline["stages_ms"]["per_frame_generation"] = (
round(summary.avg_frame_time_ms, 2)
if summary.avg_frame_time_ms
else None
)
output = f"""
To add this baseline, copy the following JSON snippet into
the "scenarios" section of perf_baselines.json:
"{case.id}": {json.dumps(baseline, indent=4)}
"""
print(output)
def test_diffusion_perf( def test_diffusion_perf(
self, self,
case: DiffusionCase, case: DiffusionTestCase,
diffusion_server: ServerContext, diffusion_server: ServerContext,
): ):
"""Single parametrized test that runs for all cases. """Single parametrized test that runs for all cases.
python/sglang/multimodal_gen/test/server/diffusion_server.py → python/sglang/multimodal_gen/test/server/test_server_utils.py
+133 -85
View File
@@ -7,7 +7,9 @@ from __future__ import annotations
import os import os
import statistics import statistics
import subprocess import subprocess
import sys
import tempfile import tempfile
import threading
import time import time
from dataclasses import dataclass, field from dataclasses import dataclass, field
from pathlib import Path from pathlib import Path
@@ -18,7 +20,7 @@ from openai import OpenAI
from sglang.multimodal_gen.runtime.utils.common import kill_process_tree from sglang.multimodal_gen.runtime.utils.common import kill_process_tree
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
from sglang.multimodal_gen.test.server.diffusion_config import ( from sglang.multimodal_gen.test.server.testcase_configs import (
PerformanceSummary, PerformanceSummary,
ScenarioConfig, ScenarioConfig,
ToleranceConfig, ToleranceConfig,
@@ -74,6 +76,7 @@ class ServerContext:
perf_log_path: Path perf_log_path: Path
log_dir: Path log_dir: Path
_stdout_fh: Any = field(repr=False) _stdout_fh: Any = field(repr=False)
_log_thread: threading.Thread | None = field(default=None, repr=False)
def cleanup(self) -> None: def cleanup(self) -> None:
"""Clean up server resources.""" """Clean up server resources."""
@@ -127,19 +130,42 @@ class ServerManager:
command.extend(self.extra_args.strip().split()) command.extend(self.extra_args.strip().split())
env = os.environ.copy() env = os.environ.copy()
env["SGL_DIFFUSION_STAGE_LOGGING"] = "1" env["SGLANG_DIFFUSION_STAGE_LOGGING"] = "1"
env["SGLANG_PERF_LOG_DIR"] = log_dir.as_posix() env["SGLANG_PERF_LOG_DIR"] = log_dir.as_posix()
stdout_fh = stdout_path.open("w", encoding="utf-8", buffering=1) stdout_fh = stdout_path.open("w", encoding="utf-8", buffering=1)
process = subprocess.Popen( process = subprocess.Popen(
command, command,
stdout=stdout_fh, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, stderr=subprocess.STDOUT,
text=True, text=True,
bufsize=1, bufsize=1,
env=env, env=env,
) )
log_thread = None
if process.stdout:
def _log_pipe(pipe: Any, file: Any) -> None:
"""Read from pipe and write to file and stdout."""
try:
with pipe:
for line in iter(pipe.readline, ""):
sys.stdout.write(line)
file.write(line)
file.flush()
except Exception as e:
logger.error("Log pipe thread error: %s", e)
finally:
file.close()
logger.debug("Log pipe thread finished.")
log_thread = threading.Thread(
target=_log_pipe, args=(process.stdout, stdout_fh)
)
log_thread.daemon = True
log_thread.start()
logger.info( logger.info(
"[server-test] Starting server pid=%s, model=%s, log=%s", "[server-test] Starting server pid=%s, model=%s, log=%s",
process.pid, process.pid,
@@ -157,6 +183,7 @@ class ServerManager:
perf_log_path=perf_log_path, perf_log_path=perf_log_path,
log_dir=log_dir, log_dir=log_dir,
_stdout_fh=stdout_fh, _stdout_fh=stdout_fh,
_log_thread=log_thread,
) )
def _wait_for_ready(self, process: subprocess.Popen, stdout_path: Path) -> None: def _wait_for_ready(self, process: subprocess.Popen, stdout_path: Path) -> None:
@@ -264,6 +291,8 @@ class WarmupRunner:
class PerformanceValidator: class PerformanceValidator:
"""Validates performance metrics against expectations.""" """Validates performance metrics against expectations."""
is_video_gen: bool = False
def __init__( def __init__(
self, self,
scenario: ScenarioConfig, scenario: ScenarioConfig,
@@ -273,104 +302,134 @@ class PerformanceValidator:
self.scenario = scenario self.scenario = scenario
self.tolerances = tolerances self.tolerances = tolerances
self.step_fractions = step_fractions self.step_fractions = step_fractions
self.is_baseline_generation_mode = (
os.environ.get("SGLANG_GEN_BASELINE", "0") == "1"
)
def _assert_le(self, name: str, actual: float, expected: float, tolerance: float):
"""Assert that actual is less than or equal to expected within a tolerance."""
upper_bound = expected * (1 + tolerance)
assert actual <= upper_bound, (
f"Validation failed for '{name}'.\n"
f" - Actual: {actual:.4f}ms\n"
f" - Expected: {expected:.4f}ms\n"
f" - Limit: {upper_bound:.4f}ms (tolerance: {tolerance:.1%})"
)
def validate( def validate(
self, perf_record: dict, stage_metrics: dict, *args, **kwargs
) -> PerformanceSummary:
"""Validate all performance metrics and return summary."""
summary = self.collect_metrics(perf_record, stage_metrics)
if self.is_baseline_generation_mode:
return summary
self._validate_e2e(summary)
self._validate_denoise_agg(summary)
self._validate_denoise_steps(summary)
self._validate_stages(summary)
return summary
def collect_metrics(
self, self,
perf_record: dict, perf_record: dict,
stage_metrics: dict, stage_metrics: dict,
) -> PerformanceSummary: ) -> PerformanceSummary:
"""Validate all performance metrics and return summary.""" """Collect all performance metrics into a summary without validation."""
self._validate_e2e(perf_record)
avg_denoise, median_denoise = self._validate_denoise_agg(perf_record)
sampled_steps = self._validate_denoise_steps(perf_record)
self._validate_stages(stage_metrics)
return PerformanceSummary(
e2e_ms=float(perf_record["total_duration_ms"]),
avg_denoise_ms=avg_denoise,
median_denoise_ms=median_denoise,
stage_metrics=stage_metrics,
sampled_steps=sampled_steps,
)
def _validate_e2e(self, perf_record: dict) -> None:
"""Validate end-to-end performance."""
e2e_ms = float(perf_record.get("total_duration_ms", 0.0)) e2e_ms = float(perf_record.get("total_duration_ms", 0.0))
assert e2e_ms > 0, "E2E duration missing"
upper = self.scenario.expected_e2e_ms * (1 + self.tolerances.e2e)
assert e2e_ms <= upper, f"E2E {e2e_ms:.2f}ms exceeds {upper:.2f}ms"
def _validate_denoise_agg(self, perf_record: dict) -> tuple[float, float]:
"""Validate aggregate denoising metrics."""
steps = [ steps = [
s s
for s in perf_record.get("steps", []) or [] for s in perf_record.get("steps", []) or []
if s.get("name") == "denoising_step_guided" and "duration_ms" in s if s.get("name") == "denoising_step_guided" and "duration_ms" in s
] ]
assert steps, "Denoising step timings missing"
durations = [float(s["duration_ms"]) for s in steps] avg_denoise = 0.0
avg = sum(durations) / len(durations) median_denoise = 0.0
median = statistics.median(durations) if steps:
durations = [float(s["duration_ms"]) for s in steps]
avg_upper = self.scenario.expected_avg_denoise_ms * ( avg_denoise = sum(durations) / len(durations)
1 + self.tolerances.denoise_agg median_denoise = statistics.median(durations)
)
med_upper = self.scenario.expected_median_denoise_ms * (
1 + self.tolerances.denoise_agg
)
assert avg <= avg_upper, f"Avg denoise {avg:.2f}ms exceeds {avg_upper:.2f}ms"
assert (
median <= med_upper
), f"Median denoise {median:.2f}ms exceeds {med_upper:.2f}ms"
return avg, median
def _validate_denoise_steps(self, perf_record: dict) -> dict[int, float]:
"""Validate individual denoising steps."""
steps = [
s
for s in perf_record.get("steps", []) or []
if s.get("name") == "denoising_step_guided" and "duration_ms" in s
]
per_step = { per_step = {
int(s["index"]): float(s["duration_ms"]) int(s["index"]): float(s["duration_ms"])
for s in steps for s in steps
if s.get("index") is not None if s.get("index") is not None
} }
sample_indices = sample_step_indices(per_step, self.step_fractions) sample_indices = sample_step_indices(per_step, self.step_fractions)
sampled = {idx: per_step[idx] for idx in sample_indices} sampled_steps = {idx: per_step[idx] for idx in sample_indices}
for idx in sample_indices: return PerformanceSummary(
e2e_ms=e2e_ms,
avg_denoise_ms=avg_denoise,
median_denoise_ms=median_denoise,
stage_metrics=stage_metrics,
sampled_steps=sampled_steps,
)
def _validate_e2e(self, summary: PerformanceSummary) -> None:
"""Validate end-to-end performance."""
assert summary.e2e_ms > 0, "E2E duration missing"
self._assert_le(
"E2E Latency",
summary.e2e_ms,
self.scenario.expected_e2e_ms,
self.tolerances.e2e,
)
def _validate_denoise_agg(self, summary: PerformanceSummary) -> None:
"""Validate aggregate denoising metrics."""
assert summary.avg_denoise_ms > 0, "Denoising step timings missing"
self._assert_le(
"Average Denoise Step",
summary.avg_denoise_ms,
self.scenario.expected_avg_denoise_ms,
self.tolerances.denoise_agg,
)
self._assert_le(
"Median Denoise Step",
summary.median_denoise_ms,
self.scenario.expected_median_denoise_ms,
self.tolerances.denoise_agg,
)
def _validate_denoise_steps(self, summary: PerformanceSummary) -> None:
"""Validate individual denoising steps."""
for idx, actual in summary.sampled_steps.items():
expected = self.scenario.denoise_step_ms.get(idx) expected = self.scenario.denoise_step_ms.get(idx)
if expected is None: if expected is None:
continue continue
self._assert_le(
f"Denoise Step {idx}",
actual,
expected,
self.tolerances.denoise_step,
)
actual = per_step[idx] def _validate_stages(self, summary: PerformanceSummary) -> None:
upper = expected * (1 + self.tolerances.denoise_step)
assert actual <= upper, f"Step {idx}: {actual:.2f}ms > {upper:.2f}ms"
return sampled
def _validate_stages(self, stage_metrics: dict) -> None:
"""Validate stage-level metrics.""" """Validate stage-level metrics."""
assert stage_metrics, "Stage metrics missing" assert summary.stage_metrics, "Stage metrics missing"
for stage, expected in self.scenario.stages_ms.items(): for stage, expected in self.scenario.stages_ms.items():
actual = stage_metrics.get(stage) if stage == "per_frame_generation" and self.is_video_gen:
continue
actual = summary.stage_metrics.get(stage)
assert actual is not None, f"Stage {stage} timing missing" assert actual is not None, f"Stage {stage} timing missing"
upper = expected * (1 + self.tolerances.stage) self._assert_le(
assert actual <= upper, f"Stage {stage}: {actual:.2f}ms > {upper:.2f}ms" f"Stage '{stage}'",
actual,
expected,
self.tolerances.stage,
)
class VideoPerformanceValidator(PerformanceValidator): class VideoPerformanceValidator(PerformanceValidator):
"""Extended validator for video diffusion with frame-level metrics.""" """Extended validator for video diffusion with frame-level metrics."""
is_video_gen = True
def validate( def validate(
self, self,
perf_record: dict, perf_record: dict,
@@ -385,7 +444,8 @@ class VideoPerformanceValidator(PerformanceValidator):
summary.avg_frame_time_ms = summary.e2e_ms / num_frames summary.avg_frame_time_ms = summary.e2e_ms / num_frames
summary.frames_per_second = 1000.0 / summary.avg_frame_time_ms summary.frames_per_second = 1000.0 / summary.avg_frame_time_ms
self._validate_frame_rate(summary) if not self.is_baseline_generation_mode:
self._validate_frame_rate(summary)
return summary return summary
@@ -393,24 +453,12 @@ class VideoPerformanceValidator(PerformanceValidator):
"""Validate frame generation performance.""" """Validate frame generation performance."""
expected_frame_time = self.scenario.stages_ms.get("per_frame_generation") expected_frame_time = self.scenario.stages_ms.get("per_frame_generation")
if expected_frame_time and summary.avg_frame_time_ms: if expected_frame_time and summary.avg_frame_time_ms:
upper = expected_frame_time * (1 + self.tolerances.stage) self._assert_le(
assert ( "Average Frame Time",
summary.avg_frame_time_ms <= upper summary.avg_frame_time_ms,
), f"Avg frame time {summary.avg_frame_time_ms:.2f}ms exceeds {upper:.2f}ms" expected_frame_time,
self.tolerances.stage,
def _validate_stages(self, stage_metrics: dict) -> None: )
"""Validate video-specific stages."""
assert stage_metrics, "Stage metrics missing"
for stage, expected in self.scenario.stages_ms.items():
if stage == "per_frame_generation":
continue
actual = stage_metrics.get(stage)
assert actual is not None, f"Stage {stage} timing missing"
upper = expected * (1 + self.tolerances.stage)
assert actual <= upper, f"Stage {stage}: {actual:.2f}ms > {upper:.2f}ms"
# Registry of validators by name # Registry of validators by name
python/sglang/multimodal_gen/test/server/diffusion_config.py → python/sglang/multimodal_gen/test/server/testcase_configs.py
+75 -37
View File
@@ -1,5 +1,19 @@
""" """
Configuration and data structures for diffusion performance tests. Configuration and data structures for diffusion performance tests.
Usage:
pytest python/sglang/multimodal_gen/test/server/test_server_performance.py
# for a single testcase, look for the name of the testcases in DIFFUSION_CASES
pytest python/sglang/multimodal_gen/test/server/test_server_performance.py -k qwen_image_t2i
To add a new testcase:
1. add your testcase with case-id: `my_new_test_case_id` to DIFFUSION_CASES
2. run `SGLANG_GEN_BASELINE=1 pytest -s python/sglang/multimodal_gen/test/server/test_server_performance.py -k my_new_test_case_id`
3. insert or override the corresponding scenario in `scenarios` section of perf_baselines.json with the output baseline of step-2
""" """
from __future__ import annotations from __future__ import annotations
@@ -78,34 +92,36 @@ class BaselineConfig:
@dataclass(frozen=True) @dataclass(frozen=True)
class DiffusionCase: class DiffusionTestCase:
"""Configuration for a single model/scenario test case.""" """Configuration for a single model/scenario test case."""
id: str # pytest test id id: str # pytest test id and scenario name
model_path: str # HF repo or local path model_path: str # HF repo or local path
scenario_name: str # key into BASELINE_CONFIG.scenarios
modality: str = "image" # "image" or "video" or "3d" modality: str = "image" # "image" or "video" or "3d"
prompt: str | None = None # text prompt for generation
output_size: str = "1024x1024" # output image dimensions (or video resolution) output_size: str = "1024x1024" # output image dimensions (or video resolution)
# inputs and conditioning
prompt: str | None = None # text prompt for generation
edit_prompt: str | None = None # prompt for editing
image_path: Path | str | None = None # input image/video for editing (Path or URL)
# duration
seconds: int = 4 # for video: duration in seconds
num_frames: int | None = None # for video: number of frames num_frames: int | None = None # for video: number of frames
fps: int | None = None # for video: frames per second fps: int | None = None # for video: frames per second
warmup_text: int = 1 # number of text-to-image/video warmups warmup_text: int = 1 # number of text-to-image/video warmups
warmup_edit: int = 0 # number of image/video-edit warmups warmup_edit: int = 0 # number of image/video-edit warmups
image_edit_prompt: str | None = None # prompt for editing
image_edit_path: Path | str | None = (
None # input image/video for editing (Path or URL)
)
startup_grace_seconds: float = 0.0 # wait time after server starts startup_grace_seconds: float = 0.0 # wait time after server starts
custom_validator: str | None = None # optional custom validator name custom_validator: str | None = None # optional custom validator name
seconds: int = 4 # for video: duration in seconds
def is_image_url(self) -> bool: def is_image_url(self) -> bool:
"""Check if image_edit_path is a URL.""" """Check if image_edit_path is a URL."""
if self.image_edit_path is None: if self.image_path is None:
return False return False
return isinstance(self.image_edit_path, str) and ( return isinstance(self.image_path, str) and (
self.image_edit_path.startswith("http://") self.image_path.startswith("http://")
or self.image_edit_path.startswith("https://") or self.image_path.startswith("https://")
) )
@@ -128,12 +144,11 @@ IMAGE_INPUT_FILE = Path(__file__).resolve().parents[1] / "test_files" / "girl.jp
# All test cases with clean default values # All test cases with clean default values
# To test different models, simply add more DiffusionCase entries # To test different models, simply add more DiffusionCase entries
DIFFUSION_CASES: list[DiffusionCase] = [ DIFFUSION_CASES: list[DiffusionTestCase] = [
# === Text to Image (T2I) === # === Text to Image (T2I) ===
DiffusionCase( DiffusionTestCase(
id="qwen_image_t2i", id="qwen_image_t2i",
model_path="Qwen/Qwen-Image", model_path="Qwen/Qwen-Image",
scenario_name="text_to_image",
modality="image", modality="image",
prompt="A futuristic cityscape at sunset with flying cars", prompt="A futuristic cityscape at sunset with flying cars",
output_size="1024x1024", output_size="1024x1024",
@@ -141,10 +156,9 @@ DIFFUSION_CASES: list[DiffusionCase] = [
warmup_edit=0, warmup_edit=0,
startup_grace_seconds=30.0, startup_grace_seconds=30.0,
), ),
DiffusionCase( DiffusionTestCase(
id="flux_image_t2i", id="flux_image_t2i",
model_path="black-forest-labs/FLUX.1-dev", model_path="black-forest-labs/FLUX.1-dev",
scenario_name="text_to_image",
modality="image", modality="image",
prompt="A futuristic cityscape at sunset with flying cars", prompt="A futuristic cityscape at sunset with flying cars",
output_size="1024x1024", output_size="1024x1024",
@@ -153,24 +167,23 @@ DIFFUSION_CASES: list[DiffusionCase] = [
startup_grace_seconds=30.0, startup_grace_seconds=30.0,
), ),
# === Text and Image to Image (TI2I) === # === Text and Image to Image (TI2I) ===
DiffusionCase( DiffusionTestCase(
id="qwen_image_edit_ti2i", id="qwen_image_edit_ti2i",
model_path="Qwen/Qwen-Image-Edit", model_path="Qwen/Qwen-Image-Edit",
scenario_name="image_edit",
modality="image", modality="image",
prompt=None, # not used for editing prompt=None, # not used for editing
output_size="1024x1536", output_size="1024x1536",
warmup_text=0, warmup_text=0,
warmup_edit=1, warmup_edit=1,
image_edit_prompt="Convert 2D style to 3D style", edit_prompt="Convert 2D style to 3D style",
image_edit_path="https://github.com/lm-sys/lm-sys.github.io/releases/download/test/TI2I_Qwen_Image_Edit_Input.jpg", image_path="https://github.com/lm-sys/lm-sys.github.io/releases/download/test/TI2I_Qwen_Image_Edit_Input.jpg",
startup_grace_seconds=30.0, startup_grace_seconds=30.0,
), ),
# === Text to Video (T2V) === # === Text to Video (T2V) ===
DiffusionCase( # TODO: FastWan2.1, FastWan2.2
DiffusionTestCase(
id="fastwan2_1_t2v", id="fastwan2_1_t2v",
model_path="Wan-AI/Wan2.1-T2V-1.3B-Diffusers", model_path="Wan-AI/Wan2.1-T2V-1.3B-Diffusers",
scenario_name="text_to_video",
modality="video", modality="video",
prompt="A curious raccoon", prompt="A curious raccoon",
output_size="848x480", output_size="848x480",
@@ -181,39 +194,64 @@ DIFFUSION_CASES: list[DiffusionCase] = [
custom_validator="video", custom_validator="video",
), ),
# === Image to Video (I2V) === # === Image to Video (I2V) ===
DiffusionCase( DiffusionTestCase(
id="wan2_2_i2v", id="wan2_2_i2v_a14b",
model_path="Wan-AI/Wan2.2-I2V-A14B-Diffusers", model_path="Wan-AI/Wan2.2-I2V-A14B-Diffusers",
scenario_name="image_to_video",
modality="video", modality="video",
prompt="generate", # passing in something since failing if no prompt is passed prompt="generate", # passing in something since failing if no prompt is passed
warmup_text=0, # warmups only for image gen models warmup_text=0, # warmups only for image gen models
warmup_edit=0, warmup_edit=0,
output_size="832x1104", output_size="832x1104",
image_edit_prompt="generate", edit_prompt="generate",
image_edit_path="https://github.com/Wan-Video/Wan2.2/blob/990af50de458c19590c245151197326e208d7191/examples/i2v_input.JPG?raw=true", image_path="https://github.com/Wan-Video/Wan2.2/blob/990af50de458c19590c245151197326e208d7191/examples/i2v_input.JPG?raw=true",
startup_grace_seconds=30.0, startup_grace_seconds=30.0,
custom_validator="video", custom_validator="video",
seconds=1, seconds=1,
), ),
# === Text and Image to Video (TI2V) === # === Text and Image to Video (TI2V) ===
DiffusionCase( DiffusionTestCase(
id="wan2_2_ti2v_5b", id="wan2_1_i2v_14b_480P",
model_path="Wan-AI/Wan2.2-TI2V-5B-Diffusers", model_path="Wan-AI/Wan2.1-I2V-14B-480P-Diffusers",
scenario_name="text_image_to_video", output_size="832x1104",
modality="video", modality="video",
prompt="Animate this image", prompt="Animate this image",
edit_prompt="Add dynamic motion to the scene",
image_path="https://github.com/lm-sys/lm-sys.github.io/releases/download/test/TI2I_Qwen_Image_Edit_Input.jpg",
warmup_text=0, # warmups only for image gen models
warmup_edit=0,
startup_grace_seconds=30.0,
custom_validator="video",
seconds=1,
),
DiffusionTestCase(
id="wan2_2_i2v_14b_720P",
model_path="Wan-AI/Wan2.1-I2V-14B-720P-Diffusers",
modality="video",
prompt="Animate this image",
edit_prompt="Add dynamic motion to the scene",
image_path="https://github.com/lm-sys/lm-sys.github.io/releases/download/test/TI2I_Qwen_Image_Edit_Input.jpg",
output_size="832x1104", output_size="832x1104",
warmup_text=0, # warmups only for image gen models warmup_text=0, # warmups only for image gen models
warmup_edit=0, warmup_edit=0,
image_edit_prompt="Add dynamic motion to the scene",
image_edit_path="https://github.com/lm-sys/lm-sys.github.io/releases/download/test/TI2I_Qwen_Image_Edit_Input.jpg",
startup_grace_seconds=30.0, startup_grace_seconds=30.0,
custom_validator="video", custom_validator="video",
seconds=4, seconds=1,
),
DiffusionTestCase(
id="wan2_2_ti2v_5b",
model_path="Wan-AI/Wan2.2-TI2V-5B-Diffusers",
modality="video",
output_size="832x1104",
prompt="Animate this image",
edit_prompt="Add dynamic motion to the scene",
image_path="https://github.com/lm-sys/lm-sys.github.io/releases/download/test/TI2I_Qwen_Image_Edit_Input.jpg",
warmup_text=0, # warmups only for image gen models
warmup_edit=0,
startup_grace_seconds=30.0,
custom_validator="video",
seconds=1,
), ),
] ]
# Load global configuration # Load global configuration
BASELINE_CONFIG = BaselineConfig.load(Path(__file__).with_name("perf_baselines.json")) BASELINE_CONFIG = BaselineConfig.load(Path(__file__).with_name("perf_baselines.json"))
python/sglang/multimodal_gen/test/test_utils.py
+18 -8
View File
@@ -172,6 +172,11 @@ def wait_for_perf_record(
if rec.get("tag") == tag: if rec.get("tag") == tag:
return rec, len(records) return rec, len(records)
time.sleep(0.5) time.sleep(0.5)
if os.environ.get("SGLANG_GEN_BASELINE", "0") == "1":
records = read_perf_records(log_path)
return {}, len(records)
raise AssertionError( raise AssertionError(
f"Timeout waiting for perf log entry '{tag}' (start_len={prev_len})" f"Timeout waiting for perf log entry '{tag}' (start_len={prev_len})"
) )
@@ -180,15 +185,21 @@ def wait_for_perf_record(
def wait_for_stage_metrics( def wait_for_stage_metrics(
request_id: str, request_id: str,
prev_len: int, prev_len: int,
expected_count: int,
log_path: Path, log_path: Path,
timeout: float = 120.0, timeout: float = 300.0,
) -> tuple[dict[str, float], int]: ) -> tuple[dict[str, float], int]:
deadline = time.time() + timeout deadline = time.time() + timeout
metrics: dict[str, float] = {} metrics: dict[str, float] = {}
while time.time() < deadline: while time.time() < deadline:
records = read_perf_records(log_path) records = read_perf_records(log_path)
for rec in records[prev_len:]: for rec in records[prev_len:]:
# Check if the request is completed
if (
rec.get("tag") == "total_inference_time"
and rec.get("request_id") == request_id
):
return metrics, len(records)
if ( if (
rec.get("tag") == "pipeline_stage_metric" rec.get("tag") == "pipeline_stage_metric"
and rec.get("request_id") == request_id and rec.get("request_id") == request_id
@@ -197,13 +208,12 @@ def wait_for_stage_metrics(
duration = rec.get("duration_ms") duration = rec.get("duration_ms")
if stage is not None and duration is not None: if stage is not None and duration is not None:
metrics[str(stage)] = float(duration) metrics[str(stage)] = float(duration)
if len(metrics) >= expected_count:
return metrics, len(records)
time.sleep(0.5) time.sleep(0.5)
raise AssertionError(
f"Timeout waiting for stage metrics for request {request_id} " if os.environ.get("SGLANG_GEN_BASELINE", "0") == "1":
f"(collected={len(metrics)} expected={expected_count})" records = read_perf_records(log_path)
) return {}, len(records)
raise AssertionError(f"Timeout waiting for stage metrics for request {request_id} ")
def sample_step_indices( def sample_step_indices(
python/sglang/multimodal_gen/utils.py
+4 -4
View File
@@ -48,8 +48,8 @@ PRECISION_TO_TYPE = {
"bf16": torch.bfloat16, "bf16": torch.bfloat16,
} }
STR_BACKEND_ENV_VAR: str = "SGL_DIFFUSION_ATTENTION_BACKEND" STR_BACKEND_ENV_VAR: str = "SGLANG_DIFFUSION_ATTENTION_BACKEND"
STR_ATTN_CONFIG_ENV_VAR: str = "SGL_DIFFUSION_ATTENTION_CONFIG" STR_ATTN_CONFIG_ENV_VAR: str = "SGLANG_DIFFUSION_ATTENTION_CONFIG"
def find_nccl_library() -> str: def find_nccl_library() -> str:
@@ -59,12 +59,12 @@ def find_nccl_library() -> str:
After importing `torch`, `libnccl.so.2` or `librccl.so.1` can be After importing `torch`, `libnccl.so.2` or `librccl.so.1` can be
found by `ctypes` automatically. found by `ctypes` automatically.
""" """
so_file = envs.SGL_DIFFUSION_NCCL_SO_PATH so_file = envs.SGLANG_DIFFUSION_NCCL_SO_PATH
# manually load the nccl library # manually load the nccl library
if so_file: if so_file:
logger.info( logger.info(
"Found nccl from environment variable SGL_DIFFUSION_NCCL_SO_PATH=%s", "Found nccl from environment variable SGLANG_DIFFUSION_NCCL_SO_PATH=%s",
so_file, so_file,
) )
else: else:
scripts/ci/validate_and_download_models.py
+7 -2
View File
@@ -66,11 +66,16 @@ RUNNER_LABEL_MODEL_MAP: Dict[str, List[str]] = {
"Qwen/Qwen3-Embedding-8B", "Qwen/Qwen3-Embedding-8B",
"Qwen/QwQ-32B-AWQ", "Qwen/QwQ-32B-AWQ",
"Qwen/Qwen3-30B-A3B", "Qwen/Qwen3-30B-A3B",
"Skywork/Skywork-Reward-Llama-3.1-8B-v0.2",
# diffusion
"Qwen/Qwen-Image", "Qwen/Qwen-Image",
"Qwen/Qwen-Image-Edit", "Qwen/Qwen-Image-Edit",
"Skywork/Skywork-Reward-Llama-3.1-8B-v0.2", "black-forest-labs/FLUX.1-dev",
"Wan-AI/Wan2.2-I2V-A14B-Diffusers", "Wan-AI/Wan2.1-T2V-1.3B-Diffusers",
"Wan-AI/Wan2.1-I2V-14B-480P-Diffusers",
"Wan-AI/Wan2.1-I2V-14B-720P-Diffusers",
"Wan-AI/Wan2.2-TI2V-5B-Diffusers", "Wan-AI/Wan2.2-TI2V-5B-Diffusers",
"Wan-AI/Wan2.2-I2V-A14B-Diffusers",
], ],
"2-gpu-runner": [ "2-gpu-runner": [
"mistralai/Mixtral-8x7B-Instruct-v0.1", "mistralai/Mixtral-8x7B-Instruct-v0.1",