[diffusion] fix: fix hunyuanvideo and add 2-gpu ci test (#13720)
Co-authored-by: Mick <mickjagger19@icloud.com>
This commit is contained in:
Yuhao Yang
@@ -15,7 +15,10 @@ from sglang.multimodal_gen.configs.models.encoders import (
|
||||
LlamaConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.models.vaes import HunyuanVAEConfig
|
||||
from sglang.multimodal_gen.configs.pipeline_configs import PipelineConfig
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.base import (
|
||||
ModelTaskType,
|
||||
PipelineConfig,
|
||||
)
|
||||
|
||||
PROMPT_TEMPLATE_ENCODE_VIDEO = (
|
||||
"<|start_header_id|>system<|end_header_id|>\n\nDescribe the video by detailing the following aspects: "
|
||||
@@ -66,6 +69,8 @@ def clip_postprocess_text(outputs: BaseEncoderOutput, _text_inputs) -> torch.ten
|
||||
class HunyuanConfig(PipelineConfig):
|
||||
"""Base configuration for HunYuan pipeline architecture."""
|
||||
|
||||
task_type: ModelTaskType = ModelTaskType.T2V
|
||||
|
||||
# HunyuanConfig-specific parameters with defaults
|
||||
# DiT
|
||||
dit_config: DiTConfig = field(default_factory=HunyuanVideoConfig)
|
||||
|
||||
@@ -414,6 +414,8 @@ class SamplingParams:
|
||||
if user_params is None:
|
||||
return
|
||||
|
||||
subclass_defined_fields = set(type(self).__annotations__.keys())
|
||||
|
||||
# Compare against current instance to avoid constructing a default instance
|
||||
default_params = SamplingParams()
|
||||
|
||||
@@ -430,7 +432,7 @@ class SamplingParams:
|
||||
if field_name != "output_file_name"
|
||||
else user_params.output_file_path is not None
|
||||
)
|
||||
if is_user_modified:
|
||||
if is_user_modified and field_name not in subclass_defined_fields:
|
||||
if hasattr(self, field_name):
|
||||
setattr(self, field_name, user_value)
|
||||
|
||||
|
||||
@@ -874,7 +874,7 @@ def norm_infer(
|
||||
out: Optional[Tensor] = None,
|
||||
):
|
||||
M, N = x.shape
|
||||
assert x.stride(-1) == 1
|
||||
x = x.contiguous()
|
||||
if weight is not None:
|
||||
assert weight.shape == (N,)
|
||||
assert weight.stride(-1) == 1
|
||||
|
||||
@@ -29,8 +29,12 @@ from sglang.multimodal_gen.runtime.pipelines_core.stages.denoising import (
|
||||
vsa_available,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.server_args import ServerArgs
|
||||
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
|
||||
from sglang.multimodal_gen.runtime.utils.perf_logger import StageProfiler
|
||||
from sglang.multimodal_gen.utils import dict_to_3d_list
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
# TODO: use base methods of DenoisingStage
|
||||
class DmdDenoisingStage(DenoisingStage):
|
||||
@@ -139,132 +143,140 @@ class DmdDenoisingStage(DenoisingStage):
|
||||
|
||||
# Run denoising loop
|
||||
denoising_loop_start_time = time.time()
|
||||
self.start_profile(batch=batch)
|
||||
with self.progress_bar(total=len(timesteps)) as progress_bar:
|
||||
for i, t in enumerate(timesteps):
|
||||
# Skip if interrupted
|
||||
if hasattr(self, "interrupt") and self.interrupt:
|
||||
continue
|
||||
# Expand latents for I2V
|
||||
noise_latents = latents.clone()
|
||||
latent_model_input = latents.to(target_dtype)
|
||||
|
||||
if batch.image_latent is not None:
|
||||
latent_model_input = torch.cat(
|
||||
[
|
||||
latent_model_input,
|
||||
batch.image_latent.permute(0, 2, 1, 3, 4),
|
||||
],
|
||||
dim=2,
|
||||
).to(target_dtype)
|
||||
assert not torch.isnan(
|
||||
latent_model_input
|
||||
).any(), "latent_model_input contains nan"
|
||||
|
||||
# Prepare inputs for transformer
|
||||
t_expand = t.repeat(latent_model_input.shape[0])
|
||||
guidance_expand = (
|
||||
torch.tensor(
|
||||
[server_args.pipeline_config.embedded_cfg_scale]
|
||||
* latent_model_input.shape[0],
|
||||
dtype=torch.float32,
|
||||
device=get_local_torch_device(),
|
||||
).to(target_dtype)
|
||||
* 1000.0
|
||||
if server_args.pipeline_config.embedded_cfg_scale is not None
|
||||
else None
|
||||
)
|
||||
|
||||
# Predict noise residual
|
||||
with torch.autocast(
|
||||
device_type="cuda",
|
||||
dtype=target_dtype,
|
||||
enabled=autocast_enabled,
|
||||
with StageProfiler(
|
||||
f"denoising_step_{i}", logger=logger, timings=batch.timings
|
||||
):
|
||||
if (
|
||||
vsa_available
|
||||
and self.attn_backend == VideoSparseAttentionBackend
|
||||
):
|
||||
self.attn_metadata_builder_cls = (
|
||||
self.attn_backend.get_builder_cls()
|
||||
)
|
||||
# Expand latents for I2V
|
||||
noise_latents = latents.clone()
|
||||
latent_model_input = latents.to(target_dtype)
|
||||
|
||||
if self.attn_metadata_builder_cls is not None:
|
||||
self.attn_metadata_builder = (
|
||||
self.attn_metadata_builder_cls()
|
||||
if batch.image_latent is not None:
|
||||
latent_model_input = torch.cat(
|
||||
[
|
||||
latent_model_input,
|
||||
batch.image_latent.permute(0, 2, 1, 3, 4),
|
||||
],
|
||||
dim=2,
|
||||
).to(target_dtype)
|
||||
assert not torch.isnan(
|
||||
latent_model_input
|
||||
).any(), "latent_model_input contains nan"
|
||||
|
||||
# Prepare inputs for transformer
|
||||
t_expand = t.repeat(latent_model_input.shape[0])
|
||||
guidance_expand = (
|
||||
torch.tensor(
|
||||
[server_args.pipeline_config.embedded_cfg_scale]
|
||||
* latent_model_input.shape[0],
|
||||
dtype=torch.float32,
|
||||
device=get_local_torch_device(),
|
||||
).to(target_dtype)
|
||||
* 1000.0
|
||||
if server_args.pipeline_config.embedded_cfg_scale is not None
|
||||
else None
|
||||
)
|
||||
|
||||
# Predict noise residual
|
||||
with torch.autocast(
|
||||
device_type="cuda",
|
||||
dtype=target_dtype,
|
||||
enabled=autocast_enabled,
|
||||
):
|
||||
if (
|
||||
vsa_available
|
||||
and self.attn_backend == VideoSparseAttentionBackend
|
||||
):
|
||||
self.attn_metadata_builder_cls = (
|
||||
self.attn_backend.get_builder_cls()
|
||||
)
|
||||
# TODO(will): clean this up
|
||||
attn_metadata = self.attn_metadata_builder.build( # type: ignore
|
||||
current_timestep=i, # type: ignore
|
||||
raw_latent_shape=batch.raw_latent_shape[2:5], # type: ignore
|
||||
patch_size=server_args.pipeline_config.dit_config.patch_size, # type: ignore
|
||||
STA_param=batch.STA_param, # type: ignore
|
||||
VSA_sparsity=server_args.VSA_sparsity, # type: ignore
|
||||
device=get_local_torch_device(), # type: ignore
|
||||
) # type: ignore
|
||||
assert (
|
||||
attn_metadata is not None
|
||||
), "attn_metadata cannot be None"
|
||||
|
||||
if self.attn_metadata_builder_cls is not None:
|
||||
self.attn_metadata_builder = (
|
||||
self.attn_metadata_builder_cls()
|
||||
)
|
||||
# TODO(will): clean this up
|
||||
attn_metadata = self.attn_metadata_builder.build( # type: ignore
|
||||
current_timestep=i, # type: ignore
|
||||
raw_latent_shape=batch.raw_latent_shape[2:5], # type: ignore
|
||||
patch_size=server_args.pipeline_config.dit_config.patch_size, # type: ignore
|
||||
STA_param=batch.STA_param, # type: ignore
|
||||
VSA_sparsity=server_args.VSA_sparsity, # type: ignore
|
||||
device=get_local_torch_device(), # type: ignore
|
||||
) # type: ignore
|
||||
assert (
|
||||
attn_metadata is not None
|
||||
), "attn_metadata cannot be None"
|
||||
else:
|
||||
attn_metadata = None
|
||||
else:
|
||||
attn_metadata = None
|
||||
else:
|
||||
attn_metadata = None
|
||||
|
||||
batch.is_cfg_negative = False
|
||||
with set_forward_context(
|
||||
current_timestep=i,
|
||||
attn_metadata=attn_metadata,
|
||||
forward_batch=batch,
|
||||
# server_args=server_args
|
||||
):
|
||||
# Run transformer
|
||||
pred_noise = self.transformer(
|
||||
latent_model_input.permute(0, 2, 1, 3, 4),
|
||||
prompt_embeds,
|
||||
t_expand,
|
||||
guidance=guidance_expand,
|
||||
**image_kwargs,
|
||||
**pos_cond_kwargs,
|
||||
).permute(0, 2, 1, 3, 4)
|
||||
batch.is_cfg_negative = False
|
||||
with set_forward_context(
|
||||
current_timestep=i,
|
||||
attn_metadata=attn_metadata,
|
||||
forward_batch=batch,
|
||||
# server_args=server_args
|
||||
):
|
||||
# Run transformer
|
||||
pred_noise = self.transformer(
|
||||
latent_model_input.permute(0, 2, 1, 3, 4),
|
||||
prompt_embeds,
|
||||
t_expand,
|
||||
guidance=guidance_expand,
|
||||
**image_kwargs,
|
||||
**pos_cond_kwargs,
|
||||
).permute(0, 2, 1, 3, 4)
|
||||
|
||||
pred_video = pred_noise_to_pred_video(
|
||||
pred_noise=pred_noise.flatten(0, 1),
|
||||
noise_input_latent=noise_latents.flatten(0, 1),
|
||||
timestep=t_expand,
|
||||
scheduler=self.scheduler,
|
||||
).unflatten(0, pred_noise.shape[:2])
|
||||
pred_video = pred_noise_to_pred_video(
|
||||
pred_noise=pred_noise.flatten(0, 1),
|
||||
noise_input_latent=noise_latents.flatten(0, 1),
|
||||
timestep=t_expand,
|
||||
scheduler=self.scheduler,
|
||||
).unflatten(0, pred_noise.shape[:2])
|
||||
|
||||
if i < len(timesteps) - 1:
|
||||
next_timestep = timesteps[i + 1] * torch.ones(
|
||||
[1], dtype=torch.long, device=pred_video.device
|
||||
)
|
||||
noise = torch.randn(
|
||||
video_raw_latent_shape,
|
||||
dtype=pred_video.dtype,
|
||||
generator=batch.generator[0],
|
||||
).to(self.device)
|
||||
if sp_group:
|
||||
noise = rearrange(
|
||||
noise,
|
||||
"b (n t) c h w -> b n t c h w",
|
||||
n=sp_world_size,
|
||||
).contiguous()
|
||||
noise = noise[:, rank_in_sp_group, :, :, :, :]
|
||||
latents = self.scheduler.add_noise(
|
||||
pred_video.flatten(0, 1),
|
||||
noise.flatten(0, 1),
|
||||
next_timestep,
|
||||
).unflatten(0, pred_video.shape[:2])
|
||||
else:
|
||||
latents = pred_video
|
||||
if i < len(timesteps) - 1:
|
||||
next_timestep = timesteps[i + 1] * torch.ones(
|
||||
[1], dtype=torch.long, device=pred_video.device
|
||||
)
|
||||
noise = torch.randn(
|
||||
video_raw_latent_shape,
|
||||
dtype=pred_video.dtype,
|
||||
generator=batch.generator[0],
|
||||
).to(self.device)
|
||||
if sp_group:
|
||||
noise = rearrange(
|
||||
noise,
|
||||
"b (n t) c h w -> b n t c h w",
|
||||
n=sp_world_size,
|
||||
).contiguous()
|
||||
noise = noise[:, rank_in_sp_group, :, :, :, :]
|
||||
latents = self.scheduler.add_noise(
|
||||
pred_video.flatten(0, 1),
|
||||
noise.flatten(0, 1),
|
||||
next_timestep,
|
||||
).unflatten(0, pred_video.shape[:2])
|
||||
else:
|
||||
latents = pred_video
|
||||
|
||||
# Update progress bar
|
||||
if i == len(timesteps) - 1 or (
|
||||
(i + 1) > num_warmup_steps
|
||||
and (i + 1) % self.scheduler.order == 0
|
||||
and progress_bar is not None
|
||||
):
|
||||
progress_bar.update()
|
||||
# Update progress bar
|
||||
if i == len(timesteps) - 1 or (
|
||||
(i + 1) > num_warmup_steps
|
||||
and (i + 1) % self.scheduler.order == 0
|
||||
and progress_bar is not None
|
||||
):
|
||||
progress_bar.update()
|
||||
|
||||
self.step_profile()
|
||||
|
||||
self.stop_profile(batch)
|
||||
denoising_loop_end_time = time.time()
|
||||
if len(timesteps) > 0:
|
||||
self.log_info(
|
||||
|
||||
@@ -124,15 +124,20 @@ def _print_warning_once(logger: Logger, msg: str) -> None:
|
||||
logger.warning(msg, stacklevel=2)
|
||||
|
||||
|
||||
def _get_rank_info():
|
||||
"""Get rank and local rank from environment variables."""
|
||||
def get_is_main_process():
|
||||
try:
|
||||
rank = int(os.environ["RANK"])
|
||||
local_rank = int(os.environ["LOCAL_RANK"])
|
||||
except (KeyError, ValueError):
|
||||
rank = 0
|
||||
local_rank = 0
|
||||
return rank, local_rank
|
||||
return rank == 0
|
||||
|
||||
|
||||
def get_is_local_main_process():
|
||||
try:
|
||||
rank = int(os.environ["LOCAL_RANK"])
|
||||
except (KeyError, ValueError):
|
||||
rank = 0
|
||||
return rank == 0
|
||||
|
||||
|
||||
def _log_process_aware(
|
||||
@@ -145,9 +150,8 @@ def _log_process_aware(
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
"""Helper function to log a message if the process rank matches the criteria."""
|
||||
rank, local_rank = _get_rank_info()
|
||||
is_main_process = rank == 0
|
||||
is_local_main_process = local_rank == 0
|
||||
is_main_process = get_is_main_process()
|
||||
is_local_main_process = get_is_local_main_process()
|
||||
|
||||
should_log = (
|
||||
not main_process_only
|
||||
@@ -393,8 +397,7 @@ def suppress_other_loggers(not_suppress_on_main_rank: bool = False):
|
||||
|
||||
should_suppress = True
|
||||
if not_suppress_on_main_rank:
|
||||
rank, _ = _get_rank_info()
|
||||
if rank == 0:
|
||||
if get_is_main_process() == 0:
|
||||
should_suppress = False
|
||||
|
||||
loggers_to_suppress = ["urllib3"]
|
||||
|
||||
@@ -15,7 +15,10 @@ from dateutil.tz import UTC
|
||||
|
||||
import sglang
|
||||
import sglang.multimodal_gen.envs as envs
|
||||
from sglang.multimodal_gen.runtime.utils.logging_utils import _SGLDiffusionLogger
|
||||
from sglang.multimodal_gen.runtime.utils.logging_utils import (
|
||||
_SGLDiffusionLogger,
|
||||
get_is_main_process,
|
||||
)
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
@@ -246,14 +249,15 @@ class PerformanceLogger:
|
||||
)
|
||||
|
||||
try:
|
||||
log_dir = get_diffusion_perf_log_dir()
|
||||
if not os.path.exists(log_dir):
|
||||
os.makedirs(log_dir, exist_ok=True)
|
||||
if get_is_main_process():
|
||||
log_dir = get_diffusion_perf_log_dir()
|
||||
if not os.path.exists(log_dir):
|
||||
os.makedirs(log_dir, exist_ok=True)
|
||||
|
||||
log_file = os.path.join(log_dir, "performance.log")
|
||||
log_file = os.path.join(log_dir, "performance.log")
|
||||
|
||||
with open(log_file, "a", encoding="utf-8") as f:
|
||||
f.write(json.dumps(dataclasses.asdict(record)) + "\n")
|
||||
with open(log_file, "a", encoding="utf-8") as f:
|
||||
f.write(json.dumps(dataclasses.asdict(record)) + "\n")
|
||||
|
||||
except (OSError, PermissionError) as e:
|
||||
print(f"WARNING: Failed to log performance record: {e}", file=sys.stderr)
|
||||
|
||||
@@ -297,259 +297,434 @@
|
||||
"expected_avg_denoise_ms": 1820.02,
|
||||
"expected_median_denoise_ms": 1798.65
|
||||
},
|
||||
"wan2_2_i2v_a14b": {
|
||||
|
||||
"wan2_2_ti2v_5b": {
|
||||
"stages_ms": {
|
||||
"InputValidationStage": 33.55,
|
||||
"TextEncodingStage": 2291.94,
|
||||
"ConditioningStage": 0.03,
|
||||
"TimestepPreparationStage": 2.37,
|
||||
"LatentPreparationStage": 8.87,
|
||||
"ImageVAEEncodingStage": 2092.43,
|
||||
"DenoisingStage": 255868.61,
|
||||
"DecodingStage": 2747.78,
|
||||
"InputValidationStage": 96.27,
|
||||
"TextEncodingStage": 2238.81,
|
||||
"ConditioningStage": 0.02,
|
||||
"TimestepPreparationStage": 2.39,
|
||||
"LatentPreparationStage": 27.62,
|
||||
"DenoisingStage": 134069.79,
|
||||
"DecodingStage": 13559.79,
|
||||
"per_frame_generation": null
|
||||
},
|
||||
"denoise_step_ms": {
|
||||
"0": 24266.38,
|
||||
"1": 4987.24,
|
||||
"2": 5023.9,
|
||||
"3": 5033.86,
|
||||
"4": 5039.19,
|
||||
"5": 5049.18,
|
||||
"6": 5036.97,
|
||||
"7": 5037.92,
|
||||
"8": 5040.91,
|
||||
"9": 5051.29,
|
||||
"10": 5039.41,
|
||||
"11": 5049.76,
|
||||
"12": 5044.17,
|
||||
"13": 5049.54,
|
||||
"14": 5050.61,
|
||||
"15": 39939.62,
|
||||
"16": 5037.41,
|
||||
"17": 5055.67,
|
||||
"18": 5059.63,
|
||||
"19": 5070.04,
|
||||
"20": 5078.54,
|
||||
"21": 5079.15,
|
||||
"22": 5065.42,
|
||||
"23": 5064.19,
|
||||
"24": 5065.35,
|
||||
"25": 5059.83,
|
||||
"26": 5055.47,
|
||||
"27": 5048.03,
|
||||
"28": 5044.28,
|
||||
"29": 5044.19,
|
||||
"30": 5040.91,
|
||||
"31": 5037.98,
|
||||
"32": 5026.15,
|
||||
"33": 5027.36,
|
||||
"34": 5018.56,
|
||||
"35": 5007.23,
|
||||
"36": 5012.65,
|
||||
"37": 5013.68,
|
||||
"38": 5005.83,
|
||||
"39": 5000.26
|
||||
"0": 3181.0,
|
||||
"1": 2561.67,
|
||||
"2": 2578.49,
|
||||
"3": 2582.1,
|
||||
"4": 2572.24,
|
||||
"5": 2577.72,
|
||||
"6": 2581.35,
|
||||
"7": 2578.79,
|
||||
"8": 2584.98,
|
||||
"9": 2588.49,
|
||||
"10": 2594.37,
|
||||
"11": 2591.19,
|
||||
"12": 2591.32,
|
||||
"13": 2595.35,
|
||||
"14": 2594.35,
|
||||
"15": 2595.62,
|
||||
"16": 2596.35,
|
||||
"17": 2596.11,
|
||||
"18": 2597.24,
|
||||
"19": 2603.13,
|
||||
"20": 2599.9,
|
||||
"21": 2601.48,
|
||||
"22": 2603.58,
|
||||
"23": 2601.13,
|
||||
"24": 2600.47,
|
||||
"25": 2604.13,
|
||||
"26": 2606.04,
|
||||
"27": 2605.3,
|
||||
"28": 2602.02,
|
||||
"29": 2601.83,
|
||||
"30": 2603.57,
|
||||
"31": 2606.63,
|
||||
"32": 2606.1,
|
||||
"33": 2602.24,
|
||||
"34": 2603.29,
|
||||
"35": 2602.34,
|
||||
"36": 2602.16,
|
||||
"37": 2608.14,
|
||||
"38": 2603.48,
|
||||
"39": 2601.7,
|
||||
"40": 2603.96,
|
||||
"41": 2604.58,
|
||||
"42": 2606.67,
|
||||
"43": 2603.52,
|
||||
"44": 2599.88,
|
||||
"45": 2598.66,
|
||||
"46": 2600.74,
|
||||
"47": 2602.31,
|
||||
"48": 2608.4,
|
||||
"49": 2606.02
|
||||
},
|
||||
"expected_e2e_ms": 263069.24,
|
||||
"expected_avg_denoise_ms": 6393.94,
|
||||
"expected_median_denoise_ms": 5044.18
|
||||
"expected_e2e_ms": 150004.2,
|
||||
"expected_avg_denoise_ms": 2608.84,
|
||||
"expected_median_denoise_ms": 2601.59
|
||||
},
|
||||
"wan2_1_i2v_14b_480P": {
|
||||
"fastwan2_2_ti2v_5b": {
|
||||
"stages_ms": {
|
||||
"InputValidationStage": 88.86,
|
||||
"TextEncodingStage": 2327.87,
|
||||
"ConditioningStage": 0.01,
|
||||
"TimestepPreparationStage": 58.66,
|
||||
"LatentPreparationStage": 28.55,
|
||||
"DmdDenoisingStage": 4438.3,
|
||||
"DecodingStage": 14177.77,
|
||||
"per_frame_generation": null
|
||||
},
|
||||
"denoise_step_ms": {
|
||||
"0": 5432.37,
|
||||
"1": 4861.66,
|
||||
"2": 4890.23,
|
||||
"3": 4919.9,
|
||||
"4": 4929.49,
|
||||
"5": 4927.35,
|
||||
"6": 4927.6,
|
||||
"7": 4932.02,
|
||||
"8": 4897.83,
|
||||
"9": 4897.16,
|
||||
"10": 4899.59,
|
||||
"11": 4901.51,
|
||||
"12": 4896.38,
|
||||
"13": 4914.3,
|
||||
"14": 4901.91,
|
||||
"15": 4898.58,
|
||||
"16": 4903.18,
|
||||
"17": 4899.54,
|
||||
"18": 4910.5,
|
||||
"19": 4898.18,
|
||||
"20": 4901.56,
|
||||
"21": 4903.12,
|
||||
"22": 4896.82,
|
||||
"23": 4901.01,
|
||||
"24": 4897.42,
|
||||
"25": 4911.94,
|
||||
"26": 4901.29,
|
||||
"27": 4894.79,
|
||||
"28": 4899.57,
|
||||
"29": 4900.63,
|
||||
"30": 4904.78,
|
||||
"31": 4899.79,
|
||||
"32": 4898.01,
|
||||
"33": 4910.81,
|
||||
"34": 4901.16,
|
||||
"35": 4898.48,
|
||||
"36": 4894.97,
|
||||
"37": 4905.9,
|
||||
"38": 4901.52,
|
||||
"39": 4898.56,
|
||||
"40": 4900.19,
|
||||
"41": 4896.5,
|
||||
"42": 4899.89,
|
||||
"43": 4898.45,
|
||||
"44": 4897.27,
|
||||
"45": 4895.47,
|
||||
"46": 4889.26,
|
||||
"47": 4887.44,
|
||||
"48": 4894.07,
|
||||
"49": 4888.59
|
||||
"0": 2022.21,
|
||||
"1": 1263.17,
|
||||
"2": 1149.59
|
||||
},
|
||||
"expected_e2e_ms": 254552.63,
|
||||
"expected_avg_denoise_ms": 4912.17,
|
||||
"expected_median_denoise_ms": 4899.69
|
||||
"expected_e2e_ms": 21133.36,
|
||||
"expected_avg_denoise_ms": 1478.32,
|
||||
"expected_median_denoise_ms": 1263.17
|
||||
},
|
||||
"wan2_1_i2v_14b_720P": {
|
||||
"fast_hunyuan_video": {
|
||||
"stages_ms": {
|
||||
"InputValidationStage": 0.07,
|
||||
"TextEncodingStage": 598.66,
|
||||
"ConditioningStage": 0.01,
|
||||
"TimestepPreparationStage": 57.26,
|
||||
"LatentPreparationStage": 15.38,
|
||||
"DenoisingStage": 3367.01,
|
||||
"DecodingStage": 2457.57,
|
||||
"per_frame_generation": null
|
||||
},
|
||||
"denoise_step_ms": {
|
||||
"0": 1577.86,
|
||||
"1": 161.04,
|
||||
"2": 406.81,
|
||||
"3": 408.67,
|
||||
"4": 403.55,
|
||||
"5": 405.31
|
||||
},
|
||||
"expected_e2e_ms": 6521.03,
|
||||
"expected_avg_denoise_ms": 560.54,
|
||||
"expected_median_denoise_ms": 406.06
|
||||
},
|
||||
"wan2_2_i2v_a14b_2gpu": {
|
||||
"stages_ms": {
|
||||
"InputValidationStage": 59.33,
|
||||
"TextEncodingStage": 6062.41,
|
||||
"ConditioningStage": 0.02,
|
||||
"TimestepPreparationStage": 2.2,
|
||||
"LatentPreparationStage": 8.93,
|
||||
"ImageVAEEncodingStage": 2075.47,
|
||||
"DenoisingStage": 382628.41,
|
||||
"DecodingStage": 2820.89
|
||||
},
|
||||
"denoise_step_ms": {
|
||||
"0": 31228.27,
|
||||
"1": 7723.86,
|
||||
"2": 7769.69,
|
||||
"3": 7795.93,
|
||||
"4": 7815.58,
|
||||
"5": 7829.48,
|
||||
"6": 7827.34,
|
||||
"7": 7825.35,
|
||||
"8": 7828.05,
|
||||
"9": 7809.53,
|
||||
"10": 7801.29,
|
||||
"11": 7790.96,
|
||||
"12": 7785.88,
|
||||
"13": 7785.5,
|
||||
"14": 7780.32,
|
||||
"15": 55411.1,
|
||||
"16": 7722.27,
|
||||
"17": 7761.31,
|
||||
"18": 7789.46,
|
||||
"19": 7800.6,
|
||||
"20": 7814.91,
|
||||
"21": 7799.62,
|
||||
"22": 7801.25,
|
||||
"23": 7798.27,
|
||||
"24": 7797.67,
|
||||
"25": 7795.97,
|
||||
"26": 7781.74,
|
||||
"27": 7784.16,
|
||||
"28": 7796.64,
|
||||
"29": 7789.75,
|
||||
"30": 7792.13,
|
||||
"31": 7790.99,
|
||||
"32": 7778.1,
|
||||
"33": 7777.78,
|
||||
"34": 7780.56,
|
||||
"35": 7778.22,
|
||||
"36": 7770.88,
|
||||
"37": 7771.56,
|
||||
"38": 7767.82,
|
||||
"39": 7769.23
|
||||
},
|
||||
"expected_e2e_ms": 393606.77,
|
||||
"expected_avg_denoise_ms": 9565.48,
|
||||
"expected_median_denoise_ms": 7790.98
|
||||
},
|
||||
"wan2_1_i2v_14b_480P_2gpu": {
|
||||
"stages_ms": {
|
||||
"InputValidationStage": 32.94,
|
||||
"TextEncodingStage": 2316.5,
|
||||
"ImageEncodingStage": 3026.2,
|
||||
"ConditioningStage": 0.01,
|
||||
"TimestepPreparationStage": 2.69,
|
||||
"LatentPreparationStage": 9.73,
|
||||
"ImageVAEEncodingStage": 2290.98,
|
||||
"DenoisingStage": 385080.09,
|
||||
"DecodingStage": 2984.69,
|
||||
"per_frame_generation": null
|
||||
},
|
||||
"denoise_step_ms": {
|
||||
"0": 8785.36,
|
||||
"1": 7644.16,
|
||||
"2": 7687.27,
|
||||
"3": 7703.9,
|
||||
"4": 7710.61,
|
||||
"5": 7716.32,
|
||||
"6": 7714.26,
|
||||
"7": 7711.27,
|
||||
"8": 7711.08,
|
||||
"9": 7706.57,
|
||||
"10": 7700.78,
|
||||
"11": 7696.03,
|
||||
"12": 7704.73,
|
||||
"13": 7699.99,
|
||||
"14": 7705.33,
|
||||
"15": 7701.11,
|
||||
"16": 7704.04,
|
||||
"17": 7695.31,
|
||||
"18": 7693.63,
|
||||
"19": 7686.34,
|
||||
"20": 7683.27,
|
||||
"21": 7689.82,
|
||||
"22": 7688.74,
|
||||
"23": 7686.01,
|
||||
"24": 7675.43,
|
||||
"25": 7679.86,
|
||||
"26": 7676.75,
|
||||
"27": 7671.65,
|
||||
"28": 7667.0,
|
||||
"29": 7669.83,
|
||||
"30": 7660.5,
|
||||
"31": 7666.82,
|
||||
"32": 7660.89,
|
||||
"33": 7668.75,
|
||||
"34": 7662.27,
|
||||
"35": 7659.71,
|
||||
"36": 7661.36,
|
||||
"37": 7664.87,
|
||||
"38": 7666.93,
|
||||
"39": 7661.05,
|
||||
"40": 7661.88,
|
||||
"41": 7657.96,
|
||||
"42": 7660.6,
|
||||
"43": 7669.82,
|
||||
"44": 7655.78,
|
||||
"45": 7654.25,
|
||||
"46": 7656.56,
|
||||
"47": 7652.37,
|
||||
"48": 7657.61,
|
||||
"49": 7644.6
|
||||
},
|
||||
"expected_e2e_ms": 395758.23,
|
||||
"expected_avg_denoise_ms": 7701.42,
|
||||
"expected_median_denoise_ms": 7676.09
|
||||
},
|
||||
"wan2_1_i2v_14b_720P_2gpu": {
|
||||
"stages_ms": {
|
||||
"InputValidationStage": 53.67,
|
||||
"TextEncodingStage": 2838,
|
||||
"ImageEncodingStage": 2694.14,
|
||||
"ConditioningStage": 0.01,
|
||||
"TimestepPreparationStage": 2.25,
|
||||
"LatentPreparationStage": 5.27,
|
||||
"ImageVAEEncodingStage": 1887.72,
|
||||
"DenoisingStage": 245828.78,
|
||||
"DecodingStage": 2872.0,
|
||||
"per_frame_generation": null
|
||||
},
|
||||
"denoise_step_ms": {
|
||||
"0": 5432.36,
|
||||
"1": 4867.45,
|
||||
"2": 4903.52,
|
||||
"3": 4916.85,
|
||||
"4": 4928.58,
|
||||
"5": 4927.72,
|
||||
"6": 4916.65,
|
||||
"7": 4910.61,
|
||||
"8": 4900.32,
|
||||
"9": 4901.14,
|
||||
"10": 4907.02,
|
||||
"11": 4908.95,
|
||||
"12": 4903.16,
|
||||
"13": 4910.68,
|
||||
"14": 4907.48,
|
||||
"15": 4901.52,
|
||||
"16": 4909.76,
|
||||
"17": 4905.68,
|
||||
"18": 4904.09,
|
||||
"19": 4908.87,
|
||||
"20": 4897.14,
|
||||
"21": 4905.94,
|
||||
"22": 4916.29,
|
||||
"23": 4904.12,
|
||||
"24": 4912.86,
|
||||
"25": 4904.16,
|
||||
"26": 4916.7,
|
||||
"27": 4918.63,
|
||||
"28": 4903.83,
|
||||
"29": 4909.2,
|
||||
"30": 4898.29,
|
||||
"31": 4901.82,
|
||||
"32": 4902.38,
|
||||
"33": 4910.32,
|
||||
"34": 4903.55,
|
||||
"35": 4902.21,
|
||||
"36": 4907.05,
|
||||
"37": 4911.2,
|
||||
"38": 4898.33,
|
||||
"39": 4902.63,
|
||||
"40": 4895.15,
|
||||
"41": 4898.9,
|
||||
"42": 4901.14,
|
||||
"43": 4896.49,
|
||||
"44": 4899.01,
|
||||
"45": 4896.71,
|
||||
"46": 4887.24,
|
||||
"47": 4892.91,
|
||||
"48": 4886.71,
|
||||
"49": 4885.16
|
||||
},
|
||||
"expected_e2e_ms": 254809.55,
|
||||
"expected_avg_denoise_ms": 4914.77,
|
||||
"expected_median_denoise_ms": 4903.96
|
||||
},
|
||||
"wan2_2_ti2v_5b": {
|
||||
"stages_ms": {
|
||||
"InputValidationStage": 94.18,
|
||||
"TextEncodingStage": 3413.73,
|
||||
"ImageEncodingStage": 3123.99,
|
||||
"ConditioningStage": 0.02,
|
||||
"TimestepPreparationStage": 2.48,
|
||||
"LatentPreparationStage": 22.52,
|
||||
"DenoisingStage": 30000,
|
||||
"DecodingStage": 12793.58,
|
||||
"TimestepPreparationStage": 3.39,
|
||||
"LatentPreparationStage": 6.68,
|
||||
"ImageVAEEncodingStage": 2261.05,
|
||||
"DenoisingStage": 386761.14,
|
||||
"DecodingStage": 2968.35,
|
||||
"per_frame_generation": null
|
||||
},
|
||||
"denoise_step_ms": {
|
||||
"0": 1021.97,
|
||||
"1": 407.7,
|
||||
"2": 410.44,
|
||||
"3": 411.69,
|
||||
"4": 411.45,
|
||||
"5": 410.84,
|
||||
"6": 411.26,
|
||||
"7": 412.68,
|
||||
"8": 412.24,
|
||||
"9": 410.15,
|
||||
"10": 413.4,
|
||||
"11": 410.18,
|
||||
"12": 411.94,
|
||||
"13": 411.54,
|
||||
"14": 409.91,
|
||||
"15": 412.89,
|
||||
"16": 412.08,
|
||||
"17": 411.64,
|
||||
"18": 411.58,
|
||||
"19": 410.54,
|
||||
"20": 411.42,
|
||||
"21": 412.88,
|
||||
"22": 412.22,
|
||||
"23": 412.97,
|
||||
"24": 412.13,
|
||||
"25": 413.21,
|
||||
"26": 413.07,
|
||||
"27": 410.89,
|
||||
"28": 411.56,
|
||||
"29": 414.19,
|
||||
"30": 412.68,
|
||||
"31": 411.13,
|
||||
"32": 412.41,
|
||||
"33": 412.79,
|
||||
"34": 411.02,
|
||||
"35": 410.2,
|
||||
"36": 410.27,
|
||||
"37": 411.63,
|
||||
"38": 410.8,
|
||||
"39": 411.52,
|
||||
"40": 411.6,
|
||||
"41": 411.75,
|
||||
"42": 410.56,
|
||||
"43": 411.65,
|
||||
"44": 411.82,
|
||||
"45": 410.83,
|
||||
"46": 410.61,
|
||||
"47": 411.5,
|
||||
"48": 410.61,
|
||||
"49": 411.6
|
||||
"0": 10021.98,
|
||||
"1": 7633.62,
|
||||
"2": 7676.46,
|
||||
"3": 7704.68,
|
||||
"4": 7725.09,
|
||||
"5": 7732.86,
|
||||
"6": 7735.42,
|
||||
"7": 7739.05,
|
||||
"8": 7740.89,
|
||||
"9": 7724.35,
|
||||
"10": 7730.2,
|
||||
"11": 7713.23,
|
||||
"12": 7715.93,
|
||||
"13": 7710.93,
|
||||
"14": 7699.95,
|
||||
"15": 7704.72,
|
||||
"16": 7704.03,
|
||||
"17": 7700.47,
|
||||
"18": 7702.0,
|
||||
"19": 7705.92,
|
||||
"20": 7704.35,
|
||||
"21": 7705.11,
|
||||
"22": 7693.85,
|
||||
"23": 7696.91,
|
||||
"24": 7689.6,
|
||||
"25": 7681.2,
|
||||
"26": 7675.63,
|
||||
"27": 7678.95,
|
||||
"28": 7683.82,
|
||||
"29": 7681.07,
|
||||
"30": 7671.07,
|
||||
"31": 7674.65,
|
||||
"32": 7679.56,
|
||||
"33": 7674.59,
|
||||
"34": 7672.16,
|
||||
"35": 7679.68,
|
||||
"36": 7670.81,
|
||||
"37": 7661.84,
|
||||
"38": 7668.58,
|
||||
"39": 7667.1,
|
||||
"40": 7670.22,
|
||||
"41": 7664.97,
|
||||
"42": 7667.3,
|
||||
"43": 7668.87,
|
||||
"44": 7663.43,
|
||||
"45": 7656.34,
|
||||
"46": 7662.81,
|
||||
"47": 7662.05,
|
||||
"48": 7654.13,
|
||||
"49": 7648.62
|
||||
},
|
||||
"expected_e2e_ms": 32954.73,
|
||||
"expected_avg_denoise_ms": 423.75,
|
||||
"expected_median_denoise_ms": 411.59
|
||||
"expected_e2e_ms": 397541.45,
|
||||
"expected_avg_denoise_ms": 7735.02,
|
||||
"expected_median_denoise_ms": 7681.14
|
||||
},
|
||||
"wan2_2_t2v_a14b_2gpu": {
|
||||
"stages_ms": {
|
||||
"InputValidationStage": 0.09,
|
||||
"TextEncodingStage": 2322.57,
|
||||
"ConditioningStage": 0.03,
|
||||
"TimestepPreparationStage": 2.29,
|
||||
"LatentPreparationStage": 3.08,
|
||||
"DenoisingStage": 79913.08,
|
||||
"DecodingStage": 1339.58
|
||||
},
|
||||
"denoise_step_ms": {
|
||||
"0": 19269.37,
|
||||
"1": 691.64,
|
||||
"2": 699.28,
|
||||
"3": 696.55,
|
||||
"4": 698.6,
|
||||
"5": 704.56,
|
||||
"6": 699.26,
|
||||
"7": 700.84,
|
||||
"8": 700.27,
|
||||
"9": 704.15,
|
||||
"10": 699.04,
|
||||
"11": 704.79,
|
||||
"12": 701.48,
|
||||
"13": 707.24,
|
||||
"14": 697.54,
|
||||
"15": 698.89,
|
||||
"16": 697.97,
|
||||
"17": 699.34,
|
||||
"18": 697.68,
|
||||
"19": 697.42,
|
||||
"20": 697.14,
|
||||
"21": 700.14,
|
||||
"22": 696.75,
|
||||
"23": 702.36,
|
||||
"24": 697.3,
|
||||
"25": 703.97,
|
||||
"26": 33676.93,
|
||||
"27": 700.4,
|
||||
"28": 703.68,
|
||||
"29": 691.86,
|
||||
"30": 706.1,
|
||||
"31": 704.18,
|
||||
"32": 700.34,
|
||||
"33": 698.62,
|
||||
"34": 698.66,
|
||||
"35": 699.77,
|
||||
"36": 700.96,
|
||||
"37": 701.02,
|
||||
"38": 703.98,
|
||||
"39": 702.18
|
||||
},
|
||||
"expected_e2e_ms": 83595.94,
|
||||
"expected_avg_denoise_ms": 1988.81,
|
||||
"expected_median_denoise_ms": 700.2
|
||||
},
|
||||
"wan2_1_t2v_14b_2gpu": {
|
||||
"stages_ms": {
|
||||
"InputValidationStage": 0.05,
|
||||
"TextEncodingStage": 2310.34,
|
||||
"ConditioningStage": 0.02,
|
||||
"TimestepPreparationStage": 2.42,
|
||||
"LatentPreparationStage": 27.7,
|
||||
"DenoisingStage": 803631.52,
|
||||
"DecodingStage": 8898.74,
|
||||
"per_frame_generation": null
|
||||
},
|
||||
"denoise_step_ms": {
|
||||
"0": 17347.88,
|
||||
"1": 15956.93,
|
||||
"2": 16027.54,
|
||||
"3": 16054.15,
|
||||
"4": 16081.46,
|
||||
"5": 16062.7,
|
||||
"6": 16058.56,
|
||||
"7": 16057.58,
|
||||
"8": 16061.04,
|
||||
"9": 16120.97,
|
||||
"10": 16036.84,
|
||||
"11": 16019.6,
|
||||
"12": 16042.29,
|
||||
"13": 16039.87,
|
||||
"14": 16063.0,
|
||||
"15": 16036.16,
|
||||
"16": 16079.82,
|
||||
"17": 16019.7,
|
||||
"18": 16061.5,
|
||||
"19": 16039.95,
|
||||
"20": 16009.42,
|
||||
"21": 16051.01,
|
||||
"22": 16039.31,
|
||||
"23": 16048.22,
|
||||
"24": 16071.41,
|
||||
"25": 16078.75,
|
||||
"26": 16061.78,
|
||||
"27": 16018.39,
|
||||
"28": 16041.44,
|
||||
"29": 16039.64,
|
||||
"30": 16041.89,
|
||||
"31": 16039.6,
|
||||
"32": 16038.97,
|
||||
"33": 15999.48,
|
||||
"34": 16019.93,
|
||||
"35": 16040.27,
|
||||
"36": 16020.3,
|
||||
"37": 16039.38,
|
||||
"38": 15999.4,
|
||||
"39": 16022.15,
|
||||
"40": 16042.32,
|
||||
"41": 16016.62,
|
||||
"42": 15998.92,
|
||||
"43": 16041.48,
|
||||
"44": 15999.63,
|
||||
"45": 16003.21,
|
||||
"46": 15995.91,
|
||||
"47": 16023.52,
|
||||
"48": 16016.64,
|
||||
"49": 16019.6
|
||||
},
|
||||
"expected_e2e_ms": 814884.71,
|
||||
"expected_avg_denoise_ms": 16062.92,
|
||||
"expected_median_denoise_ms": 16039.62
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
"""
|
||||
2 GPU Performance tests (A14B models) with --num-gpus 2 --ulysses-degree 2.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from sglang.multimodal_gen.test.server.test_server_perf_common import ( # noqa: F401
|
||||
DiffusionPerformanceBase,
|
||||
diffusion_server,
|
||||
)
|
||||
from sglang.multimodal_gen.test.server.testcase_configs import (
|
||||
TWO_GPU_CASES,
|
||||
DiffusionTestCase,
|
||||
)
|
||||
|
||||
|
||||
class TestDiffusionPerformanceTwoGpu(DiffusionPerformanceBase):
|
||||
"""Performance tests for 2-GPU diffusion cases."""
|
||||
|
||||
@pytest.fixture(params=TWO_GPU_CASES, ids=lambda c: c.id)
|
||||
def case(self, request) -> DiffusionTestCase:
|
||||
"""Provide a DiffusionTestCase for each 2-GPU test."""
|
||||
return request.param
|
||||
@@ -0,0 +1,31 @@
|
||||
"""
|
||||
Config-driven diffusion performance test with pytest parametrization.
|
||||
|
||||
|
||||
If the actual run is significantly better than the baseline, the improved cases with their updated baseline will be printed
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
|
||||
from sglang.multimodal_gen.test.server.test_server_perf_common import ( # noqa: F401
|
||||
DiffusionPerformanceBase,
|
||||
diffusion_server,
|
||||
)
|
||||
from sglang.multimodal_gen.test.server.testcase_configs import (
|
||||
ONE_GPU_CASES_A,
|
||||
DiffusionTestCase,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class TestDiffusionPerformanceOneGpu(DiffusionPerformanceBase):
|
||||
"""Performance tests for 1-GPU diffusion cases."""
|
||||
|
||||
@pytest.fixture(params=ONE_GPU_CASES_A, ids=lambda c: c.id)
|
||||
def case(self, request) -> DiffusionTestCase:
|
||||
"""Provide a DiffusionTestCase for each 1-GPU test."""
|
||||
return request.param
|
||||
@@ -0,0 +1,31 @@
|
||||
"""
|
||||
Config-driven diffusion performance test with pytest parametrization.
|
||||
|
||||
|
||||
If the actual run is significantly better than the baseline, the improved cases with their updated baseline will be printed
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
|
||||
from sglang.multimodal_gen.test.server.test_server_perf_common import ( # noqa: F401
|
||||
DiffusionPerformanceBase,
|
||||
diffusion_server,
|
||||
)
|
||||
from sglang.multimodal_gen.test.server.testcase_configs import (
|
||||
ONE_GPU_CASES_A,
|
||||
DiffusionTestCase,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class TestDiffusionPerformanceOneGpu(DiffusionPerformanceBase):
|
||||
"""Performance tests for 1-GPU diffusion cases."""
|
||||
|
||||
@pytest.fixture(params=ONE_GPU_CASES_A, ids=lambda c: c.id)
|
||||
def case(self, request) -> DiffusionTestCase:
|
||||
"""Provide a DiffusionTestCase for each 1-GPU test."""
|
||||
return request.param
|
||||
@@ -28,7 +28,6 @@ from sglang.multimodal_gen.test.server.test_server_utils import (
|
||||
)
|
||||
from sglang.multimodal_gen.test.server.testcase_configs import (
|
||||
BASELINE_CONFIG,
|
||||
DIFFUSION_CASES,
|
||||
DiffusionTestCase,
|
||||
PerformanceSummary,
|
||||
ScenarioConfig,
|
||||
@@ -44,24 +43,21 @@ from sglang.multimodal_gen.test.test_utils import (
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@pytest.fixture(params=DIFFUSION_CASES, ids=lambda c: c.id)
|
||||
def case(request) -> DiffusionTestCase:
|
||||
"""Provide a DiffusionTestCase for each test."""
|
||||
return request.param
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def diffusion_server(case: DiffusionTestCase) -> ServerContext:
|
||||
"""Start a diffusion server for a single case and tear it down afterwards."""
|
||||
default_port = get_dynamic_server_port()
|
||||
port = int(os.environ.get("SGLANG_TEST_SERVER_PORT", default_port))
|
||||
|
||||
extra_args = os.environ.get("SGLANG_TEST_SERVE_ARGS", "")
|
||||
extra_args += f" --num-gpus {case.num_gpus} --ulysses-degree {case.num_gpus}"
|
||||
|
||||
# start server
|
||||
manager = ServerManager(
|
||||
model=case.model_path,
|
||||
port=port,
|
||||
wait_deadline=float(os.environ.get("SGLANG_TEST_WAIT_SECS", "1200")),
|
||||
extra_args=os.environ.get("SGLANG_TEST_SERVE_ARGS", ""),
|
||||
extra_args=extra_args,
|
||||
)
|
||||
ctx = manager.start()
|
||||
|
||||
@@ -98,10 +94,10 @@ def diffusion_server(case: DiffusionTestCase) -> ServerContext:
|
||||
ctx.cleanup()
|
||||
|
||||
|
||||
class TestDiffusionPerformance:
|
||||
class DiffusionPerformanceBase:
|
||||
"""Performance tests for all diffusion models/scenarios.
|
||||
|
||||
This single test class runs against all cases defined in DIFFUSION_CASES.
|
||||
This single test class runs against all cases defined in ONE_GPU_CASES.
|
||||
Each case gets its own server instance via the parametrized fixture.
|
||||
"""
|
||||
|
||||
@@ -148,7 +144,7 @@ Consider updating perf_baselines.json with the snippets below:
|
||||
"""Run generation and collect performance records."""
|
||||
log_path = ctx.perf_log_path
|
||||
prev_len = len(read_perf_logs(log_path))
|
||||
log_wait_timeout = 1200
|
||||
log_wait_timeout = 30
|
||||
|
||||
rid = generate_fn()
|
||||
|
||||
@@ -200,7 +196,7 @@ Consider updating perf_baselines.json with the snippets below:
|
||||
is_baseline_generation_mode = (
|
||||
os.environ.get("SGLANG_GEN_BASELINE", "0") == "1"
|
||||
)
|
||||
timeout = 3600.0 if is_baseline_generation_mode else 600.0
|
||||
timeout = 3600.0 if is_baseline_generation_mode else 1200.0
|
||||
deadline = time.time() + timeout
|
||||
while True:
|
||||
page = client.videos.list() # type: ignore[attr-defined]
|
||||
@@ -395,7 +391,7 @@ Consider updating perf_baselines.json with the snippets below:
|
||||
summary = validator.collect_metrics(perf_record)
|
||||
|
||||
if is_baseline_generation_mode or missing_scenario:
|
||||
self._dump_baseline_for_testcase(case, summary)
|
||||
self._dump_baseline_for_testcase(case, summary, missing_scenario)
|
||||
if missing_scenario:
|
||||
pytest.fail(f"Testcase '{case.id}' not found in perf_baselines.json")
|
||||
return
|
||||
@@ -406,7 +402,7 @@ Consider updating perf_baselines.json with the snippets below:
|
||||
validator.validate(perf_record, case.num_frames)
|
||||
except AssertionError as e:
|
||||
logger.error(f"Performance validation failed for {case.id}:\n{e}")
|
||||
self._dump_baseline_for_testcase(case, summary)
|
||||
self._dump_baseline_for_testcase(case, summary, missing_scenario)
|
||||
raise
|
||||
|
||||
result = {
|
||||
@@ -510,7 +506,10 @@ Consider updating perf_baselines.json with the snippets below:
|
||||
self._improved_baselines.append({"id": case.id, "baseline": new_baseline})
|
||||
|
||||
def _dump_baseline_for_testcase(
|
||||
self, case: DiffusionTestCase, summary: "PerformanceSummary"
|
||||
self,
|
||||
case: DiffusionTestCase,
|
||||
summary: "PerformanceSummary",
|
||||
missing_scenario: bool = False,
|
||||
) -> None:
|
||||
"""Dump performance metrics as a JSON scenario for baselines."""
|
||||
import json
|
||||
@@ -536,10 +535,9 @@ Consider updating perf_baselines.json with the snippets below:
|
||||
if summary.avg_frame_time_ms
|
||||
else None
|
||||
)
|
||||
|
||||
action = "add" if missing_scenario else "update"
|
||||
output = f"""
|
||||
To add this baseline, copy the following JSON snippet into
|
||||
the "scenarios" section of perf_baselines.json:
|
||||
{action} this baseline in the "scenarios" section of perf_baselines.json:
|
||||
|
||||
"{case.id}": {json.dumps(baseline, indent=4)}
|
||||
|
||||
@@ -553,7 +551,7 @@ the "scenarios" section of perf_baselines.json:
|
||||
):
|
||||
"""Single parametrized test that runs for all cases.
|
||||
|
||||
Pytest will execute this test once per case in DIFFUSION_CASES,
|
||||
Pytest will execute this test once per case in ONE_GPU_CASES,
|
||||
with test IDs like:
|
||||
- test_diffusion_perf[qwen_image_text]
|
||||
- test_diffusion_perf[qwen_image_edit]
|
||||
@@ -5,6 +5,7 @@ Server management and performance validation for diffusion tests.
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import shlex
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
@@ -130,7 +131,9 @@ class ServerManager:
|
||||
env["SGLANG_DIFFUSION_STAGE_LOGGING"] = "1"
|
||||
env["SGLANG_PERF_LOG_DIR"] = log_dir.as_posix()
|
||||
|
||||
stdout_fh = stdout_path.open("w", encoding="utf-8", buffering=1)
|
||||
# TODO: unify with run_command
|
||||
print(f"Running command: {shlex.join(command)}")
|
||||
|
||||
process = subprocess.Popen(
|
||||
command,
|
||||
stdout=subprocess.PIPE,
|
||||
@@ -141,6 +144,7 @@ class ServerManager:
|
||||
)
|
||||
|
||||
log_thread = None
|
||||
stdout_fh = stdout_path.open("w", encoding="utf-8", buffering=1)
|
||||
if process.stdout:
|
||||
|
||||
def _log_pipe(pipe: Any, file: Any) -> None:
|
||||
|
||||
@@ -122,7 +122,7 @@ class DiffusionTestCase:
|
||||
image_path: Path | str | None = None # input image/video for editing (Path or URL)
|
||||
|
||||
# duration
|
||||
seconds: int = 4 # for video: duration in seconds
|
||||
seconds: int = 1 # for video: duration in seconds
|
||||
num_frames: int | None = None # for video: number of frames
|
||||
fps: int | None = None # for video: frames per second
|
||||
|
||||
@@ -130,6 +130,9 @@ class DiffusionTestCase:
|
||||
warmup_edit: int = 0 # number of image/video-edit warmups
|
||||
custom_validator: str | None = None # optional custom validator name
|
||||
|
||||
# resources
|
||||
num_gpus: int = 1
|
||||
|
||||
def is_image_url(self) -> bool:
|
||||
"""Check if image_edit_path is a URL."""
|
||||
if self.image_path is None:
|
||||
@@ -206,26 +209,9 @@ class PerformanceSummary:
|
||||
)
|
||||
|
||||
|
||||
# Common paths
|
||||
IMAGE_INPUT_FILE = Path(__file__).resolve().parents[1] / "test_files" / "girl.jpg"
|
||||
|
||||
# All test cases with clean default values
|
||||
# To test different models, simply add more DiffusionCase entries
|
||||
DIFFUSION_CASES: list[DiffusionTestCase] = [
|
||||
# === Image to Video (I2V) ===
|
||||
DiffusionTestCase(
|
||||
id="wan2_2_i2v_a14b",
|
||||
model_path="Wan-AI/Wan2.2-I2V-A14B-Diffusers",
|
||||
modality="video",
|
||||
prompt="generate", # passing in something since failing if no prompt is passed
|
||||
warmup_text=0, # warmups only for image gen models
|
||||
warmup_edit=0,
|
||||
output_size="832x1104",
|
||||
edit_prompt="generate",
|
||||
image_path="https://github.com/Wan-Video/Wan2.2/blob/990af50de458c19590c245151197326e208d7191/examples/i2v_input.JPG?raw=true",
|
||||
custom_validator="video",
|
||||
seconds=1,
|
||||
),
|
||||
ONE_GPU_CASES_A: list[DiffusionTestCase] = [
|
||||
# === Text to Image (T2I) ===
|
||||
DiffusionTestCase(
|
||||
id="qwen_image_t2i",
|
||||
@@ -257,46 +243,43 @@ DIFFUSION_CASES: list[DiffusionTestCase] = [
|
||||
edit_prompt="Convert 2D style to 3D style",
|
||||
image_path="https://github.com/lm-sys/lm-sys.github.io/releases/download/test/TI2I_Qwen_Image_Edit_Input.jpg",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
ONE_GPU_CASES_B: list[DiffusionTestCase] = [
|
||||
# === Text to Video (T2V) ===
|
||||
# TODO: FastWan2.1, FastWan2.2
|
||||
DiffusionTestCase(
|
||||
id="wan2_1_t2v_1.3b",
|
||||
model_path="Wan-AI/Wan2.1-T2V-1.3B-Diffusers",
|
||||
modality="video",
|
||||
prompt="A curious raccoon",
|
||||
output_size="848x480",
|
||||
seconds=4,
|
||||
warmup_text=0, # warmups only for image gen models
|
||||
warmup_text=0,
|
||||
warmup_edit=0,
|
||||
custom_validator="video",
|
||||
),
|
||||
# NOTE(mick): flaky
|
||||
# DiffusionTestCase(
|
||||
# id="hunyuan_video",
|
||||
# model_path="hunyuanvideo-community/HunyuanVideo",
|
||||
# modality="video",
|
||||
# prompt="A curious raccoon",
|
||||
# output_size="720x480",
|
||||
# warmup_text=0,
|
||||
# warmup_edit=0,
|
||||
# custom_validator="video",
|
||||
# ),
|
||||
DiffusionTestCase(
|
||||
id="fast_hunyuan_video",
|
||||
model_path="FastVideo/FastHunyuan-diffusers",
|
||||
modality="video",
|
||||
prompt="A curious raccoon",
|
||||
output_size="720x480",
|
||||
warmup_text=0,
|
||||
warmup_edit=0,
|
||||
custom_validator="video",
|
||||
),
|
||||
# === Text and Image to Video (TI2V) ===
|
||||
DiffusionTestCase(
|
||||
id="wan2_1_i2v_14b_480P",
|
||||
model_path="Wan-AI/Wan2.1-I2V-14B-480P-Diffusers",
|
||||
output_size="832x1104",
|
||||
modality="video",
|
||||
prompt="Animate this image",
|
||||
edit_prompt="Add dynamic motion to the scene",
|
||||
image_path="https://github.com/lm-sys/lm-sys.github.io/releases/download/test/TI2I_Qwen_Image_Edit_Input.jpg",
|
||||
warmup_text=0, # warmups only for image gen models
|
||||
warmup_edit=0,
|
||||
custom_validator="video",
|
||||
seconds=1,
|
||||
),
|
||||
DiffusionTestCase(
|
||||
id="wan2_1_i2v_14b_720P",
|
||||
model_path="Wan-AI/Wan2.1-I2V-14B-720P-Diffusers",
|
||||
modality="video",
|
||||
prompt="Animate this image",
|
||||
edit_prompt="Add dynamic motion to the scene",
|
||||
image_path="https://github.com/lm-sys/lm-sys.github.io/releases/download/test/TI2I_Qwen_Image_Edit_Input.jpg",
|
||||
output_size="832x1104",
|
||||
warmup_text=0, # warmups only for image gen models
|
||||
warmup_edit=0,
|
||||
custom_validator="video",
|
||||
seconds=1,
|
||||
),
|
||||
DiffusionTestCase(
|
||||
id="wan2_2_ti2v_5b",
|
||||
model_path="Wan-AI/Wan2.2-TI2V-5B-Diffusers",
|
||||
@@ -305,10 +288,86 @@ DIFFUSION_CASES: list[DiffusionTestCase] = [
|
||||
prompt="Animate this image",
|
||||
edit_prompt="Add dynamic motion to the scene",
|
||||
image_path="https://github.com/lm-sys/lm-sys.github.io/releases/download/test/TI2I_Qwen_Image_Edit_Input.jpg",
|
||||
warmup_text=0, # warmups only for image gen models
|
||||
warmup_text=0,
|
||||
warmup_edit=0,
|
||||
custom_validator="video",
|
||||
seconds=1,
|
||||
),
|
||||
DiffusionTestCase(
|
||||
id="fastwan2_2_ti2v_5b",
|
||||
model_path="FastVideo/FastWan2.2-TI2V-5B-FullAttn-Diffusers",
|
||||
modality="video",
|
||||
output_size="832x1104",
|
||||
prompt="Animate this image",
|
||||
edit_prompt="Add dynamic motion to the scene",
|
||||
image_path="https://github.com/lm-sys/lm-sys.github.io/releases/download/test/TI2I_Qwen_Image_Edit_Input.jpg",
|
||||
warmup_text=0,
|
||||
warmup_edit=0,
|
||||
custom_validator="video",
|
||||
),
|
||||
]
|
||||
|
||||
TWO_GPU_CASES = [
|
||||
DiffusionTestCase(
|
||||
id="wan2_2_i2v_a14b_2gpu",
|
||||
model_path="Wan-AI/Wan2.2-I2V-A14B-Diffusers",
|
||||
modality="video",
|
||||
prompt="generate",
|
||||
warmup_text=0,
|
||||
warmup_edit=0,
|
||||
output_size="832x1104",
|
||||
edit_prompt="generate",
|
||||
image_path="https://github.com/Wan-Video/Wan2.2/blob/990af50de458c19590c245151197326e208d7191/examples/i2v_input.JPG?raw=true",
|
||||
custom_validator="video",
|
||||
num_gpus=2,
|
||||
num_frames=1,
|
||||
),
|
||||
DiffusionTestCase(
|
||||
id="wan2_2_t2v_a14b_2gpu",
|
||||
model_path="Wan-AI/Wan2.2-T2V-A14B-Diffusers",
|
||||
modality="video",
|
||||
prompt="A curious raccoon",
|
||||
output_size="720x480",
|
||||
warmup_text=0,
|
||||
warmup_edit=0,
|
||||
custom_validator="video",
|
||||
num_gpus=2,
|
||||
),
|
||||
DiffusionTestCase(
|
||||
id="wan2_1_t2v_14b_2gpu",
|
||||
model_path="Wan-AI/Wan2.1-T2V-14B-Diffusers",
|
||||
modality="video",
|
||||
prompt="A curious raccoon",
|
||||
output_size="720x480",
|
||||
warmup_text=0,
|
||||
warmup_edit=0,
|
||||
custom_validator="video",
|
||||
num_gpus=2,
|
||||
),
|
||||
DiffusionTestCase(
|
||||
id="wan2_1_i2v_14b_480P_2gpu",
|
||||
model_path="Wan-AI/Wan2.1-I2V-14B-480P-Diffusers",
|
||||
output_size="832x1104",
|
||||
modality="video",
|
||||
prompt="Animate this image",
|
||||
edit_prompt="Add dynamic motion to the scene",
|
||||
image_path="https://github.com/lm-sys/lm-sys.github.io/releases/download/test/TI2I_Qwen_Image_Edit_Input.jpg",
|
||||
warmup_text=0,
|
||||
warmup_edit=0,
|
||||
custom_validator="video",
|
||||
num_gpus=2,
|
||||
),
|
||||
DiffusionTestCase(
|
||||
id="wan2_1_i2v_14b_720P_2gpu",
|
||||
model_path="Wan-AI/Wan2.1-I2V-14B-720P-Diffusers",
|
||||
modality="video",
|
||||
prompt="Animate this image",
|
||||
edit_prompt="Add dynamic motion to the scene",
|
||||
image_path="https://github.com/lm-sys/lm-sys.github.io/releases/download/test/TI2I_Qwen_Image_Edit_Input.jpg",
|
||||
output_size="832x1104",
|
||||
warmup_text=0,
|
||||
warmup_edit=0,
|
||||
custom_validator="video",
|
||||
num_gpus=2,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 212 KiB |
@@ -164,7 +164,7 @@ def wait_for_req_perf_record(
|
||||
request_id: str,
|
||||
prev_len: int,
|
||||
log_path: Path,
|
||||
timeout: float = 300.0,
|
||||
timeout: float = 30.0,
|
||||
) -> tuple[RequestPerfRecord | None, int]:
|
||||
"""
|
||||
the stage metrics of this request should be in the performance_log file with {request-id}
|
||||
@@ -173,7 +173,7 @@ def wait_for_req_perf_record(
|
||||
deadline = time.time() + timeout
|
||||
while time.time() < deadline:
|
||||
records = read_perf_logs(log_path)
|
||||
if len(records) == prev_len + 1:
|
||||
if len(records) >= prev_len + 1:
|
||||
# FIXME: unable to get rid from openai apis, this is a hack. we should compare rid
|
||||
# potential error when there are multiple servers
|
||||
return records[-1], len(records)
|
||||
@@ -183,6 +183,8 @@ def wait_for_req_perf_record(
|
||||
if os.environ.get("SGLANG_GEN_BASELINE", "0") == "1":
|
||||
records = read_perf_logs(log_path)
|
||||
return None, len(records)
|
||||
|
||||
logger.error(f"record: {records}")
|
||||
raise AssertionError(f"Timeout waiting for stage metrics for request {request_id} ")
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user