chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 13 Packages Projects Releases Wiki Activity

[diffusion] Postprocess: implement frame interpolation using RIFE (#19384)

Browse Source
This commit is contained in:
Junhao Liu
2026-02-28 14:13:20 +08:00
committed by GitHub
parent b01b07aa16
commit 53c767d224
15 changed files with 745 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/multimodal_gen/test/server/perf_baselines.json
+69 -3
View File
@@ -1210,7 +1210,7 @@
},
"fastwan2_2_ti2v_5b": {
"stages_ms": {
"InputValidationStage": 300.00,
"InputValidationStage": 300.0,
"TextEncodingStage": 843.86,
"TimestepPreparationStage": 58.66,
"LatentPreparationStage": 28.55,
@@ -1303,7 +1303,7 @@
"39": 1599.78
},
"expected_e2e_ms": 123182.9887,
"expected_avg_denoise_ms": 2831.00,
"expected_avg_denoise_ms": 2831.0,
"expected_median_denoise_ms": 1600.09
},
"turbo_wan2_2_i2v_a14b_2gpu": {
@@ -1320,7 +1320,7 @@
"denoise_step_ms": {
"0": 3042.56,
"1": 485.88,
"2": 721.20,
"2": 721.2,
"3": 475.58
},
"expected_e2e_ms": 11605.97,
@@ -1919,6 +1919,72 @@
"expected_e2e_ms": 2103.05,
"expected_avg_denoise_ms": 173.83,
"expected_median_denoise_ms": 178.08
},
"wan2_1_t2v_1.3b_frame_interp_2x": {
"stages_ms": {
"InputValidationStage": 0.03,
"TextEncodingStage": 1155.78,
"LatentPreparationStage": 0.12,
"TimestepPreparationStage": 2.17,
"DenoisingStage": 4977.09,
"DecodingStage": 98.46,
"per_frame_generation": 1271.97
},
"denoise_step_ms": {
"0": 85.33,
"1": 76.41,
"2": 100.5,
"3": 99.28,
"4": 100.14,
"5": 100.33,
"6": 100.17,
"7": 100.35,
"8": 100.14,
"9": 100.32,
"10": 100.18,
"11": 100.37,
"12": 100.17,
"13": 100.34,
"14": 100.21,
"15": 100.35,
"16": 100.15,
"17": 100.32,
"18": 100.23,
"19": 100.35,
"20": 100.19,
"21": 100.57,
"22": 100.27,
"23": 100.24,
"24": 100.27,
"25": 100.27,
"26": 100.27,
"27": 100.29,
"28": 100.25,
"29": 100.25,
"30": 100.3,
"31": 100.3,
"32": 100.23,
"33": 100.25,
"34": 100.26,
"35": 100.27,
"36": 100.24,
"37": 100.26,
"38": 100.26,
"39": 100.29,
"40": 100.31,
"41": 100.31,
"42": 100.31,
"43": 100.26,
"44": 100.26,
"45": 100.27,
"46": 100.3,
"47": 100.27,
"48": 100.23,
"49": 99.65
},
"expected_e2e_ms": 6359.87,
"expected_avg_denoise_ms": 99.47,
"expected_median_denoise_ms": 100.27
}
}
}
python/sglang/multimodal_gen/test/server/test_server_utils.py
+24
View File
@@ -37,6 +37,7 @@ from sglang.multimodal_gen.test.server.testcase_configs import (
from sglang.multimodal_gen.test.slack_utils import upload_file_to_slack
from sglang.multimodal_gen.test.test_utils import (
get_expected_image_format,
get_video_frame_count,
is_image_url,
prepare_perf_log,
validate_image,
@@ -663,6 +664,7 @@ def get_generate_fn(
seconds: int | None = None,
input_reference: Any | None = None,
extra_body: dict[Any] | None = None,
expected_frame_count: int | None = None,
) -> str:
"""
Create a video job via /v1/videos, poll until completion,
@@ -746,6 +748,13 @@ def get_generate_fn(
tmp_path, expected_filename, expected_width, expected_height
)
if expected_frame_count is not None:
actual_count = get_video_frame_count(tmp_path)
assert actual_count == expected_frame_count, (
f"{case_id}: frame count mismatch after interpolation — "
f"expected {expected_frame_count}, got {actual_count}"
)
upload_file_to_slack(
case_id=case_id,
model=model_path,
@@ -976,6 +985,20 @@ def get_generate_fn(
extra_body = {}
if sampling_params.enable_teacache:
extra_body["enable_teacache"] = True
if sampling_params.num_frames:
extra_body["num_frames"] = sampling_params.num_frames
if sampling_params.enable_frame_interpolation:
extra_body["enable_frame_interpolation"] = True
extra_body["frame_interpolation_exp"] = (
sampling_params.frame_interpolation_exp
)
# Compute expected output frame count for validation
expected_frame_count = None
if sampling_params.enable_frame_interpolation and sampling_params.num_frames:
n = sampling_params.num_frames
exp = sampling_params.frame_interpolation_exp
expected_frame_count = (n - 1) * (2**exp) + 1
return _create_and_download_video(
client,
@@ -985,6 +1008,7 @@ def get_generate_fn(
size=output_size,
seconds=video_seconds,
extra_body=extra_body if extra_body else None,
expected_frame_count=expected_frame_count,
)
def generate_image_to_video(case_id, client) -> tuple[str, bytes]:
python/sglang/multimodal_gen/test/server/testcase_configs.py
+20
View File
@@ -218,6 +218,10 @@ class DiffusionSamplingParams:
# TeaCache acceleration
enable_teacache: bool = False
# Frame interpolation
enable_frame_interpolation: bool = False
frame_interpolation_exp: int = 1 # 1 = 2×, 2 = 4×
@dataclass(frozen=True)
class DiffusionTestCase:
@@ -494,6 +498,22 @@ ONE_GPU_CASES_B: list[DiffusionTestCase] = [
enable_teacache=True,
),
),
# Frame interpolation correctness (2× / exp=1)
# Uses the same 1.3B model already in the suite;
DiffusionTestCase(
"wan2_1_t2v_1.3b_frame_interp_2x",
DiffusionServerArgs(
model_path="Wan-AI/Wan2.1-T2V-1.3B-Diffusers",
modality="video",
custom_validator="video",
),
DiffusionSamplingParams(
prompt=T2V_PROMPT,
num_frames=5,
enable_frame_interpolation=True,
frame_interpolation_exp=1,
),
),
# LoRA test case for single transformer + merge/unmerge API test
# Note: Uses dynamic_lora_path instead of lora_path to test LayerwiseOffload + set_lora interaction
# Server starts WITHOUT LoRA, then set_lora is called after startup (Wan models auto-enable layerwise offload)
python/sglang/multimodal_gen/test/test_utils.py
+16
View File
@@ -337,6 +337,22 @@ def get_video_dimensions(file_path: str) -> tuple[int, int]:
cap.release()
def get_video_frame_count(file_path: str) -> int:
"""Return the number of frames in a video file using OpenCV."""
cap = cv2.VideoCapture(file_path)
try:
count = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
if count > 0:
return count
# Fallback: count frames manually
n = 0
while cap.read()[0]:
n += 1
return n
finally:
cap.release()
def validate_video_file(
file_path: str,
expected_filename: str,