From d8736c756a2ebba0f23fbaca051ff5ab299dfd9f Mon Sep 17 00:00:00 2001 From: Yuhao Yang <47235274+yhyang201@users.noreply.github.com> Date: Fri, 7 Nov 2025 13:19:49 +0800 Subject: [PATCH] fix multimodal gen issues (#12765) Co-authored-by: Mick --- python/sglang/multimodal_gen/README.md | 2 +- python/sglang/multimodal_gen/docs/cli.md | 2 +- python/sglang/multimodal_gen/docs/install.md | 10 +-- .../runtime/entrypoints/openai/video_api.py | 2 +- .../multimodal_gen/runtime/server_args.py | 18 ++++-- .../multimodal_gen/test/cli/test_serve.py | 64 +++++++++++-------- 6 files changed, 56 insertions(+), 42 deletions(-) diff --git a/python/sglang/multimodal_gen/README.md b/python/sglang/multimodal_gen/README.md index d45337491..96880bd33 100644 --- a/python/sglang/multimodal_gen/README.md +++ b/python/sglang/multimodal_gen/README.md @@ -24,7 +24,7 @@ sgl-diffusion has the following features: ## Getting Started ```bash -uv pip install sglang[.diffusion] --prerelease=allow +uv pip install 'sglang[diffusion]' --prerelease=allow ``` For more information, check the [docs](https://github.com/sgl-project/sglang/tree/main/python/sglang/multimodal_gen/docs/install.md). diff --git a/python/sglang/multimodal_gen/docs/cli.md b/python/sglang/multimodal_gen/docs/cli.md index 0b28f8ad6..86a9643d3 100644 --- a/python/sglang/multimodal_gen/docs/cli.md +++ b/python/sglang/multimodal_gen/docs/cli.md @@ -143,7 +143,7 @@ SERVER_ARGS=( --ring-degree=2 ) -sglang serve"${SERVER_ARGS[@]}" +sglang serve "${SERVER_ARGS[@]}" ``` - **--model-path**: Which model to load. The example uses `Wan-AI/Wan2.1-T2V-1.3B-Diffusers`. diff --git a/python/sglang/multimodal_gen/docs/install.md b/python/sglang/multimodal_gen/docs/install.md index 8b8321533..46aaec4f7 100644 --- a/python/sglang/multimodal_gen/docs/install.md +++ b/python/sglang/multimodal_gen/docs/install.md @@ -11,7 +11,7 @@ It is recommended to use uv for a faster installation: ```bash pip install --upgrade pip pip install uv -uv pip install sglang[.diffusion] --prerelease=allow +uv pip install 'sglang[diffusion]' --prerelease=allow ``` ## Method 2: From source @@ -29,13 +29,9 @@ pip install -e "python/.[diffusion]" uv pip install --prerelease=allow -e "python/.[diffusion]" ``` -**Quick fixes for common problems:** - -- If you want to develop sgl-diffusion, it is recommended to use Docker. The Docker image is `lmsysorg/sgl-diffusion:latest`. - ## Method 3: Using Docker -The Docker images are available on Docker Hub at [lmsysorg/sgl-diffusion](), built from the [Dockerfile](https://github.com/sgl-project/sgl-diffusion/tree/main/docker). +The Docker images are available on Docker Hub at [lmsysorg/sglang](), built from the [Dockerfile](https://github.com/sgl-project/sglang/tree/main/docker). Replace `` below with your HuggingFace Hub [token](https://huggingface.co/docs/hub/en/security-tokens). ```bash @@ -45,7 +41,7 @@ docker run --gpus all \ -v ~/.cache/huggingface:/root/.cache/huggingface \ --env "HF_TOKEN=" \ --ipc=host \ - lmsysorg/sglang:diffusion \ + lmsysorg/sglang:dev \ sglang generate --model-path black-forest-labs/FLUX.1-dev \ --prompt "A logo With Bold Large text: SGL Diffusion" \ --save-output diff --git a/python/sglang/multimodal_gen/runtime/entrypoints/openai/video_api.py b/python/sglang/multimodal_gen/runtime/entrypoints/openai/video_api.py index c6bf59235..0d5b8353a 100644 --- a/python/sglang/multimodal_gen/runtime/entrypoints/openai/video_api.py +++ b/python/sglang/multimodal_gen/runtime/entrypoints/openai/video_api.py @@ -69,8 +69,8 @@ def _build_sampling_params_from_request( save_output=True, ) sampling_params = sampling_params.from_user_sampling_params(user_params) + sampling_params.set_output_file_name() sampling_params.log(server_args) - sampling_params.set_output_file_ext() return sampling_params diff --git a/python/sglang/multimodal_gen/runtime/server_args.py b/python/sglang/multimodal_gen/runtime/server_args.py index 1d722c9de..95a7f12d0 100644 --- a/python/sglang/multimodal_gen/runtime/server_args.py +++ b/python/sglang/multimodal_gen/runtime/server_args.py @@ -777,13 +777,6 @@ class ServerArgs: ) self.sp_degree = self.ulysses_degree = self.ring_degree = 1 - if ( - self.ring_degree is not None - and self.ring_degree > 1 - and self.attention_backend != "fa3" - ): - raise ValueError("Ring Attention is only supported for fa3 backend for now") - if self.sp_degree == -1: # assume we leave all remaining gpus to sp num_gpus_per_group = self.dp_size * self.tp_size @@ -815,6 +808,17 @@ class ServerArgs: f"Ring degree not set, " f"using default value {self.ring_degree}" ) + if self.ring_degree > 1: + if self.attention_backend != None and self.attention_backend != "fa3": + raise ValueError( + "Ring Attention is only supported for fa3 backend for now" + ) + else: + self.attention_backend = "fa3" + logger.info( + "Ring Attention is currently only supported for fa3, attention_backend has been automatically set to fa3" + ) + if self.sp_degree == -1: self.sp_degree = self.ring_degree * self.ulysses_degree logger.info( diff --git a/python/sglang/multimodal_gen/test/cli/test_serve.py b/python/sglang/multimodal_gen/test/cli/test_serve.py index fe39d7fff..156cf2157 100644 --- a/python/sglang/multimodal_gen/test/cli/test_serve.py +++ b/python/sglang/multimodal_gen/test/cli/test_serve.py @@ -3,9 +3,13 @@ import asyncio import base64 import subprocess +import tempfile import time import unittest +import uuid +from contextlib import contextmanager from pathlib import Path +from urllib.request import urlopen from openai import OpenAI @@ -13,6 +17,20 @@ from sglang.multimodal_gen.runtime.utils.common import kill_process_tree from sglang.multimodal_gen.test.test_utils import is_mp4, is_png, wait_for_port +@contextmanager +def downloaded_temp_file(url: str, prefix: str = "i2v_input_", suffix: str = ".jpg"): + tmp_path = Path(tempfile.gettempdir()) / f"{prefix}{uuid.uuid4().hex}{suffix}" + with urlopen(url) as resp: + tmp_path.write_bytes(resp.read()) + try: + yield tmp_path + finally: + try: + tmp_path.unlink(missing_ok=True) + except Exception: + pass + + def wait_for_video_completion(client, video_id, timeout=300, check_interval=3): start = time.time() video = client.videos.retrieve(video_id) @@ -27,7 +45,7 @@ def wait_for_video_completion(client, video_id, timeout=300, check_interval=3): class TestVideoHttpServer(unittest.TestCase): model_name = "Wan-AI/Wan2.1-T2V-1.3B-Diffusers" - timeout = 120 + timeout = 500 extra_args = [] def _create_wait_and_download( @@ -77,7 +95,7 @@ class TestVideoHttpServer(unittest.TestCase): api_key="sk-proj-1234567890", base_url="http://localhost:30010/v1" ) content = self._create_wait_and_download( - client, "A calico cat playing a piano on stage", "832x480" + client, "A plane is taking off.", "832x480" ) self.assertTrue(is_mp4(content)) @@ -97,7 +115,7 @@ class TestVideoHttpServer(unittest.TestCase): async def send_concurrent_requests(): tasks = [ generate_and_check_video( - "A dog playing a piano on stage", + "A ship is beside the port.", "832x480", ) for _ in range(num_requests) @@ -107,14 +125,6 @@ class TestVideoHttpServer(unittest.TestCase): asyncio.run(send_concurrent_requests()) -class TestFastWan2_1HttpServer(TestVideoHttpServer): - model_name = "FastVideo/FastWan2.1-T2V-1.3B-Diffusers" - - -class TestFastWan2_2HttpServer(TestVideoHttpServer): - model_name = "FastVideo/FastWan2.2-TI2V-5B-FullAttn-Diffusers" - - class TestImage2VideoHttpServer(unittest.TestCase): model_name = "Wan-AI/Wan2.2-I2V-A14B-Diffusers" timeout = 1200 @@ -124,15 +134,17 @@ class TestImage2VideoHttpServer(unittest.TestCase): self, client: OpenAI, prompt: str, size: str ) -> bytes: - image_path = "https://github.com/Wan-Video/Wan2.2/blob/990af50de458c19590c245151197326e208d7191/examples/i2v_input.JPG?raw=true" - image_path = Path(image_path) - video = client.videos.create( - prompt=prompt, - input_reference=image_path, - size=size, - seconds=10, - extra_body={"fps": 16, "num_frames": 125}, - ) + image_url = "https://github.com/Wan-Video/Wan2.2/blob/990af50de458c19590c245151197326e208d7191/examples/i2v_input.JPG?raw=true" + with downloaded_temp_file( + image_url, prefix="i2v_input_", suffix=".jpg" + ) as tmp_path: + video = client.videos.create( + prompt=prompt, + input_reference=tmp_path, + size=size, + seconds=10, + extra_body={"fps": 16, "num_frames": 125}, + ) # TODO: Some combinations of num_frames and fps may cause errors and need further investigation. video_id = video.id self.assertEqual(video.status, "queued") @@ -149,18 +161,20 @@ class TestImage2VideoHttpServer(unittest.TestCase): @classmethod def setUpClass(cls): cls.base_command = [ - "sgl-diffusion", + "sglang", "serve", "--model-path", f"{cls.model_name}", + "--num-gpus", + "4", + "--ulysses-degree", + "4", "--port", "30010", ] process = subprocess.Popen( cls.base_command + cls.extra_args, - # stdout=subprocess.PIPE, - # stderr=subprocess.PIPE, text=True, bufsize=1, ) @@ -176,7 +190,7 @@ class TestImage2VideoHttpServer(unittest.TestCase): api_key="sk-proj-1234567890", base_url="http://localhost:30010/v1" ) content = self._create_wait_and_download( - client, "A girl is fighting a monster.", "832x480" + client, "A cat surfing on the sea.", "832x480" ) self.assertTrue(is_mp4(content)) @@ -196,7 +210,7 @@ class TestImage2VideoHttpServer(unittest.TestCase): async def send_concurrent_requests(): tasks = [ generate_and_check_video( - "A dog playing a piano on stage", + "A cat surfing on the sea.", "832x480", ) for _ in range(num_requests)