Fix Diffusion Request Validation to allow missing input artifacts if the input only contains text (#16610)

python/sglang/multimodal_gen/runtime/entrypoints/openai/protocol.py

@@ -71,6 +71,7 @@ class VideoResponse(BaseModel):
 class VideoGenerationsRequest(BaseModel):
     prompt: str
     input_reference: Optional[str] = None
+    reference_url: Optional[str] = None
     model: Optional[str] = None
     seconds: Optional[int] = 4
     size: Optional[str] = ""

python/sglang/multimodal_gen/runtime/entrypoints/openai/video_api.py

@@ -178,27 +178,33 @@ async def create_video(
     content_type = request.headers.get("content-type", "").lower()
     request_id = generate_request_id()
 
+    server_args = get_global_server_args()
+    task_type = server_args.pipeline_config.task_type
+
     if "multipart/form-data" in content_type:
         if not prompt:
             raise HTTPException(status_code=400, detail="prompt is required")
-        if input_reference is None and reference_url is None:
+        # Validate image input based on model task type
+        image_list = merge_image_input_list(input_reference, reference_url)
+        if task_type.requires_image_input() and not image_list:
             raise HTTPException(
                 status_code=400,
-                detail="input_reference file or reference_url is required",
-            )
-        image_list = merge_image_input_list(input_reference, reference_url)
-        # Save first input image
-        image = image_list[0]
-        uploads_dir = os.path.join("outputs", "uploads")
-        os.makedirs(uploads_dir, exist_ok=True)
-        filename = image.filename if hasattr(image, "filename") else f"url_image"
-        input_path = os.path.join(uploads_dir, f"{request_id}_{filename}")
-        try:
-            input_path = await save_image_to_path(image, input_path)
-        except Exception as e:
-            raise HTTPException(
-                status_code=400, detail=f"Failed to process image source: {str(e)}"
+                detail="input_reference or reference_url is required for image-to-video generation",
             )
+        input_path = None
+        if image_list:
+            # Save first input image for image-to-video generation
+            image = image_list[0]
+            uploads_dir = os.path.join("outputs", "uploads")
+            os.makedirs(uploads_dir, exist_ok=True)
+            filename = image.filename if hasattr(image, "filename") else "url_image"
+            input_path = os.path.join(uploads_dir, f"{request_id}_{filename}")
+            try:
+                input_path = await save_image_to_path(image, input_path)
+            except Exception as e:
+                raise HTTPException(
+                    status_code=400, detail=f"Failed to process image source: {str(e)}"
+                )
 
         # Parse extra_body JSON (if provided in multipart form) to get fps/num_frames overrides
         extra_from_form: Dict[str, Any] = {}
@@ -246,6 +252,15 @@ async def create_video(
             extra_json = payload.pop("extra_json", None)
             if isinstance(extra_json, dict):
                 payload.update(extra_json)
+            # Validate image input based on model task type
+            has_image_input = payload.get("reference_url") or payload.get(
+                "input_reference"
+            )
+            if task_type.requires_image_input() and not has_image_input:
+                raise HTTPException(
+                    status_code=400,
+                    detail="input_reference or reference_url is required for image-to-video generation",
+                )
             # for not multipart/form-data type
             if payload.get("reference_url"):
                 image_list = merge_image_input_list(payload.get("reference_url"))
@@ -253,9 +268,7 @@ async def create_video(
                 image = image_list[0]
                 uploads_dir = os.path.join("outputs", "uploads")
                 os.makedirs(uploads_dir, exist_ok=True)
-                filename = (
-                    image.filename if hasattr(image, "filename") else f"url_image"
-                )
+                filename = image.filename if hasattr(image, "filename") else "url_image"
                 input_path = os.path.join(uploads_dir, f"{request_id}_{filename}")
                 try:
                     input_path = await save_image_to_path(image, input_path)
@@ -277,7 +290,7 @@ async def create_video(
 
     # Build Req for scheduler
     batch = prepare_request(
-        server_args=get_global_server_args(),
+        server_args=server_args,
         sampling_params=sampling_params,
     )
     # Add diffusers_kwargs if provided