fix: duplicate resize images logic of qwen-vl series models (#12458)
Signed-off-by: yangsijia.614 <yangsijia.614@bytedance.com>
This commit is contained in:
@@ -1,16 +1,12 @@
|
||||
import asyncio
|
||||
import re
|
||||
from typing import Dict, List, Union
|
||||
|
||||
from PIL import Image
|
||||
|
||||
from sglang.srt.models.dots_ocr import DotsOCRForCausalLM
|
||||
from sglang.srt.models.dots_vlm import DotsVLMForCausalLM
|
||||
from sglang.srt.multimodal.processors.base_processor import (
|
||||
BaseMultimodalProcessor,
|
||||
MultimodalSpecialTokens,
|
||||
)
|
||||
from sglang.srt.multimodal.processors.qwen_vl import resize_image_async
|
||||
|
||||
|
||||
class DotsVLMImageProcessor(BaseMultimodalProcessor):
|
||||
@@ -70,18 +66,6 @@ class DotsVLMImageProcessor(BaseMultimodalProcessor):
|
||||
multimodal_tokens=self.mm_tokens,
|
||||
)
|
||||
|
||||
# Qwen-specific: resize images if they are raw Image objects
|
||||
if base_output.images and isinstance(base_output.images[0], Image.Image):
|
||||
resize_tasks = [
|
||||
resize_image_async(
|
||||
image,
|
||||
min_pixels=self.MIN_PIXELS,
|
||||
max_pixels=self.MAX_PIXELS,
|
||||
size_factor=self.IMAGE_FACTOR,
|
||||
)
|
||||
for image in base_output.images
|
||||
]
|
||||
base_output.images = await asyncio.gather(*resize_tasks)
|
||||
combined_mm_item, input_ids, _ = self.process_and_combine_mm_data(
|
||||
base_output, self.mm_tokens
|
||||
)
|
||||
|
||||
@@ -1,15 +1,9 @@
|
||||
# Copy from qwen_vl.py, adapted for points-v15-chat
|
||||
|
||||
import asyncio
|
||||
from typing import List, Union
|
||||
|
||||
from PIL import Image
|
||||
|
||||
from sglang.srt.models.points_v15_chat import POINTSV15ChatModel
|
||||
from sglang.srt.multimodal.processors.qwen_vl import (
|
||||
QwenVLImageProcessor,
|
||||
resize_image_async,
|
||||
)
|
||||
from sglang.srt.multimodal.processors.qwen_vl import QwenVLImageProcessor
|
||||
|
||||
|
||||
class POINTSV15ChatProcessor(QwenVLImageProcessor):
|
||||
@@ -37,10 +31,6 @@ class POINTSV15ChatProcessor(QwenVLImageProcessor):
|
||||
multimodal_tokens=self.mm_tokens,
|
||||
)
|
||||
|
||||
if base_output.images and isinstance(base_output.images[0], Image.Image):
|
||||
resize_tasks = [resize_image_async(image) for image in base_output.images]
|
||||
base_output.images = await asyncio.gather(*resize_tasks)
|
||||
|
||||
mm_items, input_ids, _ = self.process_and_combine_mm_data(
|
||||
base_output, self.mm_tokens
|
||||
)
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
import asyncio
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
@@ -79,26 +78,6 @@ def smart_resize(
|
||||
return h_bar, w_bar
|
||||
|
||||
|
||||
def resize_image(
|
||||
image,
|
||||
min_pixels: int = MIN_PIXELS,
|
||||
max_pixels: int = MAX_PIXELS,
|
||||
size_factor: int = IMAGE_FACTOR,
|
||||
) -> Image.Image:
|
||||
width, height = image.size
|
||||
min_pixels = min_pixels
|
||||
max_pixels = max_pixels
|
||||
resized_height, resized_width = smart_resize(
|
||||
height,
|
||||
width,
|
||||
factor=size_factor,
|
||||
min_pixels=min_pixels,
|
||||
max_pixels=max_pixels,
|
||||
)
|
||||
image = image.resize((resized_width, resized_height), resample=RESIZE_RESAMPLE)
|
||||
return image
|
||||
|
||||
|
||||
def round_by_factor(number: int, factor: int) -> int:
|
||||
"""Returns the closest integer to 'number' that is divisible by 'factor'."""
|
||||
return round(number / factor) * factor
|
||||
@@ -114,15 +93,6 @@ def floor_by_factor(number: int, factor: int) -> int:
|
||||
return math.floor(number / factor) * factor
|
||||
|
||||
|
||||
async def resize_image_async(
|
||||
image,
|
||||
min_pixels: int = MIN_PIXELS,
|
||||
max_pixels: int = MAX_PIXELS,
|
||||
size_factor: int = IMAGE_FACTOR,
|
||||
):
|
||||
return resize_image(image, min_pixels, max_pixels, size_factor)
|
||||
|
||||
|
||||
def smart_nframes(
|
||||
ele: dict,
|
||||
total_frames: int,
|
||||
@@ -266,11 +236,6 @@ class QwenVLImageProcessor(SGLangBaseProcessor):
|
||||
self.audio_start_token_id = getattr(hf_config, "audio_start_token_id", None)
|
||||
self.audio_token_id = getattr(hf_config, "audio_token_id", None)
|
||||
|
||||
self.NUM_TOKEN_PER_FRAME = 770
|
||||
self.IMAGE_FACTOR = 28
|
||||
self.MIN_PIXELS = 4 * 28 * 28
|
||||
self.MAX_PIXELS = 16384 * 28 * 28
|
||||
self.MAX_RATIO = 200
|
||||
self.mm_tokens = MultimodalSpecialTokens(
|
||||
image_token="<|vision_start|><|image_pad|><|vision_end|>",
|
||||
image_token_id=hf_config.image_token_id,
|
||||
@@ -301,11 +266,6 @@ class QwenVLImageProcessor(SGLangBaseProcessor):
|
||||
load_time = time.perf_counter()
|
||||
rid = getattr(request_obj, "rid", "anonymous_rid")
|
||||
|
||||
# Qwen-specific: resize images if they are raw Image objects
|
||||
if base_output.images and isinstance(base_output.images[0], Image.Image):
|
||||
resize_tasks = [resize_image_async(image) for image in base_output.images]
|
||||
base_output.images = await asyncio.gather(*resize_tasks)
|
||||
|
||||
video_metadata = None
|
||||
if base_output.videos:
|
||||
videos_processed = [
|
||||
|
||||
Reference in New Issue
Block a user