From 5f3eb377e0e8bd101c10928c4f73eb36228e6f43 Mon Sep 17 00:00:00 2001 From: Yuan Luo Date: Sun, 4 Jan 2026 13:04:43 +0800 Subject: [PATCH] [VLM] Support request level max_dynamic_patch for OpenAI request (#16268) Co-authored-by: luoyuan.luo --- .../sglang/srt/entrypoints/openai/protocol.py | 8 +++++ .../srt/entrypoints/openai/serving_chat.py | 32 +++++++++++++++++ python/sglang/srt/managers/io_struct.py | 6 ++++ .../srt/multimodal/processors/internvl.py | 36 ++++++++++++++++--- .../sglang/srt/parser/jinja_template_utils.py | 22 ++++++++++-- python/sglang/srt/utils/common.py | 1 + 6 files changed, 98 insertions(+), 7 deletions(-) diff --git a/python/sglang/srt/entrypoints/openai/protocol.py b/python/sglang/srt/entrypoints/openai/protocol.py index 34aa364cf..14cc9676b 100644 --- a/python/sglang/srt/entrypoints/openai/protocol.py +++ b/python/sglang/srt/entrypoints/openai/protocol.py @@ -339,10 +339,14 @@ class ChatCompletionMessageContentTextPart(BaseModel): class ChatCompletionMessageContentImageURL(BaseModel): url: str detail: Optional[Literal["auto", "low", "high"]] = "auto" + max_dynamic_patch: Optional[int] = None + min_dynamic_patch: Optional[int] = None class ChatCompletionMessageContentVideoURL(BaseModel): url: str + max_dynamic_patch: Optional[int] = None + min_dynamic_patch: Optional[int] = None class ChatCompletionMessageContentAudioURL(BaseModel): @@ -516,6 +520,10 @@ class ChatCompletionRequest(BaseModel): stream_reasoning: bool = True chat_template_kwargs: Optional[Dict] = None + # SGLang multimodal tiling controls (extensions) + max_dynamic_patch: Optional[int] = None + min_dynamic_patch: Optional[int] = None + # Custom logit processor for advanced sampling control custom_logit_processor: Optional[Union[List[Optional[str]], str]] = None custom_params: Optional[Dict] = None diff --git a/python/sglang/srt/entrypoints/openai/serving_chat.py b/python/sglang/srt/entrypoints/openai/serving_chat.py index cb0c084a3..dbdb4a6b6 100644 --- a/python/sglang/srt/entrypoints/openai/serving_chat.py +++ b/python/sglang/srt/entrypoints/openai/serving_chat.py @@ -55,6 +55,32 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) +def _extract_max_dynamic_patch(request: ChatCompletionRequest): + img_vals = [] + vid_vals = [] + for msg in request.messages or []: + content = getattr(msg, "content", None) + if not isinstance(content, list): + continue + for part in content: + # pydantic object or dict type + if getattr(part, "type", None) == "image_url": + iu = getattr(part, "image_url", None) + mdp = getattr(iu, "max_dynamic_patch", None) if iu else None + if mdp is not None: + img_vals.append(int(mdp)) + elif getattr(part, "type", None) == "video_url": + vu = getattr(part, "video_url", None) + mdp = getattr(vu, "max_dynamic_patch", None) if vu else None + if mdp is not None: + vid_vals.append(int(mdp)) + + # TODO(yuan-luo): per-item max_dynamic_patch for both image and video + img_max_dynamic_patch = min(img_vals) if img_vals else None + vid_max_dynamic_patch = min(vid_vals) if vid_vals else None + return img_max_dynamic_patch, vid_max_dynamic_patch + + class OpenAIServingChat(OpenAIServingBase): """Handler for /v1/chat/completions requests""" @@ -195,6 +221,9 @@ class OpenAIServingChat(OpenAIServingBase): if first_adapter: self._validate_lora_enabled(first_adapter) + img_max_dynamic_patch, vid_max_dynamic_patch = _extract_max_dynamic_patch( + request + ) adapted_request = GenerateReqInput( **prompt_kwargs, image_data=processed_messages.image_data, @@ -219,6 +248,9 @@ class OpenAIServingChat(OpenAIServingBase): priority=request.priority, custom_labels=custom_labels, custom_logit_processor=request.custom_logit_processor, + image_max_dynamic_patch=img_max_dynamic_patch, + video_max_dynamic_patch=vid_max_dynamic_patch, + max_dynamic_patch=getattr(request, "max_dynamic_patch", None), ) return adapted_request, request diff --git a/python/sglang/srt/managers/io_struct.py b/python/sglang/srt/managers/io_struct.py index 879e1bfa6..2ecd8542f 100644 --- a/python/sglang/srt/managers/io_struct.py +++ b/python/sglang/srt/managers/io_struct.py @@ -262,6 +262,12 @@ class GenerateReqInput(BaseReq, APIServingTimingMixin): need_wait_for_image: Optional[bool] = None num_items_assigned: Optional[List] = None + # Multimodal tiling controls (extensions) + max_dynamic_patch: Optional[int] = None + min_dynamic_patch: Optional[int] = None + image_max_dynamic_patch: Optional[int] = None + video_max_dynamic_patch: Optional[int] = None + def contains_mm_input(self) -> bool: return ( has_valid_data(self.image_data) diff --git a/python/sglang/srt/multimodal/processors/internvl.py b/python/sglang/srt/multimodal/processors/internvl.py index 1349fd6dc..9b1cf3ffd 100644 --- a/python/sglang/srt/multimodal/processors/internvl.py +++ b/python/sglang/srt/multimodal/processors/internvl.py @@ -25,6 +25,7 @@ class InternVLProcessor(BaseMultimodalProcessor): IMAGENET_MEAN = [0.485, 0.456, 0.406] IMAGENET_STD = [0.229, 0.224, 0.225] + IMAGE_MAX_NUM = 12 DEFAULT_VIDEO_NUM_FRAMES = 32 VIDEO_MAX_NUM = 1 @@ -86,6 +87,11 @@ class InternVLProcessor(BaseMultimodalProcessor): else None ) + self.image_token_id = ( + tokenizer.convert_tokens_to_ids(self.IMG_CONTEXT) + if self.IMG_CONTEXT + else None + ) self.num_image_token = int( (image_size // patch_size) ** 2 * (hf_config.downsample_ratio**2) ) @@ -97,7 +103,7 @@ class InternVLProcessor(BaseMultimodalProcessor): # Offset token id use IMG_CONTEXT / VIDEO_CONTEXT self.mm_tokens = MultimodalSpecialTokens( image_token=self.IMAGE_PLACEHOLDER_TOKEN, - image_token_id=tokenizer.convert_tokens_to_ids(self.IMG_CONTEXT), + image_token_id=self.image_token_id, video_token=self.VIDEO_PLACEHOLDER_TOKEN, video_token_id=self.video_token_id, ).build(_image_processor) @@ -122,7 +128,9 @@ class InternVLProcessor(BaseMultimodalProcessor): ) @staticmethod - def dynamic_preprocess(tensor, image_size=448, max_num=12, use_thumbnail=False): + def dynamic_preprocess( + tensor, image_size=448, max_num=IMAGE_MAX_NUM, use_thumbnail=False + ): # Tensor: (C,H,W) float on GPU C, H, W = tensor.shape aspect_ratio = W / H @@ -264,6 +272,25 @@ class InternVLProcessor(BaseMultimodalProcessor): async def process_qwen_mm_data_async( self, image_data, input_text, request_obj, **kwargs ): + + img_max_num = ( + getattr(request_obj, "image_max_dynamic_patch", None) + or getattr(request_obj, "max_dynamic_patch", None) + or kwargs.get("image_max_dynamic_patch") + or kwargs.get("max_dynamic_patch") + or self.IMAGE_MAX_NUM + ) + img_max_num = max(1, int(img_max_num)) + + vid_max_num = ( + getattr(request_obj, "video_max_dynamic_patch", None) + or getattr(request_obj, "max_dynamic_patch", None) + or kwargs.get("video_max_dynamic_patch") + or kwargs.get("max_dynamic_patch") + or self.VIDEO_MAX_NUM + ) + vid_max_num = max(1, int(vid_max_num)) + # Qwen/Qwen3 branch: OpenAI-style placeholders /