diff --git a/python/sglang/srt/configs/model_config.py b/python/sglang/srt/configs/model_config.py index bdf7ca22a..b51d19d26 100644 --- a/python/sglang/srt/configs/model_config.py +++ b/python/sglang/srt/configs/model_config.py @@ -180,6 +180,14 @@ class ModelConfig: self.is_audio_model = enable_multimodal and is_audio_model( self.hf_config.architectures ) + # TODO: requires further polishing + self.is_image_understandable_model = enable_multimodal and hasattr( + self.hf_config, "vision_config" + ) + self.is_audio_understandable_model = enable_multimodal and hasattr( + self.hf_config, "audio_config" + ) + self.is_multimodal_chunked_prefill_supported = ( enable_multimodal and is_multimodal_chunked_prefill_supported(self.hf_config.architectures) diff --git a/python/sglang/srt/entrypoints/http_server.py b/python/sglang/srt/entrypoints/http_server.py index 289db654c..76e999498 100644 --- a/python/sglang/srt/entrypoints/http_server.py +++ b/python/sglang/srt/entrypoints/http_server.py @@ -483,6 +483,8 @@ async def get_model_info(): "is_generation": _global_state.tokenizer_manager.is_generation, "preferred_sampling_params": _global_state.tokenizer_manager.server_args.preferred_sampling_params, "weight_version": _global_state.tokenizer_manager.server_args.weight_version, + "has_image_understanding": _global_state.tokenizer_manager.model_config.is_image_understandable_model, + "has_audio_understanding": _global_state.tokenizer_manager.model_config.is_audio_understandable_model, } return result @@ -1429,6 +1431,10 @@ def launch_server( _global_state.tokenizer_manager.socket_mapping.clear_all_sockets() +# Minimal 2x2 black PNG (base64, 1x1 image would cause ambiguous in image channel dimension) +MINIMUM_PNG_PICTURE_BASE64 = "iVBORw0KGgoAAAANSUhEUgAAAAIAAAACCAYAAABytg0kAAAAEElEQVQImWNgYGD4z8DAAAQYAAH7u8WFAAAAAElFTkSuQmCC" + + def _execute_server_warmup( server_args: ServerArgs, pipe_finish_writer: Optional[multiprocessing.connection.Connection], @@ -1460,8 +1466,16 @@ def _execute_server_warmup( model_info = res.json() + is_vlm = bool(model_info.get("has_image_understanding", False)) + # Send a warmup request - request_name = "/generate" if model_info["is_generation"] else "/encode" + if model_info["is_generation"]: + if is_vlm and not server_args.skip_tokenizer_init: + request_name = "/v1/chat/completions" + else: + request_name = "/generate" + else: + request_name = "/encode" max_new_tokens = 8 if model_info["is_generation"] else 1 json_data = { "sampling_params": { @@ -1474,6 +1488,31 @@ def _execute_server_warmup( # TODO Workaround the bug that embedding errors for list of size 1 if server_args.dp_size == 1: json_data["input_ids"] = json_data["input_ids"][0] + elif is_vlm and server_args.disaggregation_mode == "null": + # TODO: ChatCompletionRequest does not have bootstrap info required by disaggregation mode, disable image-warmup for now + json_data = { + "model": _global_state.tokenizer_manager.served_model_name, + "messages": [ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": { + "url": f"data:image/png;base64,{MINIMUM_PNG_PICTURE_BASE64}" + }, + }, + { + "type": "text", + "text": "Describe the image.", + }, + ], + } + ], + "max_tokens": max_new_tokens, + "stream": False, + "temperature": 0.0, + } else: json_data["text"] = ["The capital city of France is"] * server_args.dp_size # TODO Workaround the bug that embedding errors for list of size 1 diff --git a/python/sglang/srt/models/qwen2_5_vl.py b/python/sglang/srt/models/qwen2_5_vl.py index 29a9f37b0..b6abf722b 100644 --- a/python/sglang/srt/models/qwen2_5_vl.py +++ b/python/sglang/srt/models/qwen2_5_vl.py @@ -23,7 +23,7 @@ # limitations under the License. """Inference-only Qwen2-VL model compatible with HuggingFace weights.""" import logging -from functools import lru_cache, partial +from functools import partial from typing import Iterable, List, Optional, Tuple, Type import torch @@ -61,7 +61,6 @@ from sglang.srt.model_loader.weight_utils import default_weight_loader from sglang.srt.models.qwen2 import Qwen2Model from sglang.srt.models.utils import permute_inv from sglang.srt.utils import add_prefix -from sglang.srt.utils.hf_transformers_utils import get_processor logger = logging.getLogger(__name__) @@ -117,8 +116,6 @@ class Qwen2_5_VisionBlock(nn.Module): rms_norm_eps: float = 1e-6, ) -> None: super().__init__() - if norm_layer is None: - norm_layer = partial(nn.LayerNorm, eps=1e-6) self.norm1 = RMSNorm(dim, eps=rms_norm_eps) self.norm2 = RMSNorm(dim, eps=rms_norm_eps) @@ -458,9 +455,6 @@ class Qwen2_5_VisionTransformer(nn.Module): return x -cached_get_processor = lru_cache(get_processor) - - class Qwen2_5_VLForConditionalGeneration(nn.Module): # BitandBytes specific attributes default_bitsandbytes_target_modules = [