[Feature] Adds basic support for image content in OpenAI chat routes (#113)

python/sglang/srt/managers/openai_protocol.py

@@ -1,5 +1,6 @@
 import time
 from typing import Dict, List, Optional, Union
+from typing_extensions import Literal
 
 from pydantic import BaseModel, Field
 
@@ -68,9 +69,44 @@ class CompletionStreamResponse(BaseModel):
     usage: UsageInfo
 
 
+class ChatCompletionMessageGenericParam(BaseModel):
+    role: Literal["system", "assistant"]
+    content: str
+
+
+class ChatCompletionMessageContentTextPart(BaseModel):
+    type: Literal["text"]
+    text: str
+
+
+class ChatCompletionMessageContentImageURL(BaseModel):
+    url: str
+    detail: Optional[Literal["auto", "low", "high"]] = "auto"
+
+
+class ChatCompletionMessageContentImagePart(BaseModel):
+    type: Literal["image_url"]
+    image_url: ChatCompletionMessageContentImageURL
+
+
+ChatCompletionMessageContentPart = Union[
+    ChatCompletionMessageContentTextPart, ChatCompletionMessageContentImagePart
+]
+
+
+class ChatCompletionMessageUserParam(BaseModel):
+    role: Literal["user"]
+    content: Union[str, List[ChatCompletionMessageContentPart]]
+
+
+ChatCompletionMessageParam = Union[
+    ChatCompletionMessageGenericParam, ChatCompletionMessageUserParam
+]
+
+
 class ChatCompletionRequest(BaseModel):
     model: str
-    messages: Union[str, List[Dict[str, str]]]
+    messages: Union[str, List[ChatCompletionMessageParam]]
     temperature: Optional[float] = 0.7
     top_p: Optional[float] = 1.0
     n: Optional[int] = 1

python/sglang/srt/managers/tokenizer_manager.py

@@ -150,12 +150,17 @@ class TokenizerManager:
             if sampling_params.max_new_tokens != 0:
                 sampling_params.normalize(self.tokenizer)
                 sampling_params.verify()
-            if obj.image_data is None:
-                pixel_values, image_hash, image_size = None, None, None
-            else:
+
+            if isinstance(obj.image_data, list) and len(obj.image_data) > 0:
+                pixel_values, image_hash, image_size = await self.get_pixel_values(
+                    obj.image_data[0]
+                )
+            elif isinstance(obj.image_data, str):
                 pixel_values, image_hash, image_size = await self.get_pixel_values(
                     obj.image_data
                 )
+            else:
+                pixel_values, image_hash, image_size = None, None, None
             tokenized_obj = TokenizedGenerateReqInput(
                 rid=rid,
                 input_text=obj.text,