[Anthropic API] Preserve image content in tool_result conversion (#19233)
Co-authored-by: Xinyuan Tong <115166877+JustinTong0323@users.noreply.github.com>
This commit is contained in:
Mohammad Miadh Angkad
co-authored by
Xinyuan Tong
Xinyuan Tong
parent
a0d8a7ae6d
commit
9c81ce4707
@@ -92,6 +92,73 @@ class AnthropicServing:
|
||||
"""Convert an Anthropic Messages request to an OpenAI ChatCompletion request."""
|
||||
openai_messages = []
|
||||
|
||||
def _convert_anthropic_image_source_to_openai_part(
|
||||
source: Optional[dict],
|
||||
) -> Optional[dict]:
|
||||
if not isinstance(source, dict):
|
||||
return None
|
||||
|
||||
source_type = source.get("type")
|
||||
if source_type == "base64":
|
||||
media_type = source.get("media_type", "image/png")
|
||||
data = source.get("data", "")
|
||||
if not data:
|
||||
return None
|
||||
return {
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
"url": f"data:{media_type};base64,{data}",
|
||||
},
|
||||
}
|
||||
|
||||
url = source.get("url")
|
||||
if url:
|
||||
return {
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
"url": url,
|
||||
},
|
||||
}
|
||||
|
||||
return None
|
||||
|
||||
def _convert_tool_result_content(
|
||||
content: Optional[str | list[dict]],
|
||||
) -> tuple[str | list[dict], str]:
|
||||
if isinstance(content, list):
|
||||
tool_content_parts = []
|
||||
tool_text_parts = []
|
||||
|
||||
for item in content:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
|
||||
item_type = item.get("type")
|
||||
if item_type == "text":
|
||||
text = item.get("text", "")
|
||||
if text:
|
||||
tool_text_parts.append(text)
|
||||
tool_content_parts.append({"type": "text", "text": text})
|
||||
elif item_type == "image":
|
||||
image_part = _convert_anthropic_image_source_to_openai_part(
|
||||
item.get("source")
|
||||
)
|
||||
if image_part is not None:
|
||||
tool_content_parts.append(image_part)
|
||||
|
||||
tool_text = "\n".join(tool_text_parts)
|
||||
if (
|
||||
len(tool_content_parts) == 1
|
||||
and tool_content_parts[0]["type"] == "text"
|
||||
):
|
||||
return tool_content_parts[0]["text"], tool_text
|
||||
if tool_content_parts:
|
||||
return tool_content_parts, tool_text
|
||||
return "", tool_text
|
||||
|
||||
tool_text = str(content) if content else ""
|
||||
return tool_text, tool_text
|
||||
|
||||
# Add system message if provided
|
||||
if anthropic_request.system:
|
||||
if isinstance(anthropic_request.system, str):
|
||||
@@ -122,16 +189,11 @@ class AnthropicServing:
|
||||
content_parts.append({"type": "text", "text": block.text})
|
||||
|
||||
elif block.type == "image" and block.source:
|
||||
media_type = block.source.get("media_type", "image/png")
|
||||
data = block.source.get("data", "")
|
||||
content_parts.append(
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
"url": f"data:{media_type};base64,{data}",
|
||||
},
|
||||
}
|
||||
image_part = _convert_anthropic_image_source_to_openai_part(
|
||||
block.source
|
||||
)
|
||||
if image_part is not None:
|
||||
content_parts.append(image_part)
|
||||
|
||||
elif block.type == "tool_use":
|
||||
tool_call = {
|
||||
@@ -145,15 +207,9 @@ class AnthropicServing:
|
||||
tool_calls.append(tool_call)
|
||||
|
||||
elif block.type == "tool_result":
|
||||
# Extract text content from list or string
|
||||
if isinstance(block.content, list):
|
||||
tool_content = "\n".join(
|
||||
item.get("text", "")
|
||||
for item in block.content
|
||||
if isinstance(item, dict) and item.get("type") == "text"
|
||||
)
|
||||
else:
|
||||
tool_content = str(block.content) if block.content else ""
|
||||
tool_content, tool_text = _convert_tool_result_content(
|
||||
block.content
|
||||
)
|
||||
|
||||
# Use tool_use_id (per spec) with fallback to id
|
||||
tool_call_id = block.tool_use_id or block.id or ""
|
||||
@@ -171,7 +227,7 @@ class AnthropicServing:
|
||||
content_parts.append(
|
||||
{
|
||||
"type": "text",
|
||||
"text": f"Tool result: {tool_content}",
|
||||
"text": f"Tool result: {tool_text}",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@@ -11,6 +11,7 @@ python3 -m unittest openai_server.basic.test_anthropic_server.TestAnthropicServe
|
||||
python3 -m unittest openai_server.basic.test_anthropic_server.TestAnthropicServer.test_error_empty_messages
|
||||
python3 -m unittest openai_server.basic.test_anthropic_server.TestAnthropicServer.test_raw_http_non_streaming
|
||||
python3 -m unittest openai_server.basic.test_anthropic_server.TestAnthropicServer.test_raw_http_streaming
|
||||
python3 -m unittest openai_server.basic.test_anthropic_server.TestAnthropicServer.test_tool_result_image_content_conversion
|
||||
"""
|
||||
|
||||
import json
|
||||
@@ -18,6 +19,8 @@ import unittest
|
||||
|
||||
import requests
|
||||
|
||||
from sglang.srt.entrypoints.anthropic.protocol import AnthropicMessagesRequest
|
||||
from sglang.srt.entrypoints.anthropic.serving import AnthropicServing
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
|
||||
from sglang.test.test_utils import (
|
||||
@@ -80,6 +83,70 @@ class TestAnthropicServer(CustomTestCase):
|
||||
|
||||
# ---- Non-streaming tests ----
|
||||
|
||||
def test_tool_result_image_content_conversion(self):
|
||||
"""Tool-result image blocks should be preserved as OpenAI image_url content."""
|
||||
anthropic_request = AnthropicMessagesRequest(
|
||||
model=self.model,
|
||||
max_tokens=64,
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": "I have called read_file to get an image. What color is it?",
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{
|
||||
"type": "tool_use",
|
||||
"id": "call_123",
|
||||
"name": "read_file",
|
||||
"input": {"file_path": "/test.png"},
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "tool_result",
|
||||
"tool_use_id": "call_123",
|
||||
"content": [
|
||||
{
|
||||
"type": "image",
|
||||
"source": {
|
||||
"type": "base64",
|
||||
"media_type": "image/png",
|
||||
"data": "abcd",
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
],
|
||||
},
|
||||
],
|
||||
)
|
||||
|
||||
serving = AnthropicServing(openai_serving_chat=object())
|
||||
chat_request = serving._convert_to_chat_completion_request(anthropic_request)
|
||||
converted = chat_request.model_dump()
|
||||
|
||||
tool_messages = [m for m in converted["messages"] if m.get("role") == "tool"]
|
||||
self.assertEqual(
|
||||
len(tool_messages),
|
||||
1,
|
||||
f"Expected one tool message, got: {converted['messages']}",
|
||||
)
|
||||
|
||||
tool_message = tool_messages[0]
|
||||
self.assertEqual(tool_message["tool_call_id"], "call_123")
|
||||
self.assertIsInstance(tool_message["content"], list)
|
||||
self.assertEqual(len(tool_message["content"]), 1)
|
||||
self.assertEqual(tool_message["content"][0]["type"], "image_url")
|
||||
self.assertEqual(
|
||||
tool_message["content"][0]["image_url"]["url"],
|
||||
"data:image/png;base64,abcd",
|
||||
)
|
||||
|
||||
def test_simple_messages(self):
|
||||
"""Test basic non-streaming message request."""
|
||||
payload = self._default_payload()
|
||||
|
||||
Reference in New Issue
Block a user