chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 11 Packages Projects Releases Wiki Activity

[router] Support complex assistant and tool messages in /chat/completions (#12860)

Browse Source 
Co-authored-by: Chang Su <chang.s.su@oracle.com>
Co-authored-by: Simo Lin <linsimo.mark@gmail.com>
This commit is contained in:
Danylo Vashchilenko
2025-11-12 00:14:15 -08:00
committed by GitHub
co-authored by Chang Su Simo Lin
parent ad8d24c39e
commit d28caaf60a
13 changed files with 127 additions and 104 deletions
Download Patch File Download Diff File Expand all files Collapse all files
sgl-router/tests/chat_template_format_detection.rs
+4 -4
View File
@@ -1,5 +1,5 @@
use sglang_router_rs::{
protocols::chat::{ChatMessage, UserMessageContent},
protocols::chat::{ChatMessage, MessageContent},
tokenizer::chat_template::{
detect_chat_template_content_format, ChatTemplateContentFormat, ChatTemplateParams,
ChatTemplateProcessor,
@@ -176,11 +176,11 @@ assistant:
let messages = [
ChatMessage::System {
content: "You are helpful".to_string(),
content: MessageContent::Text("You are helpful".to_string()),
name: None,
},
ChatMessage::User {
content: UserMessageContent::Text("Hello".to_string()),
content: MessageContent::Text("Hello".to_string()),
name: None,
},
];
@@ -216,7 +216,7 @@ fn test_chat_template_with_tokens_unit_test() {
let processor = ChatTemplateProcessor::new(template.to_string());
let messages = [ChatMessage::User {
content: UserMessageContent::Text("Test".to_string()),
content: MessageContent::Text("Test".to_string()),
name: None,
}];
sgl-router/tests/chat_template_integration.rs
+10 -10
View File
@@ -1,6 +1,6 @@
use sglang_router_rs::{
protocols::{
chat::{ChatMessage, UserMessageContent},
chat::{ChatMessage, MessageContent},
common::{ContentPart, ImageUrl},
},
tokenizer::chat_template::{
@@ -23,7 +23,7 @@ fn test_simple_chat_template() {
let processor = ChatTemplateProcessor::new(template.to_string());
let messages = [ChatMessage::User {
content: UserMessageContent::Text("Test".to_string()),
content: MessageContent::Text("Test".to_string()),
name: None,
}];
@@ -57,7 +57,7 @@ fn test_chat_template_with_tokens() {
let processor = ChatTemplateProcessor::new(template.to_string());
let messages = [ChatMessage::User {
content: UserMessageContent::Text("Test".to_string()),
content: MessageContent::Text("Test".to_string()),
name: None,
}];
@@ -118,11 +118,11 @@ fn test_llama_style_template() {
let messages = [
ChatMessage::System {
content: "You are a helpful assistant".to_string(),
content: MessageContent::Text("You are a helpful assistant".to_string()),
name: None,
},
ChatMessage::User {
content: UserMessageContent::Text("What is 2+2?".to_string()),
content: MessageContent::Text("What is 2+2?".to_string()),
name: None,
},
];
@@ -173,17 +173,17 @@ fn test_chatml_template() {
let messages = [
ChatMessage::User {
content: UserMessageContent::Text("Hello".to_string()),
content: MessageContent::Text("Hello".to_string()),
name: None,
},
ChatMessage::Assistant {
content: Some("Hi there!".to_string()),
content: Some(MessageContent::Text("Hi there!".to_string())),
name: None,
tool_calls: None,
reasoning_content: None,
},
ChatMessage::User {
content: UserMessageContent::Text("How are you?".to_string()),
content: MessageContent::Text("How are you?".to_string()),
name: None,
},
];
@@ -225,7 +225,7 @@ assistant:
let processor = ChatTemplateProcessor::new(template.to_string());
let messages = [ChatMessage::User {
content: UserMessageContent::Text("Test".to_string()),
content: MessageContent::Text("Test".to_string()),
name: None,
}];
@@ -312,7 +312,7 @@ fn test_template_with_multimodal_content() {
let processor = ChatTemplateProcessor::new(template.to_string());
let messages = [ChatMessage::User {
content: UserMessageContent::Parts(vec![
content: MessageContent::Parts(vec![
ContentPart::Text {
text: "Look at this:".to_string(),
},
sgl-router/tests/chat_template_loading.rs
+6 -6
View File
@@ -3,7 +3,7 @@ mod tests {
use std::fs;
use sglang_router_rs::{
protocols::chat::{ChatMessage, UserMessageContent},
protocols::chat::{ChatMessage, MessageContent},
tokenizer::{chat_template::ChatTemplateParams, huggingface::HuggingFaceTokenizer},
};
use tempfile::TempDir;
@@ -61,11 +61,11 @@ mod tests {
let messages = [
ChatMessage::User {
content: UserMessageContent::Text("Hello".to_string()),
content: MessageContent::Text("Hello".to_string()),
name: None,
},
ChatMessage::Assistant {
content: Some("Hi there".to_string()),
content: Some(MessageContent::Text("Hi there".to_string())),
name: None,
tool_calls: None,
reasoning_content: None,
@@ -143,7 +143,7 @@ mod tests {
.unwrap();
let messages = [ChatMessage::User {
content: UserMessageContent::Text("Test".to_string()),
content: MessageContent::Text("Test".to_string()),
name: None,
}];
@@ -202,11 +202,11 @@ mod tests {
let messages = [
ChatMessage::User {
content: UserMessageContent::Text("Hello".to_string()),
content: MessageContent::Text("Hello".to_string()),
name: None,
},
ChatMessage::Assistant {
content: Some("World".to_string()),
content: Some(MessageContent::Text("World".to_string())),
name: None,
tool_calls: None,
reasoning_content: None,
sgl-router/tests/spec/chat_completion.rs
+17 -17
View File
@@ -1,6 +1,6 @@
use serde_json::json;
use sglang_router_rs::protocols::{
chat::{ChatCompletionRequest, ChatMessage, UserMessageContent},
chat::{ChatCompletionRequest, ChatMessage, MessageContent},
common::{
Function, FunctionCall, FunctionChoice, StreamOptions, Tool, ToolChoice, ToolChoiceValue,
ToolReference,
@@ -17,7 +17,7 @@ fn test_max_tokens_normalizes_to_max_completion_tokens() {
let mut req = ChatCompletionRequest {
model: "test-model".to_string(),
messages: vec![ChatMessage::User {
content: UserMessageContent::Text("hello".to_string()),
content: MessageContent::Text("hello".to_string()),
name: None,
}],
max_tokens: Some(100),
@@ -50,7 +50,7 @@ fn test_max_completion_tokens_takes_precedence() {
let mut req = ChatCompletionRequest {
model: "test-model".to_string(),
messages: vec![ChatMessage::User {
content: UserMessageContent::Text("hello".to_string()),
content: MessageContent::Text("hello".to_string()),
name: None,
}],
max_tokens: Some(100),
@@ -76,7 +76,7 @@ fn test_functions_normalizes_to_tools() {
let mut req = ChatCompletionRequest {
model: "test-model".to_string(),
messages: vec![ChatMessage::User {
content: UserMessageContent::Text("hello".to_string()),
content: MessageContent::Text("hello".to_string()),
name: None,
}],
functions: Some(vec![Function {
@@ -112,7 +112,7 @@ fn test_function_call_normalizes_to_tool_choice() {
let mut req = ChatCompletionRequest {
model: "test-model".to_string(),
messages: vec![ChatMessage::User {
content: UserMessageContent::Text("hello".to_string()),
content: MessageContent::Text("hello".to_string()),
name: None,
}],
function_call: Some(FunctionCall::None),
@@ -148,7 +148,7 @@ fn test_function_call_function_variant_normalizes() {
let mut req = ChatCompletionRequest {
model: "test-model".to_string(),
messages: vec![ChatMessage::User {
content: UserMessageContent::Text("hello".to_string()),
content: MessageContent::Text("hello".to_string()),
name: None,
}],
function_call: Some(FunctionCall::Function {
@@ -198,7 +198,7 @@ fn test_stream_options_requires_stream_enabled() {
let req = ChatCompletionRequest {
model: "test-model".to_string(),
messages: vec![ChatMessage::User {
content: UserMessageContent::Text("hello".to_string()),
content: MessageContent::Text("hello".to_string()),
name: None,
}],
stream: false,
@@ -226,7 +226,7 @@ fn test_stream_options_valid_when_stream_enabled() {
let req = ChatCompletionRequest {
model: "test-model".to_string(),
messages: vec![ChatMessage::User {
content: UserMessageContent::Text("hello".to_string()),
content: MessageContent::Text("hello".to_string()),
name: None,
}],
stream: true,
@@ -248,7 +248,7 @@ fn test_no_stream_options_valid_when_stream_disabled() {
let req = ChatCompletionRequest {
model: "test-model".to_string(),
messages: vec![ChatMessage::User {
content: UserMessageContent::Text("hello".to_string()),
content: MessageContent::Text("hello".to_string()),
name: None,
}],
stream: false,
@@ -269,7 +269,7 @@ fn test_tool_choice_function_not_found() {
let req = ChatCompletionRequest {
model: "test-model".to_string(),
messages: vec![ChatMessage::User {
content: UserMessageContent::Text("hello".to_string()),
content: MessageContent::Text("hello".to_string()),
name: None,
}],
tools: Some(vec![Tool {
@@ -305,7 +305,7 @@ fn test_tool_choice_function_exists_valid() {
let req = ChatCompletionRequest {
model: "test-model".to_string(),
messages: vec![ChatMessage::User {
content: UserMessageContent::Text("hello".to_string()),
content: MessageContent::Text("hello".to_string()),
name: None,
}],
tools: Some(vec![Tool {
@@ -335,7 +335,7 @@ fn test_tool_choice_allowed_tools_invalid_mode() {
let req = ChatCompletionRequest {
model: "test-model".to_string(),
messages: vec![ChatMessage::User {
content: UserMessageContent::Text("hello".to_string()),
content: MessageContent::Text("hello".to_string()),
name: None,
}],
tools: Some(vec![Tool {
@@ -372,7 +372,7 @@ fn test_tool_choice_allowed_tools_valid_mode_auto() {
let req = ChatCompletionRequest {
model: "test-model".to_string(),
messages: vec![ChatMessage::User {
content: UserMessageContent::Text("hello".to_string()),
content: MessageContent::Text("hello".to_string()),
name: None,
}],
tools: Some(vec![Tool {
@@ -403,7 +403,7 @@ fn test_tool_choice_allowed_tools_valid_mode_required() {
let req = ChatCompletionRequest {
model: "test-model".to_string(),
messages: vec![ChatMessage::User {
content: UserMessageContent::Text("hello".to_string()),
content: MessageContent::Text("hello".to_string()),
name: None,
}],
tools: Some(vec![Tool {
@@ -434,7 +434,7 @@ fn test_tool_choice_allowed_tools_tool_not_found() {
let req = ChatCompletionRequest {
model: "test-model".to_string(),
messages: vec![ChatMessage::User {
content: UserMessageContent::Text("hello".to_string()),
content: MessageContent::Text("hello".to_string()),
name: None,
}],
tools: Some(vec![Tool {
@@ -471,7 +471,7 @@ fn test_tool_choice_allowed_tools_multiple_tools_valid() {
let req = ChatCompletionRequest {
model: "test-model".to_string(),
messages: vec![ChatMessage::User {
content: UserMessageContent::Text("hello".to_string()),
content: MessageContent::Text("hello".to_string()),
name: None,
}],
tools: Some(vec![
@@ -518,7 +518,7 @@ fn test_tool_choice_allowed_tools_one_invalid_among_valid() {
let req = ChatCompletionRequest {
model: "test-model".to_string(),
messages: vec![ChatMessage::User {
content: UserMessageContent::Text("hello".to_string()),
content: MessageContent::Text("hello".to_string()),
name: None,
}],
tools: Some(vec![
sgl-router/tests/spec/chat_message.rs
+13 -5
View File
@@ -1,5 +1,5 @@
use serde_json::json;
use sglang_router_rs::protocols::chat::{ChatMessage, UserMessageContent};
use sglang_router_rs::protocols::chat::{ChatMessage, MessageContent};
#[test]
fn test_chat_message_tagged_by_role_system() {
@@ -11,7 +11,10 @@ fn test_chat_message_tagged_by_role_system() {
let msg: ChatMessage = serde_json::from_value(json).unwrap();
match msg {
ChatMessage::System { content, .. } => {
assert_eq!(content, "You are a helpful assistant");
assert_eq!(
content,
MessageContent::Text("You are a helpful assistant".to_string())
)
}
_ => panic!("Expected System variant"),
}
@@ -27,7 +30,7 @@ fn test_chat_message_tagged_by_role_user() {
let msg: ChatMessage = serde_json::from_value(json).unwrap();
match msg {
ChatMessage::User { content, .. } => match content {
UserMessageContent::Text(text) => assert_eq!(text, "Hello"),
MessageContent::Text(text) => assert_eq!(text, "Hello"),
_ => panic!("Expected text content"),
},
_ => panic!("Expected User variant"),
@@ -44,7 +47,7 @@ fn test_chat_message_tagged_by_role_assistant() {
let msg: ChatMessage = serde_json::from_value(json).unwrap();
match msg {
ChatMessage::Assistant { content, .. } => {
assert_eq!(content, Some("Hi there!".to_string()));
assert_eq!(content, Some(MessageContent::Text("Hi there!".to_string())));
}
_ => panic!("Expected Assistant variant"),
}
@@ -64,7 +67,12 @@ fn test_chat_message_tagged_by_role_tool() {
content,
tool_call_id,
} => {
assert_eq!(content, "Tool result");
match content {
MessageContent::Text(text) => {
assert_eq!(text, "Tool result");
}
_ => panic!("Expected content to be a string"),
}
assert_eq!(tool_call_id, "call_123");
}
_ => panic!("Expected Tool variant"),
sgl-router/tests/test_openai_routing.rs
+2 -2
View File
@@ -23,7 +23,7 @@ use sglang_router_rs::{
},
data_connector::{ResponseId, StoredResponse},
protocols::{
chat::{ChatCompletionRequest, ChatMessage, UserMessageContent},
chat::{ChatCompletionRequest, ChatMessage, MessageContent},
common::StringOrArray,
completion::CompletionRequest,
generate::GenerateRequest,
@@ -661,7 +661,7 @@ async fn test_openai_router_chat_completion_with_mock() {
// Create a minimal chat completion request
let mut chat_request = create_minimal_chat_request();
chat_request.messages = vec![ChatMessage::User {
content: UserMessageContent::Text("Hello, how are you?".to_string()),
content: MessageContent::Text("Hello, how are you?".to_string()),
name: None,
}];
chat_request.temperature = Some(0.7);