[model-gateway]: add qwen coder tool parser support xml format for qwen3 coder and microthinker (#12909)

This commit is contained in:
ybyang
2026-01-12 08:09:41 +08:00
committed by GitHub
parent 934ae89abe
commit 503c3d9566
6 changed files with 1736 additions and 3 deletions

View File

@@ -10,7 +10,7 @@ use tokio::sync::Mutex;
use crate::tool_parser::{
parsers::{
DeepSeekParser, Glm4MoeParser, JsonParser, KimiK2Parser, LlamaParser, MinimaxM2Parser,
MistralParser, PassthroughParser, PythonicParser, QwenParser, Step3Parser,
MistralParser, PassthroughParser, PythonicParser, QwenCoderParser, QwenParser, Step3Parser,
},
traits::ToolParser,
};
@@ -236,6 +236,7 @@ impl ParserFactory {
registry.register_parser("json", || Box::new(JsonParser::new()));
registry.register_parser("mistral", || Box::new(MistralParser::new()));
registry.register_parser("qwen", || Box::new(QwenParser::new()));
registry.register_parser("qwen_coder", || Box::new(QwenCoderParser::new()));
registry.register_parser("pythonic", || Box::new(PythonicParser::new()));
registry.register_parser("llama", || Box::new(LlamaParser::new()));
registry.register_parser("deepseek", || Box::new(DeepSeekParser::new()));
@@ -264,7 +265,15 @@ impl ParserFactory {
registry.map_model("mistral-*", "mistral");
registry.map_model("mixtral-*", "mistral");
// Qwen models
// Qwen models (more specific patterns first - longer patterns take precedence)
// Qwen Coder models use XML format: <tool_call><function=name><parameter=key>value</parameter></function></tool_call>
registry.map_model("Qwen/Qwen3-Coder*", "qwen_coder");
registry.map_model("Qwen3-Coder*", "qwen_coder");
registry.map_model("qwen3-coder*", "qwen_coder");
registry.map_model("Qwen/Qwen2.5-Coder*", "qwen_coder");
registry.map_model("Qwen2.5-Coder*", "qwen_coder");
registry.map_model("qwen2.5-coder*", "qwen_coder");
// Generic Qwen models use JSON format
registry.map_model("qwen*", "qwen");
registry.map_model("Qwen*", "qwen");

View File

@@ -13,6 +13,7 @@ pub mod mistral;
pub mod passthrough;
pub mod pythonic;
pub mod qwen;
pub mod qwen_coder;
pub mod step3;
// Shared helpers and utilities
@@ -29,4 +30,5 @@ pub use mistral::MistralParser;
pub(crate) use passthrough::PassthroughParser;
pub use pythonic::PythonicParser;
pub use qwen::QwenParser;
pub use qwen_coder::QwenCoderParser;
pub use step3::Step3Parser;

View File

@@ -0,0 +1,587 @@
use async_trait::async_trait;
use regex::Regex;
use serde_json::Value;
use crate::{
protocols::common::Tool,
tool_parser::{
errors::{ParserError, ParserResult},
parsers::helpers,
traits::ToolParser,
types::{FunctionCall, StreamingParseResult, ToolCall, ToolCallItem},
},
};
/// Qwen Coder format parser for tool calls
///
/// Handles the Qwen Coder specific XML format:
/// `<tool_call>\n<function=name>\n<parameter=key>value</parameter>\n</function>\n</tool_call>`
///
/// Features:
/// - Tool Call Tags: `<tool_call>` and `</tool_call>` wrap each individual call
/// - XML-style function declaration: `<function=name>`
/// - XML-style parameters: `<parameter=key>value</parameter>`
///
/// Reference: https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8?chat_template=default
pub struct QwenCoderParser {
/// Regex for extracting tool calls in parse_complete
extractor: Regex,
/// Buffer for accumulating incomplete patterns across chunks
buffer: String,
/// Stores complete tool call info (name and arguments) for each tool being parsed
prev_tool_call_arr: Vec<Value>,
/// Index of currently streaming tool call (-1 means no active tool)
current_tool_id: i32,
/// Flag for whether current tool's name has been sent to client
current_tool_name_sent: bool,
/// Tracks raw JSON string content streamed to client for each tool's arguments
streamed_args_for_tool: Vec<String>,
/// Token configuration
tool_call_start_token: &'static str,
tool_call_end_token: &'static str,
/// XML format streaming state
in_tool_call: bool,
current_function_name: String,
current_parameters: serde_json::Map<String, Value>,
/// Precompiled regex patterns for XML format parsing
xml_function_pattern: Regex,
xml_param_pattern: Regex,
}
/// Decode HTML entities in a string (equivalent to Python's html.unescape)
///
/// Handles common HTML entities like &amp; &lt; &gt; &quot; &#39; and numeric entities
fn html_unescape(s: &str) -> String {
let mut result = String::with_capacity(s.len());
let mut chars = s.chars().peekable();
while let Some(c) = chars.next() {
if c == '&' {
let mut entity = String::new();
let mut consumed_semicolon = false;
while let Some(&next) = chars.peek() {
if next == ';' {
chars.next();
consumed_semicolon = true;
break;
}
if next.is_alphanumeric() || next == '#' {
entity.push(chars.next().unwrap());
} else {
break;
}
}
let decoded = match entity.as_str() {
"amp" => "&",
"lt" => "<",
"gt" => ">",
"quot" => "\"",
"apos" => "'",
"nbsp" => "\u{00A0}",
s if s.starts_with('#') => {
let num_str = &s[1..];
let code_point = if num_str.starts_with('x') || num_str.starts_with('X') {
u32::from_str_radix(&num_str[1..], 16).ok()
} else {
num_str.parse::<u32>().ok()
};
if let Some(cp) = code_point {
if let Some(ch) = char::from_u32(cp) {
result.push(ch);
continue;
}
}
// Invalid numeric entity, reconstruct original
result.push('&');
result.push_str(&entity);
if consumed_semicolon {
result.push(';');
}
continue;
}
_ => {
// Unknown entity, reconstruct original
result.push('&');
result.push_str(&entity);
if consumed_semicolon {
result.push(';');
}
continue;
}
};
result.push_str(decoded);
} else {
result.push(c);
}
}
result
}
/// Parse a raw parameter value, similar to Python's _safe_val
///
/// 1. Decode HTML entities
/// 2. Try to parse as JSON (numbers, booleans, null, objects, arrays)
/// 3. Fall back to string if JSON parsing fails
fn safe_val(raw: &str) -> Value {
let unescaped = html_unescape(raw.trim());
// Try JSON parsing first
if let Ok(v) = serde_json::from_str::<Value>(&unescaped) {
return v;
}
// Handle Python-style literals (True, False, None)
match unescaped.as_str() {
"True" => return Value::Bool(true),
"False" => return Value::Bool(false),
"None" => return Value::Null,
_ => {}
}
// Fall back to string
Value::String(unescaped)
}
impl QwenCoderParser {
/// Create a new Qwen Coder parser
pub fn new() -> Self {
// Support XML format: <tool_call>\n<function=name>\n<parameter=key>value</parameter>\n</function>\n</tool_call>
let pattern = r"(?s)<tool_call>\s*(.*?)\s*</tool_call>";
let extractor = Regex::new(pattern).expect("Valid regex pattern");
// Precompile XML format regex patterns for performance
let xml_function_pattern =
Regex::new(r"<function=([^>]+)>").expect("Valid XML function pattern");
let xml_param_pattern = Regex::new(r"(?s)<parameter=([^>]+)>(.*?)</parameter>")
.expect("Valid XML parameter pattern");
Self {
extractor,
buffer: String::new(),
prev_tool_call_arr: Vec::new(),
current_tool_id: -1,
current_tool_name_sent: false,
streamed_args_for_tool: Vec::new(),
tool_call_start_token: "<tool_call>",
tool_call_end_token: "</tool_call>",
in_tool_call: false,
current_function_name: String::new(),
current_parameters: serde_json::Map::new(),
xml_function_pattern,
xml_param_pattern,
}
}
/// Parse XML format tool call: <function=name><parameter=key>value</parameter></function>
fn parse_xml_format(&self, content: &str) -> ParserResult<Option<ToolCall>> {
let function_captures = self
.xml_function_pattern
.captures(content)
.ok_or_else(|| ParserError::ParsingFailed("No function name found".to_string()))?;
let function_name = function_captures
.get(1)
.ok_or_else(|| ParserError::ParsingFailed("Function name capture failed".to_string()))?
.as_str()
.trim()
.to_string();
if function_name.is_empty() {
return Ok(None);
}
let mut parameters = serde_json::Map::new();
for cap in self.xml_param_pattern.captures_iter(content) {
if let (Some(key_match), Some(value_match)) = (cap.get(1), cap.get(2)) {
let key = key_match.as_str().trim().to_string();
let value = value_match.as_str();
let json_value = safe_val(value);
parameters.insert(key, json_value);
}
}
let arguments = serde_json::to_string(&parameters)
.map_err(|e| ParserError::ParsingFailed(e.to_string()))?;
Ok(Some(ToolCall {
function: FunctionCall {
name: function_name,
arguments,
},
}))
}
/// Parse and stream complete parameters from buffer
/// Returns tool call items to emit (similar to Python's _parse_and_stream_parameters)
fn parse_and_stream_parameters(&mut self) -> ParserResult<Vec<ToolCallItem>> {
let mut calls: Vec<ToolCallItem> = vec![];
// Find all complete parameter patterns in buffer
let mut new_params = serde_json::Map::new();
for cap in self.xml_param_pattern.captures_iter(&self.buffer) {
if let (Some(key_match), Some(value_match)) = (cap.get(1), cap.get(2)) {
let key = key_match.as_str().trim().to_string();
let value = value_match.as_str();
let json_value = safe_val(value);
new_params.insert(key, json_value);
}
}
// Calculate parameter diff and stream updates
if new_params != self.current_parameters {
let current_args = &mut self.streamed_args_for_tool[self.current_tool_id as usize];
if self.current_parameters.is_empty() {
// First parameter(s) - build JSON fragment (without closing brace)
let mut items = Vec::new();
for (key, value) in &new_params {
let key_json =
serde_json::to_string(key).unwrap_or_else(|_| format!("\"{}\"", key));
let value_json = serde_json::to_string(value).unwrap_or_default();
items.push(format!("{}: {}", key_json, value_json));
}
let json_fragment = format!("{{{}", items.join(", "));
calls.push(ToolCallItem {
tool_index: self.current_tool_id as usize,
name: None,
parameters: json_fragment.clone(),
});
*current_args = json_fragment;
} else {
// Additional parameters - add them incrementally
let new_keys: Vec<_> = new_params
.keys()
.filter(|k| !self.current_parameters.contains_key(*k))
.collect();
if !new_keys.is_empty() {
let mut continuation_parts = Vec::new();
for key in new_keys {
if let Some(value) = new_params.get(key) {
let key_json = serde_json::to_string(key)
.unwrap_or_else(|_| format!("\"{}\"", key));
let value_json = serde_json::to_string(value).unwrap_or_default();
continuation_parts.push(format!("{}: {}", key_json, value_json));
}
}
let json_fragment = format!(", {}", continuation_parts.join(", "));
calls.push(ToolCallItem {
tool_index: self.current_tool_id as usize,
name: None,
parameters: json_fragment.clone(),
});
current_args.push_str(&json_fragment);
}
}
// Update current state
self.current_parameters = new_params.clone();
if let Some(tool_obj) =
self.prev_tool_call_arr[self.current_tool_id as usize].as_object_mut()
{
tool_obj.insert("arguments".to_string(), Value::Object(new_params));
}
}
Ok(calls)
}
/// Reset streaming state for next tool call
fn reset_streaming_state(&mut self) {
self.in_tool_call = false;
self.current_tool_name_sent = false;
self.current_function_name.clear();
self.current_parameters.clear();
}
}
impl Default for QwenCoderParser {
fn default() -> Self {
Self::new()
}
}
#[async_trait]
impl ToolParser for QwenCoderParser {
async fn parse_complete(&self, text: &str) -> ParserResult<(String, Vec<ToolCall>)> {
// Check if text contains Qwen Coder format
if !self.has_tool_markers(text) {
return Ok((text.to_string(), vec![]));
}
// Find where the first tool call begins
let idx = text.find(self.tool_call_start_token).unwrap();
let normal_text = text[..idx].to_string();
// Extract tool calls
let mut tools = Vec::new();
for captures in self.extractor.captures_iter(text) {
if let Some(content_str) = captures.get(1) {
let content = content_str.as_str().trim();
match self.parse_xml_format(content) {
Ok(Some(tool)) => tools.push(tool),
Ok(None) => continue,
Err(e) => {
tracing::warn!("Failed to parse XML tool call: {:?}", e);
continue;
}
}
}
}
// If no tools were successfully parsed despite having markers, return entire text
if tools.is_empty() {
return Ok((text.to_string(), vec![]));
}
Ok((normal_text, tools))
}
async fn parse_incremental(
&mut self,
chunk: &str,
tools: &[Tool],
) -> ParserResult<StreamingParseResult> {
self.buffer.push_str(chunk);
let mut normal_text = String::new();
let mut calls: Vec<ToolCallItem> = vec![];
// Build tool indices for validation
let tool_indices = helpers::get_tool_indices(tools);
loop {
// If we're not in a tool call and don't see a start token, return normal text
if !self.in_tool_call && !self.buffer.contains(self.tool_call_start_token) {
// Check for partial start token
if helpers::ends_with_partial_token(&self.buffer, self.tool_call_start_token)
.is_none()
{
normal_text.push_str(&self.buffer);
self.buffer.clear();
}
break;
}
// Look for tool call start
if !self.in_tool_call {
if let Some(s) = self.buffer.find(self.tool_call_start_token) {
normal_text.push_str(&self.buffer[..s]);
self.buffer = self.buffer[s + self.tool_call_start_token.len()..].to_string();
self.in_tool_call = true;
self.current_tool_name_sent = false;
self.current_function_name.clear();
self.current_parameters.clear();
continue;
} else {
break;
}
}
// We're in a tool call, try to parse function name if not sent yet
if !self.current_tool_name_sent {
if let Some(captures) = self.xml_function_pattern.captures(&self.buffer) {
if let Some(name_match) = captures.get(1) {
let function_name = name_match.as_str().trim().to_string();
// Validate function name
if tool_indices.contains_key(&function_name) {
self.current_function_name = function_name.clone();
self.current_tool_name_sent = true;
// Initialize tool call tracking
if self.current_tool_id == -1 {
self.current_tool_id = 0;
}
// Ensure tracking arrays are large enough
helpers::ensure_capacity(
self.current_tool_id,
&mut self.prev_tool_call_arr,
&mut self.streamed_args_for_tool,
);
// Store tool call info
self.prev_tool_call_arr[self.current_tool_id as usize] = serde_json::json!({
"name": function_name,
"arguments": {}
});
// Send tool name
calls.push(ToolCallItem {
tool_index: self.current_tool_id as usize,
name: Some(function_name),
parameters: String::new(),
});
// Remove processed function declaration from buffer
self.buffer = self.buffer[captures.get(0).unwrap().end()..].to_string();
continue;
} else {
// Invalid function name, reset state
tracing::warn!("Invalid function name: {}", function_name);
self.reset_streaming_state();
normal_text.push_str(&self.buffer);
self.buffer.clear();
break;
}
}
} else {
// Function name not complete yet, wait for more text
break;
}
}
// Parse parameters (only complete ones)
if self.current_tool_name_sent {
let param_calls = self.parse_and_stream_parameters()?;
calls.extend(param_calls);
// Check if tool call is complete
if let Some(end_pos) = self.buffer.find(self.tool_call_end_token) {
// Close JSON object if we have parameters
let current_args = &self.streamed_args_for_tool[self.current_tool_id as usize];
if !current_args.is_empty() {
// Count braces to check if JSON is complete
let open_braces = current_args.matches('{').count();
let close_braces = current_args.matches('}').count();
if open_braces > close_braces {
calls.push(ToolCallItem {
tool_index: self.current_tool_id as usize,
name: None,
parameters: "}".to_string(),
});
self.streamed_args_for_tool[self.current_tool_id as usize].push('}');
}
}
// Complete the tool call
self.buffer =
self.buffer[end_pos + self.tool_call_end_token.len()..].to_string();
self.reset_streaming_state();
self.current_tool_id += 1;
continue;
} else {
// Tool call not complete yet, wait for more text
break;
}
}
break;
}
Ok(StreamingParseResult { normal_text, calls })
}
fn has_tool_markers(&self, text: &str) -> bool {
text.contains(self.tool_call_start_token)
}
fn get_unstreamed_tool_args(&self) -> Option<Vec<ToolCallItem>> {
helpers::get_unstreamed_args(&self.prev_tool_call_arr, &self.streamed_args_for_tool)
}
fn reset(&mut self) {
helpers::reset_parser_state(
&mut self.buffer,
&mut self.prev_tool_call_arr,
&mut self.current_tool_id,
&mut self.current_tool_name_sent,
&mut self.streamed_args_for_tool,
);
self.reset_streaming_state();
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_html_unescape_basic() {
assert_eq!(html_unescape("&amp;"), "&");
assert_eq!(html_unescape("&lt;"), "<");
assert_eq!(html_unescape("&gt;"), ">");
assert_eq!(html_unescape("&quot;"), "\"");
assert_eq!(html_unescape("&apos;"), "'");
}
#[test]
fn test_html_unescape_numeric() {
assert_eq!(html_unescape("&#60;"), "<");
assert_eq!(html_unescape("&#x3C;"), "<");
assert_eq!(html_unescape("&#x3c;"), "<");
}
#[test]
fn test_html_unescape_mixed() {
assert_eq!(
html_unescape("Hello &amp; World &lt;tag&gt;"),
"Hello & World <tag>"
);
}
#[test]
fn test_html_unescape_unknown() {
// Unknown entities with semicolon should be preserved as-is
assert_eq!(html_unescape("&unknown;"), "&unknown;");
// Unterminated entities should NOT have semicolon added
assert_eq!(html_unescape("&foo bar"), "&foo bar");
assert_eq!(html_unescape("&"), "&");
assert_eq!(html_unescape("& "), "& ");
}
#[test]
fn test_safe_val_json() {
assert_eq!(safe_val("42"), Value::Number(42.into()));
assert_eq!(safe_val("1.5"), serde_json::json!(1.5));
assert_eq!(safe_val("true"), Value::Bool(true));
assert_eq!(safe_val("false"), Value::Bool(false));
assert_eq!(safe_val("null"), Value::Null);
assert_eq!(
safe_val(r#"{"key": "value"}"#),
serde_json::json!({"key": "value"})
);
assert_eq!(safe_val(r#"[1, 2, 3]"#), serde_json::json!([1, 2, 3]));
}
#[test]
fn test_safe_val_python_literals() {
assert_eq!(safe_val("True"), Value::Bool(true));
assert_eq!(safe_val("False"), Value::Bool(false));
assert_eq!(safe_val("None"), Value::Null);
}
#[test]
fn test_safe_val_string_fallback() {
assert_eq!(
safe_val("hello world"),
Value::String("hello world".to_string())
);
assert_eq!(safe_val(" spaces "), Value::String("spaces".to_string()));
}
#[test]
fn test_safe_val_html_entities() {
assert_eq!(safe_val("&lt;div&gt;"), Value::String("<div>".to_string()));
assert_eq!(
safe_val("Tom &amp; Jerry"),
Value::String("Tom & Jerry".to_string())
);
}
}

View File

@@ -1,5 +1,12 @@
use super::*;
use crate::tool_parser::{parsers::JsonParser, partial_json::PartialJson, traits::ToolParser};
use crate::{
protocols::common::{Function, Tool},
tool_parser::{
parsers::{JsonParser, QwenCoderParser},
partial_json::PartialJson,
traits::ToolParser,
},
};
#[tokio::test]
async fn test_tool_parser_factory() {
@@ -562,3 +569,172 @@ mod stress_tests {
}
}
}
#[cfg(test)]
mod qwen_coder_tests {
use super::*;
fn create_test_tools() -> Vec<Tool> {
vec![Tool {
tool_type: "function".to_string(),
function: Function {
name: "get_weather".to_string(),
description: Some("Get weather information".to_string()),
parameters: serde_json::json!({
"type": "object",
"properties": {
"city": {"type": "string"},
"units": {"type": "string"}
}
}),
strict: None,
},
}]
}
#[tokio::test]
async fn test_qwen_coder_incremental_parameter_streaming() {
let mut parser = QwenCoderParser::new();
let tools = create_test_tools();
let chunks = [
"<tool_call>",
r#"<function=get_weather>"#,
r#"<parameter=city>Paris</parameter>"#,
r#"<parameter=units>metric</parameter>"#,
"</function></tool_call>",
];
let mut all_calls = Vec::new();
// Process each chunk
for (i, chunk) in chunks.iter().enumerate() {
let result = parser.parse_incremental(chunk, &tools).await.unwrap();
println!("Chunk {}: {:?}", i, chunk);
println!(" Calls: {:?}", result.calls);
println!(" Normal text: {:?}", result.normal_text);
for call in &result.calls {
all_calls.push(call.clone());
}
}
// Verify the final result
// We should have:
// 1. Tool name call (from chunk 2)
// 2. First parameter call: {"city": "Paris"}
// 3. Second parameter call: , "units": "metric"
// Final result should be: {"city": "Paris", "units": "metric"}
assert!(!all_calls.is_empty(), "Should have at least one call");
// Check that we have the tool name
let name_call = all_calls.iter().find(|c| c.name.is_some());
assert!(name_call.is_some(), "Should have tool name call");
assert_eq!(name_call.unwrap().name.as_ref().unwrap(), "get_weather");
// Check parameter calls
let param_calls: Vec<_> = all_calls.iter().filter(|c| c.name.is_none()).collect();
assert!(!param_calls.is_empty(), "Should have parameter calls");
// Verify final arguments format by concatenating all parameter fragments
let params_str: String = param_calls.iter().map(|c| c.parameters.as_str()).collect();
println!("Final streamed args: {}", params_str);
// Should contain both city and units parameters
assert!(params_str.contains("city"), "Should contain city parameter");
assert!(params_str.contains("Paris"), "Should contain Paris value");
assert!(
params_str.contains("units"),
"Should contain units parameter"
);
assert!(params_str.contains("metric"), "Should contain metric value");
}
#[tokio::test]
async fn test_qwen_coder_incremental_parameter_streaming_with_partial_values() {
let mut parser = QwenCoderParser::new();
let tools = create_test_tools();
// Test with parameter values that arrive in multiple chunks
// This tests the buffering logic for partial XML tags
let chunks = [
"<tool_call><function=get_weather>",
r#"<parameter=city>Paris</parameter>"#,
r#"<parameter=units>metric</parameter>"#,
"</function></tool_call>",
];
let mut all_calls = Vec::new();
// Process each chunk
for (i, chunk) in chunks.iter().enumerate() {
let result = parser.parse_incremental(chunk, &tools).await.unwrap();
println!("Chunk {}: {:?}", i, chunk);
println!(" Calls: {:?}", result.calls);
for call in &result.calls {
all_calls.push(call.clone());
}
}
// Verify we got the tool name
let name_call = all_calls.iter().find(|c| c.name.is_some());
assert!(name_call.is_some(), "Should have tool name call");
assert_eq!(name_call.unwrap().name.as_ref().unwrap(), "get_weather");
// Verify we got parameter calls
let param_calls: Vec<_> = all_calls.iter().filter(|c| c.name.is_none()).collect();
assert!(!param_calls.is_empty(), "Should have parameter calls");
// Verify final arguments by concatenating all parameter fragments
let params_str: String = param_calls.iter().map(|c| c.parameters.as_str()).collect();
assert!(params_str.contains("city"), "Should contain city parameter");
assert!(params_str.contains("Paris"), "Should contain Paris value");
}
#[tokio::test]
async fn test_qwen_coder_nested_json_parameter() {
let mut parser = QwenCoderParser::new();
let tools = vec![Tool {
tool_type: "function".to_string(),
function: Function {
name: "test_function".to_string(),
description: None,
parameters: serde_json::json!({
"type": "object",
"properties": {
"nested": {"type": "object"}
}
}),
strict: None,
},
}];
let chunks = vec![
"<tool_call>",
r#"<function=test_function>"#,
r#"<parameter=nested>{"key": "value"}</parameter>"#,
"</function></tool_call>",
];
let mut all_calls = Vec::new();
for chunk in chunks {
let result = parser.parse_incremental(chunk, &tools).await.unwrap();
for call in result.calls {
all_calls.push(call);
}
}
// Verify nested JSON is parsed correctly via collected calls
let param_calls: Vec<_> = all_calls.iter().filter(|c| c.name.is_none()).collect();
assert!(!param_calls.is_empty(), "Should have parameter calls");
// The first parameter call should contain the nested JSON
let params_str: String = param_calls.iter().map(|c| c.parameters.as_str()).collect();
assert!(
params_str.contains("nested"),
"Should contain nested parameter"
);
}
}

View File

@@ -14,4 +14,5 @@ pub mod tool_parser_mixed_edge_cases;
pub mod tool_parser_partial_json;
pub mod tool_parser_pythonic;
pub mod tool_parser_qwen;
pub mod tool_parser_qwen_coder;
pub mod tool_parser_step3;

View File

@@ -0,0 +1,958 @@
//! Qwen Coder Parser Integration Tests
//!
//! Tests for the Qwen Coder parser which handles XML format:
//! <tool_call>\n<function=name>\n<parameter=key>value</parameter>\n</function>\n</tool_call>
use serde_json::json;
use smg::tool_parser::{parsers::QwenCoderParser, traits::ToolParser};
use crate::common::{create_test_tools, streaming_helpers::*};
#[tokio::test]
async fn test_qwen_coder_single_tool() {
let parser = QwenCoderParser::new();
let input = r#"<tool_call>
<function=get_weather>
<parameter=city>Beijing</parameter>
<parameter=units>celsius</parameter>
</function>
</tool_call>"#;
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, "get_weather");
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert_eq!(args["city"], "Beijing");
assert_eq!(args["units"], "celsius");
}
#[tokio::test]
async fn test_qwen_coder_multiple_sequential_tools() {
let parser = QwenCoderParser::new();
let input = r#"Let me help you with that.
<tool_call>
<function=search>
<parameter=query>Qwen model</parameter>
</function>
</tool_call>
<tool_call>
<function=translate>
<parameter=text>Hello</parameter>
<parameter=to>zh</parameter>
</function>
</tool_call>"#;
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 2);
assert_eq!(normal_text, "Let me help you with that.\n");
assert_eq!(tools[0].function.name, "search");
assert_eq!(tools[1].function.name, "translate");
}
#[tokio::test]
async fn test_qwen_coder_nested_json_in_parameters() {
let parser = QwenCoderParser::new();
let input = r#"<tool_call>
<function=process_data>
<parameter=config>{"nested": {"value": [1, 2, 3]}}</parameter>
<parameter=enabled>true</parameter>
</function>
</tool_call>"#;
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, "process_data");
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
// JSON values should be parsed
assert_eq!(args["config"]["nested"]["value"], json!([1, 2, 3]));
assert_eq!(args["enabled"], true);
}
#[tokio::test]
async fn test_qwen_coder_string_parameters() {
let parser = QwenCoderParser::new();
let input = r#"<tool_call>
<function=process>
<parameter=text>Hello World</parameter>
<parameter=number>42</parameter>
</function>
</tool_call>"#;
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert_eq!(args["text"], "Hello World");
// JSON numbers should be parsed as numbers (consistent with Python's json.loads)
assert_eq!(args["number"], 42);
}
#[tokio::test]
async fn test_qwen_coder_empty_arguments() {
let parser = QwenCoderParser::new();
let input = r#"<tool_call>
<function=get_time>
</function>
</tool_call>"#;
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, "get_time");
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert_eq!(args, json!({}));
}
#[tokio::test]
async fn test_qwen_coder_multiline_parameter_values() {
let parser = QwenCoderParser::new();
let input = r#"<tool_call>
<function=write_file>
<parameter=content>Line 1
Line 2
Line 3</parameter>
<parameter=path>/tmp/test.txt</parameter>
</function>
</tool_call>"#;
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert_eq!(args["content"], "Line 1\nLine 2\nLine 3");
assert_eq!(args["path"], "/tmp/test.txt");
}
#[tokio::test]
async fn test_qwen_coder_format_detection() {
let parser = QwenCoderParser::new();
assert!(parser.has_tool_markers("<tool_call>"));
assert!(parser.has_tool_markers("Some text <tool_call>"));
assert!(!parser.has_tool_markers("Just plain text"));
assert!(!parser.has_tool_markers("<function=test>")); // Without tool_call tags
}
#[tokio::test]
async fn test_qwen_coder_incomplete_tags() {
let parser = QwenCoderParser::new();
// Missing closing tag
let input = r#"<tool_call>
<function=get_weather>
<parameter=city>Beijing</parameter>"#;
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 0);
// Missing opening tag
let input = r#"<parameter=city>Beijing</parameter>
</function>
</tool_call>"#;
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 0);
}
#[tokio::test]
async fn test_qwen_coder_streaming_basic() {
let mut parser = QwenCoderParser::new();
let tools = create_test_tools();
// Simulate streaming chunks
let chunks = vec![
"<tool_call>",
r#"<function=get_weather>"#,
r#"<parameter=city>Shanghai</parameter>"#,
r#"<parameter=units>celsius</parameter>"#,
"</function>",
"</tool_call>",
];
let mut found_name = false;
let mut found_params = false;
for chunk in chunks {
let result = parser.parse_incremental(chunk, &tools).await.unwrap();
for call in result.calls {
if let Some(name) = call.name {
assert_eq!(name, "get_weather");
found_name = true;
}
if !call.parameters.is_empty() {
found_params = true;
}
}
}
assert!(found_name, "Should have found tool name during streaming");
assert!(found_params, "Should have streamed parameters");
}
#[tokio::test]
async fn test_qwen_coder_streaming_incremental_json() {
let mut parser = QwenCoderParser::new();
let tools = create_test_tools();
let chunks = vec![
"<tool_call>",
r#"<function=get_weather>"#,
r#"<parameter=city>Paris</parameter>"#,
r#"<parameter=units>metric</parameter>"#,
"</function></tool_call>",
];
let mut json_fragments = Vec::new();
let mut found_function = false;
for chunk in chunks {
let result = parser.parse_incremental(chunk, &tools).await.unwrap();
for call in result.calls {
if let Some(_name) = call.name {
found_function = true;
}
if !call.parameters.is_empty() {
json_fragments.push(call.parameters.clone());
}
}
}
assert!(found_function);
// Verify JSON was built incrementally
assert!(!json_fragments.is_empty());
// First fragment should start with opening brace
if let Some(first) = json_fragments.first() {
assert!(
first.starts_with('{'),
"First JSON fragment should start with '{{': {}",
first
);
}
}
#[tokio::test]
async fn test_qwen_coder_streaming_partial_tags() {
let mut parser = QwenCoderParser::new();
let tools = create_test_tools();
// Chunks split mid-tag
let chunks = vec![
"<tool_c",
"all><function=",
r#"get_weather><param"#,
r#"eter=city>Bei"#,
"jing</parameter></func",
"tion></tool_call>",
];
let mut found_name = false;
let mut buffer = String::new();
for chunk in chunks {
let result = parser.parse_incremental(chunk, &tools).await.unwrap();
buffer.push_str(&result.normal_text);
for call in result.calls {
if let Some(name) = call.name {
assert_eq!(name, "get_weather");
found_name = true;
}
}
}
assert!(
found_name,
"Should have parsed function name from partial chunks"
);
}
#[tokio::test]
async fn test_qwen_coder_multiple_tools_boundary() {
let mut parser = QwenCoderParser::new();
let tools = create_test_tools();
// Tool boundary at chunk boundary
let chunks = vec![
r#"<tool_call><function=get_weather><parameter=city>Tokyo</parameter></function></tool_call>"#,
r#"<tool_call><function=search><parameter=query>weather forecast</parameter></function></tool_call>"#,
];
let mut tool_names = Vec::new();
for chunk in chunks {
let result = parser.parse_incremental(chunk, &tools).await.unwrap();
for call in result.calls {
if let Some(name) = call.name {
tool_names.push(name);
}
}
}
assert_eq!(tool_names.len(), 2);
assert_eq!(tool_names[0], "get_weather");
assert_eq!(tool_names[1], "search");
}
#[tokio::test]
async fn test_qwen_coder_invalid_function_name() {
let mut parser = QwenCoderParser::new();
let tools = create_test_tools();
let chunks = vec![
"<tool_call>",
r#"<function=invalid_function>"#,
r#"<parameter=param>value</parameter>"#,
"</function></tool_call>",
];
let mut found_invalid = false;
for chunk in chunks {
let result = parser.parse_incremental(chunk, &tools).await.unwrap();
// Invalid function should be skipped
for call in result.calls {
if let Some(name) = call.name {
if name == "invalid_function" {
found_invalid = true;
}
}
}
}
assert!(!found_invalid, "Invalid function should not be parsed");
}
#[tokio::test]
async fn test_qwen_coder_type_conversion() {
let parser = QwenCoderParser::new();
let input = r#"<tool_call>
<function=process>
<parameter=count>42</parameter>
<parameter=rate>1.5</parameter>
<parameter=enabled>true</parameter>
<parameter=data>null</parameter>
<parameter=text>string value</parameter>
</function>
</tool_call>"#;
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
// JSON values should be parsed
assert_eq!(args["count"], 42);
assert_eq!(args["rate"], 1.5);
assert_eq!(args["enabled"], true);
assert_eq!(args["data"], serde_json::Value::Null);
assert_eq!(args["text"], "string value");
}
#[tokio::test]
async fn test_qwen_coder_special_characters_in_values() {
let parser = QwenCoderParser::new();
let input = r#"<tool_call>
<function=process>
<parameter=text>Special chars: @#$%^&*()</parameter>
<parameter=emoji>🦀 Rust 🚀</parameter>
<parameter=quotes>"double" and 'single' quotes</parameter>
</function>
</tool_call>"#;
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert_eq!(args["text"], "Special chars: @#$%^&*()");
assert_eq!(args["emoji"], "🦀 Rust 🚀");
assert_eq!(args["quotes"], "\"double\" and 'single' quotes");
}
#[tokio::test]
async fn test_qwen_coder_whitespace_handling() {
let parser = QwenCoderParser::new();
// Test with various whitespace scenarios
let input = r#"<tool_call>
<function=process>
<parameter=trimmed> spaces around </parameter>
<parameter=newlines>
Line 1
Line 2
</parameter>
</function>
</tool_call>"#;
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
// Values should preserve internal whitespace but trim edges
assert_eq!(args["trimmed"], "spaces around");
assert!(args["newlines"].as_str().unwrap().contains("Line 1"));
assert!(args["newlines"].as_str().unwrap().contains("Line 2"));
}
#[tokio::test]
async fn test_qwen_coder_no_tools() {
// Test input with no tool calls at all
let parser = QwenCoderParser::new();
let input = r#"This is just a normal response without any tool calls.
I can provide information directly without using any tools.
Even if I mention function names like get_weather or search,
they are not actual tool calls unless properly formatted."#;
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
// No tools should be extracted
assert_eq!(
tools.len(),
0,
"Should not extract any tools from plain text"
);
// All content should be returned as normal text
assert_eq!(
normal_text, input,
"All content should be returned as normal text when no tools present"
);
}
#[tokio::test]
async fn test_qwen_coder_streaming_state_reset() {
let mut parser = QwenCoderParser::new();
let tools = create_test_tools();
// First tool
let chunks1 = vec![
r#"<tool_call><function=get_weather>"#,
r#"<parameter=city>London</parameter>"#,
"</function></tool_call>",
];
for chunk in chunks1 {
parser.parse_incremental(chunk, &tools).await.unwrap();
}
// Second tool - state should be reset
let chunks2 = vec![
r#"<tool_call><function=search>"#,
r#"<parameter=query>rust</parameter>"#,
"</function></tool_call>",
];
let mut second_tool_name = None;
for chunk in chunks2 {
let result = parser.parse_incremental(chunk, &tools).await.unwrap();
for call in result.calls {
if let Some(name) = call.name {
second_tool_name = Some(name);
}
}
}
assert_eq!(second_tool_name, Some("search".to_string()));
}
#[tokio::test]
async fn test_qwen_coder_realistic_chunks() {
let tools = create_test_tools();
let mut parser = QwenCoderParser::new();
let input = r#"<tool_call>
<function=get_weather>
<parameter=city>Tokyo</parameter>
<parameter=units>celsius</parameter>
</function>
</tool_call>"#;
let chunks = create_realistic_chunks(input);
assert!(chunks.len() > 20, "Should have many small chunks");
let mut got_tool_name = false;
for chunk in chunks {
let result = parser.parse_incremental(&chunk, &tools).await.unwrap();
for call in result.calls {
if let Some(name) = call.name {
assert_eq!(name, "get_weather");
got_tool_name = true;
}
}
}
assert!(got_tool_name, "Should have parsed tool name");
}
#[tokio::test]
async fn test_qwen_coder_xml_tag_arrives_in_parts() {
let tools = create_test_tools();
let mut parser = QwenCoderParser::new();
let chunks = vec![
"<to", "ol_", "cal", "l>", "<fun", "cti", "on=", "get", "_we", "ath", "er>", "<par", "ame",
"ter=", "cit", "y>", "Tok", "yo", "</", "par", "ame", "ter>", "</", "func", "tion>", "</",
"too", "l_c", "all>",
];
let mut got_tool_name = false;
for chunk in chunks {
let result = parser.parse_incremental(chunk, &tools).await.unwrap();
for call in result.calls {
if let Some(name) = call.name {
assert_eq!(name, "get_weather");
got_tool_name = true;
}
}
}
assert!(got_tool_name, "Should have parsed tool name");
}
#[tokio::test]
async fn test_qwen_coder_content_before_and_after_tool_calls() {
let parser = QwenCoderParser::new();
let input = r#"I'll analyze the weather for you now.
<tool_call>
<function=get_weather>
<parameter=city>Boston</parameter>
<parameter=state>MA</parameter>
</function>
</tool_call>
Based on the analysis, here's what I found."#;
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
// Verify tool extraction
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, "get_weather");
// Verify content preservation (only text before tool call is returned)
assert!(normal_text.contains("I'll analyze the weather for you now."));
// Text after tool call is not included in parse_complete
assert!(!normal_text.contains("Based on the analysis, here's what I found."));
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert_eq!(args["city"], "Boston");
assert_eq!(args["state"], "MA");
}
#[tokio::test]
async fn test_qwen_coder_incomplete_tool_call() {
let parser = QwenCoderParser::new();
// Incomplete tool call - missing closing tag
let input = r#"<tool_call>
<function=get_weather>
<parameter=city>Chicago</parameter>"#;
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
// Should not extract incomplete tool calls
assert_eq!(tools.len(), 0);
assert_eq!(normal_text, input); // Should return as normal text
}
#[tokio::test]
async fn test_qwen_coder_malformed_function_tag() {
let parser = QwenCoderParser::new();
// Malformed function tag - missing name attribute
let input = r#"<tool_call>
<function>
<parameter=city>Miami</parameter>
</function>
</tool_call>"#;
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
// Should not extract tool calls with malformed function tags
assert_eq!(tools.len(), 0);
assert_eq!(normal_text, input);
}
#[tokio::test]
async fn test_qwen_coder_many_parameters() {
let parser = QwenCoderParser::new();
let mut params_xml = String::new();
for i in 1..=20 {
params_xml.push_str(&format!(
r#"<parameter=param{}>value{}</parameter>
"#,
i, i
));
}
let input = format!(
r#"<tool_call>
<function=complex_func>
{}
</function>
</tool_call>"#,
params_xml
);
let (_normal_text, tools) = parser.parse_complete(&input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, "complex_func");
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
// Verify all 20 parameters are parsed
for i in 1..=20 {
let key = format!("param{}", i);
let expected_value = format!("value{}", i);
assert_eq!(args[key], expected_value);
}
}
// ============================================================================
// Edge Case Tests
// ============================================================================
#[tokio::test]
async fn test_qwen_coder_malformed_xml_missing_parameter_close() {
let parser = QwenCoderParser::new();
// Missing </parameter> closing tag - parser regex won't match incomplete parameter
let input = r#"<tool_call>
<function=get_weather>
<parameter=city>Beijing
</function>
</tool_call>"#;
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
// The parser extracts the tool call but with empty arguments since
// the parameter block is malformed (no </parameter>)
// This is acceptable behavior - we extract what we can
if tools.is_empty() {
// If no tools extracted, input returned as normal text
assert_eq!(normal_text, input);
} else {
// If tool extracted, it should have the function name
assert_eq!(tools[0].function.name, "get_weather");
}
}
#[tokio::test]
async fn test_qwen_coder_malformed_xml_unclosed_function() {
let parser = QwenCoderParser::new();
// Missing </function> closing tag
let input = r#"<tool_call>
<function=get_weather>
<parameter=city>Beijing</parameter>
</tool_call>"#;
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
// Parser should still extract the tool since it has complete tool_call tags
// and the function name + parameters are present
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, "get_weather");
}
#[tokio::test]
async fn test_qwen_coder_malformed_xml_nested_tool_calls() {
let parser = QwenCoderParser::new();
// Nested tool_call tags (invalid)
let input = r#"<tool_call>
<function=outer>
<tool_call>
<function=inner>
</function>
</tool_call>
</function>
</tool_call>"#;
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
// Should handle gracefully - may parse first complete tool_call
// The exact behavior depends on regex matching
assert!(tools.len() <= 1);
}
#[tokio::test]
async fn test_qwen_coder_unicode_parameter_names() {
let parser = QwenCoderParser::new();
// Unicode characters in parameter names (Chinese, Japanese, emoji)
let input = r#"<tool_call>
<function=process>
<parameter=城市>北京</parameter>
<parameter=天気>晴れ</parameter>
<parameter=emoji_key>🌍🌎🌏</parameter>
</function>
</tool_call>"#;
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, "process");
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert_eq!(args["城市"], "北京");
assert_eq!(args["天気"], "晴れ");
assert_eq!(args["emoji_key"], "🌍🌎🌏");
}
#[tokio::test]
async fn test_qwen_coder_unicode_function_name() {
let parser = QwenCoderParser::new();
// Unicode function name
let input = r#"<tool_call>
<function=获取天气>
<parameter=location>上海</parameter>
</function>
</tool_call>"#;
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, "获取天气");
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert_eq!(args["location"], "上海");
}
#[tokio::test]
async fn test_qwen_coder_very_large_parameter_value() {
let parser = QwenCoderParser::new();
// Generate a large parameter value (100KB)
let large_value: String = "x".repeat(100_000);
let input = format!(
r#"<tool_call>
<function=process_large>
<parameter=data>{}</parameter>
</function>
</tool_call>"#,
large_value
);
let (_normal_text, tools) = parser.parse_complete(&input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, "process_large");
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert_eq!(args["data"].as_str().unwrap().len(), 100_000);
}
#[tokio::test]
async fn test_qwen_coder_very_large_nested_json_parameter() {
let parser = QwenCoderParser::new();
// Generate moderately nested JSON structure (10 levels to avoid stack overflow)
let mut nested_json = String::from(r#"{"level": 0}"#);
for i in 1..=10 {
nested_json = format!(r#"{{"level": {}, "child": {}}}"#, i, nested_json);
}
let input = format!(
r#"<tool_call>
<function=process_nested>
<parameter=config>{}</parameter>
</function>
</tool_call>"#,
nested_json
);
let (_normal_text, tools) = parser.parse_complete(&input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, "process_nested");
// Verify the nested JSON was parsed correctly
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert!(args["config"].is_object());
assert_eq!(args["config"]["level"], 10);
}
#[tokio::test]
async fn test_qwen_coder_streaming_malformed_recovery() {
let mut parser = QwenCoderParser::new();
let tools = create_test_tools();
// First: malformed tool call (invalid function name)
// Second: valid tool call
let chunks = vec![
r#"<tool_call><function=invalid_func><parameter=x>1</parameter></function></tool_call>"#,
r#"<tool_call><function=get_weather><parameter=city>Tokyo</parameter></function></tool_call>"#,
];
let mut valid_tool_found = false;
for chunk in chunks {
let result = parser.parse_incremental(chunk, &tools).await.unwrap();
for call in result.calls {
if let Some(name) = call.name {
if name == "get_weather" {
valid_tool_found = true;
}
}
}
}
assert!(
valid_tool_found,
"Should recover and parse valid tool after invalid one"
);
}
#[tokio::test]
async fn test_qwen_coder_parameter_with_xml_like_content() {
let parser = QwenCoderParser::new();
// Parameter value contains XML-like content that shouldn't be parsed as tags
let input = r#"<tool_call>
<function=process>
<parameter=html_content><div class="test"><span>Hello</span></div></parameter>
<parameter=xml_snippet><root><child attr="value"/></root></parameter>
</function>
</tool_call>"#;
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert!(args["html_content"]
.as_str()
.unwrap()
.contains("<div class=\"test\">"));
assert!(args["xml_snippet"]
.as_str()
.unwrap()
.contains("<root><child"));
}
#[tokio::test]
async fn test_qwen_coder_empty_parameter_value() {
let parser = QwenCoderParser::new();
let input = r#"<tool_call>
<function=process>
<parameter=empty></parameter>
<parameter=whitespace> </parameter>
<parameter=normal>value</parameter>
</function>
</tool_call>"#;
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert_eq!(args["empty"], "");
assert_eq!(args["whitespace"], ""); // Trimmed
assert_eq!(args["normal"], "value");
}
// ============================================================================
// HTML Entity and Python Literal Tests
// ============================================================================
#[tokio::test]
async fn test_qwen_coder_html_entity_decoding() {
let parser = QwenCoderParser::new();
// Test HTML entities in parameter values
let input = r#"<tool_call>
<function=process>
<parameter=ampersand>Tom &amp; Jerry</parameter>
<parameter=comparison>5 &lt; 10 &amp;&amp; 10 &gt; 5</parameter>
<parameter=quotes>&quot;Hello&quot; &amp; &apos;World&apos;</parameter>
</function>
</tool_call>"#;
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert_eq!(args["ampersand"], "Tom & Jerry");
assert_eq!(args["comparison"], "5 < 10 && 10 > 5");
assert_eq!(args["quotes"], "\"Hello\" & 'World'");
}
#[tokio::test]
async fn test_qwen_coder_html_numeric_entities() {
let parser = QwenCoderParser::new();
// Test numeric HTML entities
let input = r#"<tool_call>
<function=process>
<parameter=decimal>&#60;tag&#62;</parameter>
<parameter=hex>&#x3C;tag&#x3E;</parameter>
</function>
</tool_call>"#;
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert_eq!(args["decimal"], "<tag>");
assert_eq!(args["hex"], "<tag>");
}
#[tokio::test]
async fn test_qwen_coder_python_literals() {
let parser = QwenCoderParser::new();
// Test Python-style literals (True, False, None)
let input = r#"<tool_call>
<function=process>
<parameter=py_true>True</parameter>
<parameter=py_false>False</parameter>
<parameter=py_none>None</parameter>
<parameter=json_true>true</parameter>
<parameter=json_false>false</parameter>
<parameter=json_null>null</parameter>
</function>
</tool_call>"#;
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
// Python literals should be converted
assert_eq!(args["py_true"], true);
assert_eq!(args["py_false"], false);
assert_eq!(args["py_none"], serde_json::Value::Null);
// JSON literals should also work
assert_eq!(args["json_true"], true);
assert_eq!(args["json_false"], false);
assert_eq!(args["json_null"], serde_json::Value::Null);
}
#[tokio::test]
async fn test_qwen_coder_mixed_html_and_json() {
let parser = QwenCoderParser::new();
// Test HTML entities within JSON structures
let input = r#"<tool_call>
<function=search>
<parameter=query>price &lt; 100 &amp;&amp; rating &gt; 4</parameter>
<parameter=config>{"operator": "&amp;&amp;", "escape": true}</parameter>
</function>
</tool_call>"#;
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert_eq!(args["query"], "price < 100 && rating > 4");
// JSON with HTML entity inside - the entity gets decoded first, then JSON parsed
assert!(args["config"].is_object());
assert_eq!(args["config"]["operator"], "&&");
}