".to_string()));
+ assert_eq!(
+ safe_val("Tom & Jerry"),
+ Value::String("Tom & Jerry".to_string())
+ );
+ }
+}
diff --git a/sgl-model-gateway/src/tool_parser/tests.rs b/sgl-model-gateway/src/tool_parser/tests.rs
index 49d271723..5cfdb3746 100644
--- a/sgl-model-gateway/src/tool_parser/tests.rs
+++ b/sgl-model-gateway/src/tool_parser/tests.rs
@@ -1,5 +1,12 @@
use super::*;
-use crate::tool_parser::{parsers::JsonParser, partial_json::PartialJson, traits::ToolParser};
+use crate::{
+ protocols::common::{Function, Tool},
+ tool_parser::{
+ parsers::{JsonParser, QwenCoderParser},
+ partial_json::PartialJson,
+ traits::ToolParser,
+ },
+};
#[tokio::test]
async fn test_tool_parser_factory() {
@@ -562,3 +569,172 @@ mod stress_tests {
}
}
}
+
+#[cfg(test)]
+mod qwen_coder_tests {
+ use super::*;
+
+ fn create_test_tools() -> Vec
{
+ vec![Tool {
+ tool_type: "function".to_string(),
+ function: Function {
+ name: "get_weather".to_string(),
+ description: Some("Get weather information".to_string()),
+ parameters: serde_json::json!({
+ "type": "object",
+ "properties": {
+ "city": {"type": "string"},
+ "units": {"type": "string"}
+ }
+ }),
+ strict: None,
+ },
+ }]
+ }
+
+ #[tokio::test]
+ async fn test_qwen_coder_incremental_parameter_streaming() {
+ let mut parser = QwenCoderParser::new();
+ let tools = create_test_tools();
+
+ let chunks = [
+ "",
+ r#""#,
+ r#"Paris"#,
+ r#"metric"#,
+ "",
+ ];
+
+ let mut all_calls = Vec::new();
+
+ // Process each chunk
+ for (i, chunk) in chunks.iter().enumerate() {
+ let result = parser.parse_incremental(chunk, &tools).await.unwrap();
+ println!("Chunk {}: {:?}", i, chunk);
+ println!(" Calls: {:?}", result.calls);
+ println!(" Normal text: {:?}", result.normal_text);
+
+ for call in &result.calls {
+ all_calls.push(call.clone());
+ }
+ }
+
+ // Verify the final result
+ // We should have:
+ // 1. Tool name call (from chunk 2)
+ // 2. First parameter call: {"city": "Paris"}
+ // 3. Second parameter call: , "units": "metric"
+ // Final result should be: {"city": "Paris", "units": "metric"}
+
+ assert!(!all_calls.is_empty(), "Should have at least one call");
+
+ // Check that we have the tool name
+ let name_call = all_calls.iter().find(|c| c.name.is_some());
+ assert!(name_call.is_some(), "Should have tool name call");
+ assert_eq!(name_call.unwrap().name.as_ref().unwrap(), "get_weather");
+
+ // Check parameter calls
+ let param_calls: Vec<_> = all_calls.iter().filter(|c| c.name.is_none()).collect();
+ assert!(!param_calls.is_empty(), "Should have parameter calls");
+
+ // Verify final arguments format by concatenating all parameter fragments
+ let params_str: String = param_calls.iter().map(|c| c.parameters.as_str()).collect();
+ println!("Final streamed args: {}", params_str);
+
+ // Should contain both city and units parameters
+ assert!(params_str.contains("city"), "Should contain city parameter");
+ assert!(params_str.contains("Paris"), "Should contain Paris value");
+ assert!(
+ params_str.contains("units"),
+ "Should contain units parameter"
+ );
+ assert!(params_str.contains("metric"), "Should contain metric value");
+ }
+
+ #[tokio::test]
+ async fn test_qwen_coder_incremental_parameter_streaming_with_partial_values() {
+ let mut parser = QwenCoderParser::new();
+ let tools = create_test_tools();
+
+ // Test with parameter values that arrive in multiple chunks
+ // This tests the buffering logic for partial XML tags
+ let chunks = [
+ "",
+ r#"Paris"#,
+ r#"metric"#,
+ "",
+ ];
+
+ let mut all_calls = Vec::new();
+
+ // Process each chunk
+ for (i, chunk) in chunks.iter().enumerate() {
+ let result = parser.parse_incremental(chunk, &tools).await.unwrap();
+ println!("Chunk {}: {:?}", i, chunk);
+ println!(" Calls: {:?}", result.calls);
+
+ for call in &result.calls {
+ all_calls.push(call.clone());
+ }
+ }
+
+ // Verify we got the tool name
+ let name_call = all_calls.iter().find(|c| c.name.is_some());
+ assert!(name_call.is_some(), "Should have tool name call");
+ assert_eq!(name_call.unwrap().name.as_ref().unwrap(), "get_weather");
+
+ // Verify we got parameter calls
+ let param_calls: Vec<_> = all_calls.iter().filter(|c| c.name.is_none()).collect();
+ assert!(!param_calls.is_empty(), "Should have parameter calls");
+
+ // Verify final arguments by concatenating all parameter fragments
+ let params_str: String = param_calls.iter().map(|c| c.parameters.as_str()).collect();
+ assert!(params_str.contains("city"), "Should contain city parameter");
+ assert!(params_str.contains("Paris"), "Should contain Paris value");
+ }
+
+ #[tokio::test]
+ async fn test_qwen_coder_nested_json_parameter() {
+ let mut parser = QwenCoderParser::new();
+ let tools = vec![Tool {
+ tool_type: "function".to_string(),
+ function: Function {
+ name: "test_function".to_string(),
+ description: None,
+ parameters: serde_json::json!({
+ "type": "object",
+ "properties": {
+ "nested": {"type": "object"}
+ }
+ }),
+ strict: None,
+ },
+ }];
+
+ let chunks = vec![
+ "",
+ r#""#,
+ r#"{"key": "value"}"#,
+ "",
+ ];
+
+ let mut all_calls = Vec::new();
+ for chunk in chunks {
+ let result = parser.parse_incremental(chunk, &tools).await.unwrap();
+ for call in result.calls {
+ all_calls.push(call);
+ }
+ }
+
+ // Verify nested JSON is parsed correctly via collected calls
+ let param_calls: Vec<_> = all_calls.iter().filter(|c| c.name.is_none()).collect();
+ assert!(!param_calls.is_empty(), "Should have parameter calls");
+
+ // The first parameter call should contain the nested JSON
+ let params_str: String = param_calls.iter().map(|c| c.parameters.as_str()).collect();
+ assert!(
+ params_str.contains("nested"),
+ "Should contain nested parameter"
+ );
+ }
+}
diff --git a/sgl-model-gateway/tests/tool_parser/mod.rs b/sgl-model-gateway/tests/tool_parser/mod.rs
index 55d4082c8..b8187f3df 100644
--- a/sgl-model-gateway/tests/tool_parser/mod.rs
+++ b/sgl-model-gateway/tests/tool_parser/mod.rs
@@ -14,4 +14,5 @@ pub mod tool_parser_mixed_edge_cases;
pub mod tool_parser_partial_json;
pub mod tool_parser_pythonic;
pub mod tool_parser_qwen;
+pub mod tool_parser_qwen_coder;
pub mod tool_parser_step3;
diff --git a/sgl-model-gateway/tests/tool_parser/tool_parser_qwen_coder.rs b/sgl-model-gateway/tests/tool_parser/tool_parser_qwen_coder.rs
new file mode 100644
index 000000000..548b00f9d
--- /dev/null
+++ b/sgl-model-gateway/tests/tool_parser/tool_parser_qwen_coder.rs
@@ -0,0 +1,958 @@
+//! Qwen Coder Parser Integration Tests
+//!
+//! Tests for the Qwen Coder parser which handles XML format:
+//! \n\nvalue\n\n
+
+use serde_json::json;
+use smg::tool_parser::{parsers::QwenCoderParser, traits::ToolParser};
+
+use crate::common::{create_test_tools, streaming_helpers::*};
+
+#[tokio::test]
+async fn test_qwen_coder_single_tool() {
+ let parser = QwenCoderParser::new();
+ let input = r#"
+
+Beijing
+celsius
+
+"#;
+
+ let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
+ assert_eq!(tools.len(), 1);
+ assert_eq!(tools[0].function.name, "get_weather");
+
+ let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
+ assert_eq!(args["city"], "Beijing");
+ assert_eq!(args["units"], "celsius");
+}
+
+#[tokio::test]
+async fn test_qwen_coder_multiple_sequential_tools() {
+ let parser = QwenCoderParser::new();
+ let input = r#"Let me help you with that.
+
+
+Qwen model
+
+
+
+
+Hello
+zh
+
+"#;
+
+ let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
+ assert_eq!(tools.len(), 2);
+ assert_eq!(normal_text, "Let me help you with that.\n");
+ assert_eq!(tools[0].function.name, "search");
+ assert_eq!(tools[1].function.name, "translate");
+}
+
+#[tokio::test]
+async fn test_qwen_coder_nested_json_in_parameters() {
+ let parser = QwenCoderParser::new();
+ let input = r#"
+
+{"nested": {"value": [1, 2, 3]}}
+true
+
+"#;
+
+ let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
+ assert_eq!(tools.len(), 1);
+ assert_eq!(tools[0].function.name, "process_data");
+
+ let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
+ // JSON values should be parsed
+ assert_eq!(args["config"]["nested"]["value"], json!([1, 2, 3]));
+ assert_eq!(args["enabled"], true);
+}
+
+#[tokio::test]
+async fn test_qwen_coder_string_parameters() {
+ let parser = QwenCoderParser::new();
+ let input = r#"
+
+Hello World
+42
+
+"#;
+
+ let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
+ assert_eq!(tools.len(), 1);
+
+ let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
+ assert_eq!(args["text"], "Hello World");
+ // JSON numbers should be parsed as numbers (consistent with Python's json.loads)
+ assert_eq!(args["number"], 42);
+}
+
+#[tokio::test]
+async fn test_qwen_coder_empty_arguments() {
+ let parser = QwenCoderParser::new();
+ let input = r#"
+
+
+"#;
+
+ let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
+ assert_eq!(tools.len(), 1);
+ assert_eq!(tools[0].function.name, "get_time");
+
+ let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
+ assert_eq!(args, json!({}));
+}
+
+#[tokio::test]
+async fn test_qwen_coder_multiline_parameter_values() {
+ let parser = QwenCoderParser::new();
+ let input = r#"
+
+Line 1
+Line 2
+Line 3
+/tmp/test.txt
+
+"#;
+
+ let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
+ assert_eq!(tools.len(), 1);
+
+ let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
+ assert_eq!(args["content"], "Line 1\nLine 2\nLine 3");
+ assert_eq!(args["path"], "/tmp/test.txt");
+}
+
+#[tokio::test]
+async fn test_qwen_coder_format_detection() {
+ let parser = QwenCoderParser::new();
+
+ assert!(parser.has_tool_markers(""));
+ assert!(parser.has_tool_markers("Some text "));
+ assert!(!parser.has_tool_markers("Just plain text"));
+ assert!(!parser.has_tool_markers("")); // Without tool_call tags
+}
+
+#[tokio::test]
+async fn test_qwen_coder_incomplete_tags() {
+ let parser = QwenCoderParser::new();
+
+ // Missing closing tag
+ let input = r#"
+
+Beijing"#;
+ let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
+ assert_eq!(tools.len(), 0);
+
+ // Missing opening tag
+ let input = r#"Beijing
+
+"#;
+ let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
+ assert_eq!(tools.len(), 0);
+}
+
+#[tokio::test]
+async fn test_qwen_coder_streaming_basic() {
+ let mut parser = QwenCoderParser::new();
+ let tools = create_test_tools();
+
+ // Simulate streaming chunks
+ let chunks = vec![
+ "",
+ r#""#,
+ r#"Shanghai"#,
+ r#"celsius"#,
+ "",
+ "",
+ ];
+
+ let mut found_name = false;
+ let mut found_params = false;
+
+ for chunk in chunks {
+ let result = parser.parse_incremental(chunk, &tools).await.unwrap();
+
+ for call in result.calls {
+ if let Some(name) = call.name {
+ assert_eq!(name, "get_weather");
+ found_name = true;
+ }
+ if !call.parameters.is_empty() {
+ found_params = true;
+ }
+ }
+ }
+
+ assert!(found_name, "Should have found tool name during streaming");
+ assert!(found_params, "Should have streamed parameters");
+}
+
+#[tokio::test]
+async fn test_qwen_coder_streaming_incremental_json() {
+ let mut parser = QwenCoderParser::new();
+ let tools = create_test_tools();
+
+ let chunks = vec![
+ "",
+ r#""#,
+ r#"Paris"#,
+ r#"metric"#,
+ "",
+ ];
+
+ let mut json_fragments = Vec::new();
+ let mut found_function = false;
+
+ for chunk in chunks {
+ let result = parser.parse_incremental(chunk, &tools).await.unwrap();
+
+ for call in result.calls {
+ if let Some(_name) = call.name {
+ found_function = true;
+ }
+ if !call.parameters.is_empty() {
+ json_fragments.push(call.parameters.clone());
+ }
+ }
+ }
+
+ assert!(found_function);
+
+ // Verify JSON was built incrementally
+ assert!(!json_fragments.is_empty());
+
+ // First fragment should start with opening brace
+ if let Some(first) = json_fragments.first() {
+ assert!(
+ first.starts_with('{'),
+ "First JSON fragment should start with '{{': {}",
+ first
+ );
+ }
+}
+
+#[tokio::test]
+async fn test_qwen_coder_streaming_partial_tags() {
+ let mut parser = QwenCoderParser::new();
+ let tools = create_test_tools();
+
+ // Chunks split mid-tag
+ let chunks = vec![
+ "Bei"#,
+ "jing",
+ ];
+
+ let mut found_name = false;
+ let mut buffer = String::new();
+
+ for chunk in chunks {
+ let result = parser.parse_incremental(chunk, &tools).await.unwrap();
+
+ buffer.push_str(&result.normal_text);
+
+ for call in result.calls {
+ if let Some(name) = call.name {
+ assert_eq!(name, "get_weather");
+ found_name = true;
+ }
+ }
+ }
+
+ assert!(
+ found_name,
+ "Should have parsed function name from partial chunks"
+ );
+}
+
+#[tokio::test]
+async fn test_qwen_coder_multiple_tools_boundary() {
+ let mut parser = QwenCoderParser::new();
+ let tools = create_test_tools();
+
+ // Tool boundary at chunk boundary
+ let chunks = vec![
+ r#"Tokyo"#,
+ r#"weather forecast"#,
+ ];
+
+ let mut tool_names = Vec::new();
+
+ for chunk in chunks {
+ let result = parser.parse_incremental(chunk, &tools).await.unwrap();
+
+ for call in result.calls {
+ if let Some(name) = call.name {
+ tool_names.push(name);
+ }
+ }
+ }
+
+ assert_eq!(tool_names.len(), 2);
+ assert_eq!(tool_names[0], "get_weather");
+ assert_eq!(tool_names[1], "search");
+}
+
+#[tokio::test]
+async fn test_qwen_coder_invalid_function_name() {
+ let mut parser = QwenCoderParser::new();
+ let tools = create_test_tools();
+
+ let chunks = vec![
+ "",
+ r#""#,
+ r#"value"#,
+ "",
+ ];
+
+ let mut found_invalid = false;
+
+ for chunk in chunks {
+ let result = parser.parse_incremental(chunk, &tools).await.unwrap();
+
+ // Invalid function should be skipped
+ for call in result.calls {
+ if let Some(name) = call.name {
+ if name == "invalid_function" {
+ found_invalid = true;
+ }
+ }
+ }
+ }
+
+ assert!(!found_invalid, "Invalid function should not be parsed");
+}
+
+#[tokio::test]
+async fn test_qwen_coder_type_conversion() {
+ let parser = QwenCoderParser::new();
+
+ let input = r#"
+
+42
+1.5
+true
+null
+string value
+
+"#;
+
+ let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
+ assert_eq!(tools.len(), 1);
+
+ let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
+ // JSON values should be parsed
+ assert_eq!(args["count"], 42);
+ assert_eq!(args["rate"], 1.5);
+ assert_eq!(args["enabled"], true);
+ assert_eq!(args["data"], serde_json::Value::Null);
+ assert_eq!(args["text"], "string value");
+}
+
+#[tokio::test]
+async fn test_qwen_coder_special_characters_in_values() {
+ let parser = QwenCoderParser::new();
+
+ let input = r#"
+
+Special chars: @#$%^&*()
+π¦ Rust π
+"double" and 'single' quotes
+
+"#;
+
+ let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
+ assert_eq!(tools.len(), 1);
+
+ let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
+ assert_eq!(args["text"], "Special chars: @#$%^&*()");
+ assert_eq!(args["emoji"], "π¦ Rust π");
+ assert_eq!(args["quotes"], "\"double\" and 'single' quotes");
+}
+
+#[tokio::test]
+async fn test_qwen_coder_whitespace_handling() {
+ let parser = QwenCoderParser::new();
+
+ // Test with various whitespace scenarios
+ let input = r#"
+
+ spaces around
+
+ Line 1
+ Line 2
+
+
+"#;
+
+ let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
+ assert_eq!(tools.len(), 1);
+
+ let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
+ // Values should preserve internal whitespace but trim edges
+ assert_eq!(args["trimmed"], "spaces around");
+ assert!(args["newlines"].as_str().unwrap().contains("Line 1"));
+ assert!(args["newlines"].as_str().unwrap().contains("Line 2"));
+}
+
+#[tokio::test]
+async fn test_qwen_coder_no_tools() {
+ // Test input with no tool calls at all
+ let parser = QwenCoderParser::new();
+
+ let input = r#"This is just a normal response without any tool calls.
+I can provide information directly without using any tools.
+Even if I mention function names like get_weather or search,
+they are not actual tool calls unless properly formatted."#;
+
+ let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
+
+ // No tools should be extracted
+ assert_eq!(
+ tools.len(),
+ 0,
+ "Should not extract any tools from plain text"
+ );
+
+ // All content should be returned as normal text
+ assert_eq!(
+ normal_text, input,
+ "All content should be returned as normal text when no tools present"
+ );
+}
+
+#[tokio::test]
+async fn test_qwen_coder_streaming_state_reset() {
+ let mut parser = QwenCoderParser::new();
+ let tools = create_test_tools();
+
+ // First tool
+ let chunks1 = vec![
+ r#""#,
+ r#"London"#,
+ "",
+ ];
+
+ for chunk in chunks1 {
+ parser.parse_incremental(chunk, &tools).await.unwrap();
+ }
+
+ // Second tool - state should be reset
+ let chunks2 = vec![
+ r#""#,
+ r#"rust"#,
+ "",
+ ];
+
+ let mut second_tool_name = None;
+ for chunk in chunks2 {
+ let result = parser.parse_incremental(chunk, &tools).await.unwrap();
+ for call in result.calls {
+ if let Some(name) = call.name {
+ second_tool_name = Some(name);
+ }
+ }
+ }
+
+ assert_eq!(second_tool_name, Some("search".to_string()));
+}
+
+#[tokio::test]
+async fn test_qwen_coder_realistic_chunks() {
+ let tools = create_test_tools();
+ let mut parser = QwenCoderParser::new();
+
+ let input = r#"
+
+Tokyo
+celsius
+
+"#;
+ let chunks = create_realistic_chunks(input);
+
+ assert!(chunks.len() > 20, "Should have many small chunks");
+
+ let mut got_tool_name = false;
+
+ for chunk in chunks {
+ let result = parser.parse_incremental(&chunk, &tools).await.unwrap();
+ for call in result.calls {
+ if let Some(name) = call.name {
+ assert_eq!(name, "get_weather");
+ got_tool_name = true;
+ }
+ }
+ }
+
+ assert!(got_tool_name, "Should have parsed tool name");
+}
+
+#[tokio::test]
+async fn test_qwen_coder_xml_tag_arrives_in_parts() {
+ let tools = create_test_tools();
+ let mut parser = QwenCoderParser::new();
+
+ let chunks = vec![
+ "", "", "", "Tok", "yo", "", "", "",
+ ];
+
+ let mut got_tool_name = false;
+
+ for chunk in chunks {
+ let result = parser.parse_incremental(chunk, &tools).await.unwrap();
+ for call in result.calls {
+ if let Some(name) = call.name {
+ assert_eq!(name, "get_weather");
+ got_tool_name = true;
+ }
+ }
+ }
+
+ assert!(got_tool_name, "Should have parsed tool name");
+}
+
+#[tokio::test]
+async fn test_qwen_coder_content_before_and_after_tool_calls() {
+ let parser = QwenCoderParser::new();
+
+ let input = r#"I'll analyze the weather for you now.
+
+
+Boston
+MA
+
+
+Based on the analysis, here's what I found."#;
+
+ let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
+
+ // Verify tool extraction
+ assert_eq!(tools.len(), 1);
+ assert_eq!(tools[0].function.name, "get_weather");
+
+ // Verify content preservation (only text before tool call is returned)
+ assert!(normal_text.contains("I'll analyze the weather for you now."));
+ // Text after tool call is not included in parse_complete
+ assert!(!normal_text.contains("Based on the analysis, here's what I found."));
+
+ let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
+ assert_eq!(args["city"], "Boston");
+ assert_eq!(args["state"], "MA");
+}
+
+#[tokio::test]
+async fn test_qwen_coder_incomplete_tool_call() {
+ let parser = QwenCoderParser::new();
+
+ // Incomplete tool call - missing closing tag
+ let input = r#"
+
+Chicago"#;
+
+ let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
+
+ // Should not extract incomplete tool calls
+ assert_eq!(tools.len(), 0);
+ assert_eq!(normal_text, input); // Should return as normal text
+}
+
+#[tokio::test]
+async fn test_qwen_coder_malformed_function_tag() {
+ let parser = QwenCoderParser::new();
+
+ // Malformed function tag - missing name attribute
+ let input = r#"
+
+Miami
+
+"#;
+
+ let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
+
+ // Should not extract tool calls with malformed function tags
+ assert_eq!(tools.len(), 0);
+ assert_eq!(normal_text, input);
+}
+
+#[tokio::test]
+async fn test_qwen_coder_many_parameters() {
+ let parser = QwenCoderParser::new();
+
+ let mut params_xml = String::new();
+ for i in 1..=20 {
+ params_xml.push_str(&format!(
+ r#"value{}
+"#,
+ i, i
+ ));
+ }
+
+ let input = format!(
+ r#"
+
+{}
+
+"#,
+ params_xml
+ );
+
+ let (_normal_text, tools) = parser.parse_complete(&input).await.unwrap();
+ assert_eq!(tools.len(), 1);
+ assert_eq!(tools[0].function.name, "complex_func");
+
+ let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
+
+ // Verify all 20 parameters are parsed
+ for i in 1..=20 {
+ let key = format!("param{}", i);
+ let expected_value = format!("value{}", i);
+ assert_eq!(args[key], expected_value);
+ }
+}
+
+// ============================================================================
+// Edge Case Tests
+// ============================================================================
+
+#[tokio::test]
+async fn test_qwen_coder_malformed_xml_missing_parameter_close() {
+ let parser = QwenCoderParser::new();
+
+ // Missing closing tag - parser regex won't match incomplete parameter
+ let input = r#"
+
+Beijing
+
+"#;
+
+ let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
+
+ // The parser extracts the tool call but with empty arguments since
+ // the parameter block is malformed (no )
+ // This is acceptable behavior - we extract what we can
+ if tools.is_empty() {
+ // If no tools extracted, input returned as normal text
+ assert_eq!(normal_text, input);
+ } else {
+ // If tool extracted, it should have the function name
+ assert_eq!(tools[0].function.name, "get_weather");
+ }
+}
+
+#[tokio::test]
+async fn test_qwen_coder_malformed_xml_unclosed_function() {
+ let parser = QwenCoderParser::new();
+
+ // Missing closing tag
+ let input = r#"
+
+Beijing
+"#;
+
+ let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
+
+ // Parser should still extract the tool since it has complete tool_call tags
+ // and the function name + parameters are present
+ assert_eq!(tools.len(), 1);
+ assert_eq!(tools[0].function.name, "get_weather");
+}
+
+#[tokio::test]
+async fn test_qwen_coder_malformed_xml_nested_tool_calls() {
+ let parser = QwenCoderParser::new();
+
+ // Nested tool_call tags (invalid)
+ let input = r#"
+
+
+
+
+
+
+"#;
+
+ let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
+
+ // Should handle gracefully - may parse first complete tool_call
+ // The exact behavior depends on regex matching
+ assert!(tools.len() <= 1);
+}
+
+#[tokio::test]
+async fn test_qwen_coder_unicode_parameter_names() {
+ let parser = QwenCoderParser::new();
+
+ // Unicode characters in parameter names (Chinese, Japanese, emoji)
+ let input = r#"
+
+εδΊ¬
+ζ΄γ
+πππ
+
+"#;
+
+ let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
+ assert_eq!(tools.len(), 1);
+ assert_eq!(tools[0].function.name, "process");
+
+ let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
+ assert_eq!(args["εεΈ"], "εδΊ¬");
+ assert_eq!(args["倩ζ°"], "ζ΄γ");
+ assert_eq!(args["emoji_key"], "πππ");
+}
+
+#[tokio::test]
+async fn test_qwen_coder_unicode_function_name() {
+ let parser = QwenCoderParser::new();
+
+ // Unicode function name
+ let input = r#"
+
+δΈζ΅·
+
+"#;
+
+ let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
+ assert_eq!(tools.len(), 1);
+ assert_eq!(tools[0].function.name, "θ·ε倩ζ°");
+
+ let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
+ assert_eq!(args["location"], "δΈζ΅·");
+}
+
+#[tokio::test]
+async fn test_qwen_coder_very_large_parameter_value() {
+ let parser = QwenCoderParser::new();
+
+ // Generate a large parameter value (100KB)
+ let large_value: String = "x".repeat(100_000);
+
+ let input = format!(
+ r#"
+
+{}
+
+"#,
+ large_value
+ );
+
+ let (_normal_text, tools) = parser.parse_complete(&input).await.unwrap();
+ assert_eq!(tools.len(), 1);
+ assert_eq!(tools[0].function.name, "process_large");
+
+ let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
+ assert_eq!(args["data"].as_str().unwrap().len(), 100_000);
+}
+
+#[tokio::test]
+async fn test_qwen_coder_very_large_nested_json_parameter() {
+ let parser = QwenCoderParser::new();
+
+ // Generate moderately nested JSON structure (10 levels to avoid stack overflow)
+ let mut nested_json = String::from(r#"{"level": 0}"#);
+ for i in 1..=10 {
+ nested_json = format!(r#"{{"level": {}, "child": {}}}"#, i, nested_json);
+ }
+
+ let input = format!(
+ r#"
+
+{}
+
+"#,
+ nested_json
+ );
+
+ let (_normal_text, tools) = parser.parse_complete(&input).await.unwrap();
+ assert_eq!(tools.len(), 1);
+ assert_eq!(tools[0].function.name, "process_nested");
+
+ // Verify the nested JSON was parsed correctly
+ let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
+ assert!(args["config"].is_object());
+ assert_eq!(args["config"]["level"], 10);
+}
+
+#[tokio::test]
+async fn test_qwen_coder_streaming_malformed_recovery() {
+ let mut parser = QwenCoderParser::new();
+ let tools = create_test_tools();
+
+ // First: malformed tool call (invalid function name)
+ // Second: valid tool call
+ let chunks = vec![
+ r#"1"#,
+ r#"Tokyo"#,
+ ];
+
+ let mut valid_tool_found = false;
+
+ for chunk in chunks {
+ let result = parser.parse_incremental(chunk, &tools).await.unwrap();
+ for call in result.calls {
+ if let Some(name) = call.name {
+ if name == "get_weather" {
+ valid_tool_found = true;
+ }
+ }
+ }
+ }
+
+ assert!(
+ valid_tool_found,
+ "Should recover and parse valid tool after invalid one"
+ );
+}
+
+#[tokio::test]
+async fn test_qwen_coder_parameter_with_xml_like_content() {
+ let parser = QwenCoderParser::new();
+
+ // Parameter value contains XML-like content that shouldn't be parsed as tags
+ let input = r#"
+
+Hello
+
+
+"#;
+
+ let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
+ assert_eq!(tools.len(), 1);
+
+ let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
+ assert!(args["html_content"]
+ .as_str()
+ .unwrap()
+ .contains(""));
+ assert!(args["xml_snippet"]
+ .as_str()
+ .unwrap()
+ .contains("
+
+
+
+value
+
+"#;
+
+ let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
+ assert_eq!(tools.len(), 1);
+
+ let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
+ assert_eq!(args["empty"], "");
+ assert_eq!(args["whitespace"], ""); // Trimmed
+ assert_eq!(args["normal"], "value");
+}
+
+// ============================================================================
+// HTML Entity and Python Literal Tests
+// ============================================================================
+
+#[tokio::test]
+async fn test_qwen_coder_html_entity_decoding() {
+ let parser = QwenCoderParser::new();
+
+ // Test HTML entities in parameter values
+ let input = r#"
+
+Tom & Jerry
+5 < 10 && 10 > 5
+"Hello" & 'World'
+
+"#;
+
+ let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
+ assert_eq!(tools.len(), 1);
+
+ let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
+ assert_eq!(args["ampersand"], "Tom & Jerry");
+ assert_eq!(args["comparison"], "5 < 10 && 10 > 5");
+ assert_eq!(args["quotes"], "\"Hello\" & 'World'");
+}
+
+#[tokio::test]
+async fn test_qwen_coder_html_numeric_entities() {
+ let parser = QwenCoderParser::new();
+
+ // Test numeric HTML entities
+ let input = r#"
+
+<tag>
+<tag>
+
+"#;
+
+ let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
+ assert_eq!(tools.len(), 1);
+
+ let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
+ assert_eq!(args["decimal"], "");
+ assert_eq!(args["hex"], "");
+}
+
+#[tokio::test]
+async fn test_qwen_coder_python_literals() {
+ let parser = QwenCoderParser::new();
+
+ // Test Python-style literals (True, False, None)
+ let input = r#"
+
+True
+False
+None
+true
+false
+null
+
+"#;
+
+ let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
+ assert_eq!(tools.len(), 1);
+
+ let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
+ // Python literals should be converted
+ assert_eq!(args["py_true"], true);
+ assert_eq!(args["py_false"], false);
+ assert_eq!(args["py_none"], serde_json::Value::Null);
+ // JSON literals should also work
+ assert_eq!(args["json_true"], true);
+ assert_eq!(args["json_false"], false);
+ assert_eq!(args["json_null"], serde_json::Value::Null);
+}
+
+#[tokio::test]
+async fn test_qwen_coder_mixed_html_and_json() {
+ let parser = QwenCoderParser::new();
+
+ // Test HTML entities within JSON structures
+ let input = r#"
+
+price < 100 && rating > 4
+{"operator": "&&", "escape": true}
+
+"#;
+
+ let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
+ assert_eq!(tools.len(), 1);
+
+ let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
+ assert_eq!(args["query"], "price < 100 && rating > 4");
+ // JSON with HTML entity inside - the entity gets decoded first, then JSON parsed
+ assert!(args["config"].is_object());
+ assert_eq!(args["config"]["operator"], "&&");
+}