[model-gateway] : Rust integration tests for integration_mock replacement (#16441)
This commit is contained in:
Simo Lin
@@ -0,0 +1,17 @@
|
||||
//! Tool parser integration tests
|
||||
|
||||
pub mod tool_parser_deepseek;
|
||||
pub mod tool_parser_edge_cases;
|
||||
pub mod tool_parser_fallback;
|
||||
pub mod tool_parser_glm47_moe;
|
||||
pub mod tool_parser_glm4_moe;
|
||||
pub mod tool_parser_json;
|
||||
pub mod tool_parser_kimik2;
|
||||
pub mod tool_parser_llama;
|
||||
pub mod tool_parser_minimax_m2;
|
||||
pub mod tool_parser_mistral;
|
||||
pub mod tool_parser_mixed_edge_cases;
|
||||
pub mod tool_parser_partial_json;
|
||||
pub mod tool_parser_pythonic;
|
||||
pub mod tool_parser_qwen;
|
||||
pub mod tool_parser_step3;
|
||||
@@ -0,0 +1,160 @@
|
||||
//! DeepSeek V3 Parser Integration Tests
|
||||
|
||||
use smg::tool_parser::{DeepSeekParser, ToolParser};
|
||||
|
||||
use crate::common::create_test_tools;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_deepseek_complete_parsing() {
|
||||
let parser = DeepSeekParser::new();
|
||||
|
||||
let input = r#"Let me help you with that.
|
||||
<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>get_weather
|
||||
```json
|
||||
{"location": "Tokyo", "units": "celsius"}
|
||||
```<｜tool▁call▁end｜><｜tool▁calls▁end｜>
|
||||
The weather in Tokyo is..."#;
|
||||
|
||||
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
assert_eq!(normal_text, "Let me help you with that.\n");
|
||||
assert_eq!(tools[0].function.name, "get_weather");
|
||||
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert_eq!(args["location"], "Tokyo");
|
||||
assert_eq!(args["units"], "celsius");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_deepseek_multiple_tools() {
|
||||
let parser = DeepSeekParser::new();
|
||||
|
||||
let input = r#"<｜tool▁calls▁begin｜>
|
||||
<｜tool▁call▁begin｜>function<｜tool▁sep｜>search
|
||||
```json
|
||||
{"query": "rust programming"}
|
||||
```<｜tool▁call▁end｜>
|
||||
<｜tool▁call▁begin｜>function<｜tool▁sep｜>translate
|
||||
```json
|
||||
{"text": "Hello World", "to": "ja"}
|
||||
```<｜tool▁call▁end｜>
|
||||
<｜tool▁calls▁end｜>"#;
|
||||
|
||||
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 2);
|
||||
assert_eq!(tools[0].function.name, "search");
|
||||
assert_eq!(tools[1].function.name, "translate");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_deepseek_streaming() {
|
||||
let tools = create_test_tools();
|
||||
|
||||
let mut parser = DeepSeekParser::new();
|
||||
|
||||
// Simulate streaming chunks
|
||||
let chunks = vec![
|
||||
"<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>",
|
||||
"function<｜tool▁sep｜>get_weather\n",
|
||||
"```json\n",
|
||||
r#"{"location": "#,
|
||||
r#""Beijing", "#,
|
||||
r#""units": "metric"}"#,
|
||||
"\n```<｜tool▁call▁end｜><｜tool▁calls▁end｜>",
|
||||
];
|
||||
|
||||
let mut found_name = false;
|
||||
|
||||
for chunk in chunks {
|
||||
let result = parser.parse_incremental(chunk, &tools).await.unwrap();
|
||||
|
||||
for call in result.calls {
|
||||
if let Some(name) = call.name {
|
||||
assert_eq!(name, "get_weather");
|
||||
found_name = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
assert!(found_name, "Should have found tool name during streaming");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_deepseek_nested_json() {
|
||||
let parser = DeepSeekParser::new();
|
||||
|
||||
let input = r#"<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>process
|
||||
```json
|
||||
{
|
||||
"data": {
|
||||
"nested": {
|
||||
"deep": [1, 2, 3]
|
||||
}
|
||||
}
|
||||
}
|
||||
```<｜tool▁call▁end｜><｜tool▁calls▁end｜>"#;
|
||||
|
||||
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
assert_eq!(tools[0].function.name, "process");
|
||||
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert!(args["data"]["nested"]["deep"].is_array());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_deepseek_format_detection() {
|
||||
let parser = DeepSeekParser::new();
|
||||
|
||||
// Should detect DeepSeek format
|
||||
assert!(parser.has_tool_markers("<｜tool▁calls▁begin｜>"));
|
||||
assert!(parser.has_tool_markers("text with <｜tool▁calls▁begin｜> marker"));
|
||||
|
||||
// Should not detect other formats
|
||||
assert!(!parser.has_tool_markers("[TOOL_CALLS]"));
|
||||
assert!(!parser.has_tool_markers("<tool_call>"));
|
||||
assert!(!parser.has_tool_markers("plain text"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_deepseek_malformed_json_handling() {
|
||||
let parser = DeepSeekParser::new();
|
||||
|
||||
// Malformed JSON should be skipped
|
||||
let input = r#"<｜tool▁calls▁begin｜>
|
||||
<｜tool▁call▁begin｜>function<｜tool▁sep｜>broken
|
||||
```json
|
||||
{invalid json}
|
||||
```<｜tool▁call▁end｜>
|
||||
<｜tool▁call▁begin｜>function<｜tool▁sep｜>valid
|
||||
```json
|
||||
{"key": "value"}
|
||||
```<｜tool▁call▁end｜>
|
||||
<｜tool▁calls▁end｜>"#;
|
||||
|
||||
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
// Only the valid tool call should be parsed
|
||||
assert_eq!(tools.len(), 1);
|
||||
assert_eq!(tools[0].function.name, "valid");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_multiple_tool_calls() {
|
||||
let parser = DeepSeekParser::new();
|
||||
|
||||
let input = r#"<｜tool▁calls▁begin｜>
|
||||
<｜tool▁call▁begin｜>function<｜tool▁sep｜>get_weather
|
||||
```json
|
||||
{"location": "Tokyo"}
|
||||
```<｜tool▁call▁end｜>
|
||||
<｜tool▁call▁begin｜>function<｜tool▁sep｜>get_weather
|
||||
```json
|
||||
{"location": "Paris"}
|
||||
```<｜tool▁call▁end｜>
|
||||
<｜tool▁calls▁end｜><｜end▁of▁sentence｜>"#;
|
||||
|
||||
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 2);
|
||||
assert_eq!(tools[0].function.name, "get_weather");
|
||||
assert_eq!(tools[1].function.name, "get_weather");
|
||||
}
|
||||
@@ -0,0 +1,348 @@
|
||||
//! Edge Cases and Error Handling Tests
|
||||
//!
|
||||
//! Tests for malformed input, edge cases, and error recovery
|
||||
|
||||
use smg::tool_parser::{JsonParser, MistralParser, PythonicParser, QwenParser, ToolParser};
|
||||
|
||||
use crate::common::create_test_tools;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_empty_input() {
|
||||
// Test that all parsers handle empty input correctly
|
||||
let json_parser = JsonParser::new();
|
||||
let (_normal_text, tools) = json_parser.parse_complete("").await.unwrap();
|
||||
assert_eq!(
|
||||
tools.len(),
|
||||
0,
|
||||
"JSON parser should return empty for empty input"
|
||||
);
|
||||
|
||||
let mistral_parser = MistralParser::new();
|
||||
let (_normal_text, tools) = mistral_parser.parse_complete("").await.unwrap();
|
||||
assert_eq!(
|
||||
tools.len(),
|
||||
0,
|
||||
"Mistral parser should return empty for empty input"
|
||||
);
|
||||
|
||||
let qwen_parser = QwenParser::new();
|
||||
let (_normal_text, tools) = qwen_parser.parse_complete("").await.unwrap();
|
||||
assert_eq!(
|
||||
tools.len(),
|
||||
0,
|
||||
"Qwen parser should return empty for empty input"
|
||||
);
|
||||
|
||||
let pythonic_parser = PythonicParser::new();
|
||||
let (_normal_text, tools) = pythonic_parser.parse_complete("").await.unwrap();
|
||||
assert_eq!(
|
||||
tools.len(),
|
||||
0,
|
||||
"Pythonic parser should return empty for empty input"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_plain_text_no_tools() {
|
||||
let plain_text = "This is just a regular response with no tool calls whatsoever.";
|
||||
|
||||
let json_parser = JsonParser::new();
|
||||
assert_eq!(
|
||||
json_parser
|
||||
.parse_complete(plain_text)
|
||||
.await
|
||||
.unwrap()
|
||||
.1
|
||||
.len(),
|
||||
0
|
||||
);
|
||||
|
||||
let mistral_parser = MistralParser::new();
|
||||
assert_eq!(
|
||||
mistral_parser
|
||||
.parse_complete(plain_text)
|
||||
.await
|
||||
.unwrap()
|
||||
.1
|
||||
.len(),
|
||||
0
|
||||
);
|
||||
|
||||
let qwen_parser = QwenParser::new();
|
||||
assert_eq!(
|
||||
qwen_parser
|
||||
.parse_complete(plain_text)
|
||||
.await
|
||||
.unwrap()
|
||||
.1
|
||||
.len(),
|
||||
0
|
||||
);
|
||||
|
||||
let pythonic_parser = PythonicParser::new();
|
||||
assert_eq!(
|
||||
pythonic_parser
|
||||
.parse_complete(plain_text)
|
||||
.await
|
||||
.unwrap()
|
||||
.1
|
||||
.len(),
|
||||
0
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_incomplete_json() {
|
||||
let json_parser = JsonParser::new();
|
||||
|
||||
let incomplete_cases = vec![
|
||||
r#"{"name": "test""#, // Missing closing brace
|
||||
r#"{"name": "test", "arguments":"#, // Incomplete arguments
|
||||
r#"{"name": "test", "arguments": {"#, // Incomplete nested object
|
||||
];
|
||||
|
||||
for input in incomplete_cases {
|
||||
let (_normal_text, tools) = json_parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(
|
||||
tools.len(),
|
||||
0,
|
||||
"Should not parse incomplete JSON: {}",
|
||||
input
|
||||
);
|
||||
}
|
||||
|
||||
// This case might actually parse because [{"name": "test"}] is complete
|
||||
// The trailing comma suggests more items but the first item is valid
|
||||
let _result = json_parser
|
||||
.parse_complete(r#"[{"name": "test"},"#)
|
||||
.await
|
||||
.unwrap();
|
||||
// This could parse the first element or return empty - implementation dependent
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_malformed_mistral() {
|
||||
let parser = MistralParser::new();
|
||||
|
||||
let malformed_cases = vec![
|
||||
"[TOOL_CALLS]", // Missing array
|
||||
"[TOOL_CALLS] {", // Not an array
|
||||
"[TOOL_CALLS] [", // Incomplete array
|
||||
"[TOOL_CALLS] [{]", // Invalid JSON in array
|
||||
"[TOOL_CALLS] [{\"name\": }]", // Invalid value
|
||||
];
|
||||
|
||||
for input in malformed_cases {
|
||||
// Parser might return error or empty vec for malformed input
|
||||
if let Ok((_normal_text, tools)) = parser.parse_complete(input).await {
|
||||
assert_eq!(
|
||||
tools.len(),
|
||||
0,
|
||||
"Should not parse malformed Mistral: {}",
|
||||
input
|
||||
);
|
||||
}
|
||||
// Error is also acceptable for malformed input
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_missing_required_fields() {
|
||||
let json_parser = JsonParser::new();
|
||||
|
||||
// Missing name field
|
||||
let input = r#"{"arguments": {"x": 1}}"#;
|
||||
let (_normal_text, tools) = json_parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 0, "Should not parse without name field");
|
||||
|
||||
// Name is not a string
|
||||
let input = r#"{"name": 123, "arguments": {}}"#;
|
||||
let (_normal_text, tools) = json_parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 0, "Should not parse with non-string name");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_very_long_strings() {
|
||||
let json_parser = JsonParser::new();
|
||||
|
||||
let long_string = "x".repeat(10000);
|
||||
let input = format!(
|
||||
r#"{{"name": "test", "arguments": {{"data": "{}"}}}}"#,
|
||||
long_string
|
||||
);
|
||||
|
||||
let (_normal_text, tools) = json_parser.parse_complete(&input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
assert_eq!(tools[0].function.name, "test");
|
||||
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert_eq!(args["data"].as_str().unwrap().len(), 10000);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_unicode_edge_cases() {
|
||||
let json_parser = JsonParser::new();
|
||||
|
||||
// Various Unicode characters including emojis, CJK, RTL text
|
||||
let input = r#"{"name": "translate", "arguments": {"text": "Hello 世界 🌍 مرحبا עולם"}}"#;
|
||||
|
||||
let (_normal_text, tools) = json_parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert_eq!(args["text"], "Hello 世界 🌍 مرحبا עולם");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_nested_brackets_in_strings() {
|
||||
let mistral_parser = MistralParser::new();
|
||||
let input = r#"[TOOL_CALLS] [{"name": "echo", "arguments": {"text": "Array: [1, 2, 3]"}}]"#;
|
||||
let (_normal_text, tools) = mistral_parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert_eq!(args["text"], "Array: [1, 2, 3]");
|
||||
|
||||
let pythonic_parser = PythonicParser::new();
|
||||
let input = r#"[echo(text="List: [a, b, c]")]"#;
|
||||
let (_normal_text, tools) = pythonic_parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert_eq!(args["text"], "List: [a, b, c]");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_multiple_formats_in_text() {
|
||||
let json_parser = JsonParser::new();
|
||||
let input = r#"
|
||||
Here's some text with [TOOL_CALLS] that shouldn't trigger.
|
||||
{"name": "actual_tool", "arguments": {}}
|
||||
And some more text with <tool_call> tags.
|
||||
"#;
|
||||
|
||||
let (_normal_text, tools) = json_parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
assert_eq!(tools[0].function.name, "actual_tool");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_escaped_characters() {
|
||||
let json_parser = JsonParser::new();
|
||||
|
||||
let input = r#"{"name": "write", "arguments": {"content": "Line 1\nLine 2\r\nLine 3\tTabbed\\Backslash\"Quote"}}"#;
|
||||
|
||||
let (_normal_text, tools) = json_parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
let content = args["content"].as_str().unwrap();
|
||||
assert!(content.contains('\n'));
|
||||
assert!(content.contains('\t'));
|
||||
assert!(content.contains('\\'));
|
||||
assert!(content.contains('"'));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_numeric_edge_cases() {
|
||||
let json_parser = JsonParser::new();
|
||||
|
||||
let input = r#"{
|
||||
"name": "calculate",
|
||||
"arguments": {
|
||||
"int": 42,
|
||||
"float": 123.456,
|
||||
"scientific": 1.23e-4,
|
||||
"negative": -999,
|
||||
"zero": 0,
|
||||
"large": 9007199254740991
|
||||
}
|
||||
}"#;
|
||||
|
||||
let (_normal_text, tools) = json_parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert_eq!(args["int"], 42);
|
||||
assert_eq!(args["float"], 123.456);
|
||||
assert_eq!(args["scientific"], 0.000123);
|
||||
assert_eq!(args["negative"], -999);
|
||||
assert_eq!(args["zero"], 0);
|
||||
assert_eq!(args["large"], 9007199254740991i64);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_null_and_boolean_values() {
|
||||
let json_parser = JsonParser::new();
|
||||
|
||||
let input = r#"{
|
||||
"name": "configure",
|
||||
"arguments": {
|
||||
"enabled": true,
|
||||
"disabled": false,
|
||||
"optional": null
|
||||
}
|
||||
}"#;
|
||||
|
||||
let (_normal_text, tools) = json_parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert_eq!(args["enabled"], true);
|
||||
assert_eq!(args["disabled"], false);
|
||||
assert_eq!(args["optional"], serde_json::Value::Null);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_partial_token_at_buffer_boundary() {
|
||||
let mut parser = QwenParser::new();
|
||||
|
||||
let tools = create_test_tools();
|
||||
|
||||
// Send exactly "<tool" which is a 5-character prefix of "<tool_call>\n"
|
||||
let result = parser.parse_incremental("<tool", &tools).await.unwrap();
|
||||
assert!(
|
||||
result.calls.is_empty(),
|
||||
"Should be incomplete for partial tag"
|
||||
);
|
||||
|
||||
// Complete the token
|
||||
let result = parser
|
||||
.parse_incremental(
|
||||
"_call>\n{\"name\": \"test\", \"arguments\": {}}\n</tool_call>",
|
||||
&tools,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Should successfully parse after completing
|
||||
if !result.calls.is_empty() {
|
||||
if let Some(name) = &result.calls[0].name {
|
||||
assert_eq!(name, "test");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_exact_prefix_lengths() {
|
||||
let mut parser = QwenParser::new();
|
||||
|
||||
let tools = create_test_tools();
|
||||
|
||||
let test_cases = vec![
|
||||
("<", 1), // 1-char prefix
|
||||
("<t", 2), // 2-char prefix
|
||||
("<tool", 5), // 5-char prefix (the main bug case)
|
||||
("<tool_call", 10), // 10-char prefix
|
||||
("<tool_call>", 11), // 11-char prefix (full start without \n)
|
||||
];
|
||||
|
||||
for (prefix, expected_len) in test_cases {
|
||||
let result = parser.parse_incremental(prefix, &tools).await.unwrap();
|
||||
assert!(
|
||||
result.calls.is_empty(),
|
||||
"Prefix '{}' (len {}) should be incomplete",
|
||||
prefix,
|
||||
expected_len
|
||||
);
|
||||
// Buffer is now internal to parser - can't assert on it
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,272 @@
|
||||
//! Tests for tool parser fallback behavior
|
||||
//!
|
||||
//! When tool call parsing fails, the original text should be preserved as normal text
|
||||
//! rather than being lost. This ensures graceful degradation.
|
||||
|
||||
use smg::tool_parser::{
|
||||
DeepSeekParser, JsonParser, LlamaParser, MistralParser, QwenParser, ToolParser,
|
||||
};
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_json_parser_invalid_json_returns_as_normal_text() {
|
||||
let parser = JsonParser::new();
|
||||
|
||||
// Malformed JSON should be returned as normal text (note: commas may be processed)
|
||||
let input = r#"{"name": "test", "arguments": invalid json here}"#;
|
||||
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 0);
|
||||
assert_eq!(
|
||||
normal_text,
|
||||
r#"{"name": "test", "arguments": invalid json here}"#
|
||||
);
|
||||
|
||||
// Plain text with no JSON structure should be returned as normal text
|
||||
let input = "This is just plain text that should not be parsed as a tool call";
|
||||
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 0);
|
||||
assert_eq!(normal_text, input);
|
||||
|
||||
// Text that looks like it might have JSON but doesn't should be returned as normal text
|
||||
let input = "The user said: {something} but it's not valid JSON";
|
||||
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 0);
|
||||
assert_eq!(normal_text, input);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_qwen_parser_invalid_format_returns_as_normal_text() {
|
||||
let parser = QwenParser::new();
|
||||
|
||||
// Missing closing tag
|
||||
let input = r#"<tool_call>
|
||||
{"name": "test", "arguments": {}}
|
||||
This text is missing the closing tag"#;
|
||||
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 0);
|
||||
assert_eq!(normal_text, input); // Should preserve original text when no valid tools found
|
||||
|
||||
// Malformed JSON inside valid tags
|
||||
let input = r#"<tool_call>
|
||||
{"name": "test", "arguments": invalid}
|
||||
</tool_call>"#;
|
||||
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 0);
|
||||
// When JSON parsing fails but tags are present, it should preserve the original text
|
||||
assert_eq!(normal_text, input);
|
||||
|
||||
// Plain text without any tool markers
|
||||
let input = "This is a regular response without any tool calls.";
|
||||
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 0);
|
||||
assert_eq!(normal_text, input); // Should return original text when no markers found
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_llama_parser_invalid_format_returns_as_normal_text() {
|
||||
let parser = LlamaParser::new();
|
||||
|
||||
// Invalid JSON after python_tag
|
||||
let input = r#"<|python_tag|>{"name": "test", "arguments": invalid}"#;
|
||||
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 0);
|
||||
assert_eq!(normal_text, input); // Should preserve original text when parsing fails
|
||||
|
||||
// Plain text without markers or JSON
|
||||
let input = "Just explaining something without any function calls.";
|
||||
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 0);
|
||||
assert_eq!(normal_text, input); // Should return original text
|
||||
|
||||
// Text with python_tag but completely invalid content
|
||||
let input = r#"Here's my response <|python_tag|>not even close to JSON"#;
|
||||
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 0);
|
||||
assert_eq!(normal_text, input); // Should preserve everything when parsing fails
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_mistral_parser_invalid_format_returns_as_normal_text() {
|
||||
let parser = MistralParser::new();
|
||||
|
||||
// Missing closing bracket
|
||||
let input = r#"[TOOL_CALLS] [{"name": "test", "arguments": {}"#;
|
||||
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 0);
|
||||
assert_eq!(normal_text, input); // Should preserve original text when parsing fails
|
||||
|
||||
// Invalid JSON in tool calls section
|
||||
let input = r#"[TOOL_CALLS] [{"name": invalid json}]"#;
|
||||
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 0);
|
||||
assert_eq!(normal_text, input); // Should preserve original text when parsing fails
|
||||
|
||||
// Plain text
|
||||
let input = "No tool calls here, just regular text.";
|
||||
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 0);
|
||||
assert_eq!(normal_text, input); // Should return original text
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_deepseek_parser_invalid_format_returns_as_normal_text() {
|
||||
let parser = DeepSeekParser::new();
|
||||
|
||||
// Invalid JSON in tool call
|
||||
let input = r#"Some text<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>test
|
||||
```json
|
||||
{"name": "test", "arguments": malformed}
|
||||
```<｜tool▁call▁end｜><｜tool▁calls▁end｜>"#;
|
||||
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 0);
|
||||
assert_eq!(normal_text, input); // Should preserve original text when parsing fails
|
||||
|
||||
// Missing function marker
|
||||
let input = r#"<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>notfunction<｜tool▁sep｜>test
|
||||
```json
|
||||
{"x": 1}
|
||||
```<｜tool▁call▁end｜><｜tool▁calls▁end｜>"#;
|
||||
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 0);
|
||||
assert_eq!(normal_text, input); // Should return original text when parsing fails
|
||||
|
||||
// No tool markers at all
|
||||
let input = "Regular response without any special markers.";
|
||||
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 0);
|
||||
assert_eq!(normal_text, input); // Should return original text
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_mixed_valid_and_invalid_content() {
|
||||
let parser = QwenParser::new();
|
||||
|
||||
// Text with one valid tool call and one invalid
|
||||
let input = r#"Let me help you with that.
|
||||
<tool_call>
|
||||
{"name": "valid_tool", "arguments": {"x": 1}}
|
||||
</tool_call>
|
||||
And here's another one:
|
||||
<tool_call>
|
||||
{"name": "invalid_tool", "arguments": malformed}
|
||||
</tool_call>
|
||||
That's all!"#;
|
||||
|
||||
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1); // Should extract the valid tool
|
||||
assert_eq!(tools[0].function.name, "valid_tool");
|
||||
// Normal text should contain text before the first tool call
|
||||
assert_eq!(normal_text, "Let me help you with that.\n");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_partial_tool_markers() {
|
||||
// Test cases where tool markers are incomplete or cut off
|
||||
|
||||
let parser = QwenParser::new();
|
||||
let input = "<tool_call>\nThis looks like it might be a tool call but it's not";
|
||||
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 0);
|
||||
assert_eq!(normal_text, input);
|
||||
|
||||
let parser = MistralParser::new();
|
||||
let input = "[TOOL_CALLS] But then nothing follows...";
|
||||
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 0);
|
||||
assert_eq!(normal_text, input);
|
||||
|
||||
let parser = LlamaParser::new();
|
||||
let input = "Starting a response <|python_tag|> but no JSON";
|
||||
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 0);
|
||||
assert_eq!(normal_text, input);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_escaped_json_like_content() {
|
||||
// Test that JSON-like content in regular text doesn't get parsed as tools
|
||||
|
||||
let parser = JsonParser::new();
|
||||
let input = r#"The user typed: {"name": "example"} but this is just quoted text"#;
|
||||
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
// JsonParser should extract the valid JSON and return normal text
|
||||
assert_eq!(tools.len(), 1);
|
||||
assert_eq!(tools[0].function.name, "example");
|
||||
assert_eq!(normal_text, "The user typed: but this is just quoted text");
|
||||
|
||||
let parser = QwenParser::new();
|
||||
let input = r#"The syntax is: <tool_call>
|
||||
{"name": "example"}
|
||||
</tool_call> - that's how you format it"#;
|
||||
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
// This actually contains valid tool call syntax, so it should parse
|
||||
assert_eq!(tools.len(), 1);
|
||||
assert_eq!(tools[0].function.name, "example");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_unicode_and_special_chars_in_failed_parsing() {
|
||||
let parser = QwenParser::new();
|
||||
|
||||
// Unicode in malformed tool calls
|
||||
let input = r#"<tool_call>
|
||||
{"name": "测试", "arguments": 🚀 invalid}
|
||||
</tool_call>"#;
|
||||
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 0);
|
||||
// Should handle Unicode properly in the fallback text - malformed content should be preserved
|
||||
assert_eq!(normal_text, input);
|
||||
|
||||
// Special characters that might confuse parsers
|
||||
let input = r#"Response: <tool_call>{"name": "test\n\t", "arguments": {"]}"}</tool_call>"#;
|
||||
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
// This might or might not parse depending on JSON handling of escape sequences
|
||||
if tools.is_empty() {
|
||||
assert!(!normal_text.is_empty() || normal_text == input);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_very_long_invalid_input() {
|
||||
let parser = JsonParser::new();
|
||||
|
||||
// Generate a very long string that looks like it might be JSON but isn't
|
||||
let mut input = String::from("{\"name\": \"test\", \"arguments\": {");
|
||||
for i in 0..1000 {
|
||||
input.push_str(&format!("\"field{}\": \"value{}\", ", i, i));
|
||||
}
|
||||
input.push_str("\"final\": incomplete"); // Don't close the JSON properly
|
||||
|
||||
let (normal_text, tools) = parser.parse_complete(&input).await.unwrap();
|
||||
assert_eq!(tools.len(), 0);
|
||||
assert_eq!(normal_text, input); // Invalid JSON should be returned as normal text
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_almost_valid_tool_calls() {
|
||||
// Test tool calls that are almost valid but have small issues
|
||||
|
||||
let parser = JsonParser::new();
|
||||
|
||||
// Missing closing quote should be returned as normal text
|
||||
let input = r#"{"name": "test", "arguments": {"key": "value}}"#;
|
||||
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 0);
|
||||
assert_eq!(
|
||||
normal_text,
|
||||
r#"{"name": "test", "arguments": {"key": "value}}"#
|
||||
);
|
||||
|
||||
// Extra comma
|
||||
let input = r#"{"name": "test", "arguments": {},}"#;
|
||||
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
// Some JSON parsers might accept trailing commas
|
||||
if tools.is_empty() {
|
||||
assert_eq!(normal_text, r#"{"name": "test", "arguments": {},}"#);
|
||||
}
|
||||
|
||||
// Wrong quote types
|
||||
let input = r#"{'name': 'test', 'arguments': {}}"#;
|
||||
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 0); // Standard JSON requires double quotes
|
||||
assert_eq!(normal_text, r#"{'name': 'test', 'arguments': {}}"#);
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
//! GLM-4.7 MoE Parser Integration Tests
|
||||
|
||||
use smg::tool_parser::{Glm4MoeParser, ToolParser};
|
||||
|
||||
use crate::common::create_test_tools;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_glm47_complete_parsing() {
|
||||
let parser = Glm4MoeParser::glm47();
|
||||
|
||||
let input = r#"Let me search for that.
|
||||
<tool_call>get_weather<arg_key>city</arg_key><arg_value>Beijing</arg_value><arg_key>date</arg_key><arg_value>2024-12-25</arg_value></tool_call>
|
||||
The weather will be..."#;
|
||||
|
||||
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
assert_eq!(normal_text, "Let me search for that.\n");
|
||||
assert_eq!(tools[0].function.name, "get_weather");
|
||||
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert_eq!(args["city"], "Beijing");
|
||||
assert_eq!(args["date"], "2024-12-25");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_glm47_multiple_tools() {
|
||||
let parser = Glm4MoeParser::glm47();
|
||||
|
||||
let input = r#"<tool_call>search<arg_key>query</arg_key><arg_value>rust tutorials</arg_value></tool_call><tool_call>translate<arg_key>text</arg_key><arg_value>Hello World</arg_value><arg_key>target_lang</arg_key><arg_value>zh</arg_value></tool_call>"#;
|
||||
|
||||
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 2);
|
||||
assert_eq!(normal_text, "");
|
||||
assert_eq!(tools[0].function.name, "search");
|
||||
assert_eq!(tools[1].function.name, "translate");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_glm47_type_conversion() {
|
||||
let parser = Glm4MoeParser::glm47();
|
||||
|
||||
let input = r#"<tool_call>process<arg_key>count</arg_key><arg_value>42</arg_value><arg_key>rate</arg_key><arg_value>1.5</arg_value><arg_key>enabled</arg_key><arg_value>true</arg_value><arg_key>data</arg_key><arg_value>null</arg_value><arg_key>text</arg_key><arg_value>string value</arg_value></tool_call>"#;
|
||||
|
||||
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
assert_eq!(normal_text, "");
|
||||
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert_eq!(args["count"], 42);
|
||||
assert_eq!(args["rate"], 1.5);
|
||||
assert_eq!(args["enabled"], true);
|
||||
assert_eq!(args["data"], serde_json::Value::Null);
|
||||
assert_eq!(args["text"], "string value");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_glm47_streaming() {
|
||||
let mut parser = Glm4MoeParser::glm47();
|
||||
|
||||
let tools = create_test_tools();
|
||||
|
||||
// Simulate streaming chunks
|
||||
let chunks = vec![
|
||||
"<tool_call>",
|
||||
"get_weather",
|
||||
"<arg_key>city</arg_key>",
|
||||
"<arg_value>Shanghai</arg_value>",
|
||||
"<arg_key>units</arg_key>",
|
||||
"<arg_value>celsius</arg_value>",
|
||||
"</tool_call>",
|
||||
];
|
||||
|
||||
let mut found_name = false;
|
||||
|
||||
for chunk in chunks {
|
||||
let result = parser.parse_incremental(chunk, &tools).await.unwrap();
|
||||
|
||||
for call in result.calls {
|
||||
if let Some(name) = call.name {
|
||||
assert_eq!(name, "get_weather");
|
||||
found_name = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
assert!(found_name, "Should have found tool name during streaming");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_glm47_format_detection() {
|
||||
let parser = Glm4MoeParser::glm47();
|
||||
|
||||
// Should detect GLM-4 format
|
||||
assert!(parser.has_tool_markers("<tool_call>"));
|
||||
assert!(parser.has_tool_markers("text with <tool_call> marker"));
|
||||
|
||||
// Should not detect other formats
|
||||
assert!(!parser.has_tool_markers("[TOOL_CALLS]"));
|
||||
assert!(!parser.has_tool_markers("<｜tool▁calls▁begin｜>"));
|
||||
assert!(!parser.has_tool_markers("plain text"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_python_literals() {
|
||||
let parser = Glm4MoeParser::glm47();
|
||||
|
||||
let input = r#"<tool_call>test_func<arg_key>bool_true</arg_key><arg_value>True</arg_value><arg_key>bool_false</arg_key><arg_value>False</arg_value><arg_key>none_val</arg_key><arg_value>None</arg_value></tool_call>"#;
|
||||
|
||||
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
assert_eq!(tools[0].function.name, "test_func");
|
||||
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert_eq!(args["bool_true"], true);
|
||||
assert_eq!(args["bool_false"], false);
|
||||
assert_eq!(args["none_val"], serde_json::Value::Null);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_glm47_nested_json_in_arg_values() {
|
||||
let parser = Glm4MoeParser::glm47();
|
||||
|
||||
let input = r#"<tool_call>process<arg_key>data</arg_key><arg_value>{"nested": {"key": "value"}}</arg_value><arg_key>list</arg_key><arg_value>[1, 2, 3]</arg_value></tool_call>"#;
|
||||
|
||||
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert!(args["data"].is_object());
|
||||
assert!(args["list"].is_array());
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
//! GLM-4 MoE Parser Integration Tests
|
||||
|
||||
use smg::tool_parser::{Glm4MoeParser, ToolParser};
|
||||
|
||||
use crate::common::create_test_tools;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_glm4_complete_parsing() {
|
||||
let parser = Glm4MoeParser::glm45();
|
||||
|
||||
let input = r#"Let me search for that.
|
||||
<tool_call>get_weather
|
||||
<arg_key>city</arg_key>
|
||||
<arg_value>Beijing</arg_value>
|
||||
<arg_key>date</arg_key>
|
||||
<arg_value>2024-12-25</arg_value>
|
||||
</tool_call>
|
||||
The weather will be..."#;
|
||||
|
||||
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
assert_eq!(normal_text, "Let me search for that.\n");
|
||||
assert_eq!(tools[0].function.name, "get_weather");
|
||||
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert_eq!(args["city"], "Beijing");
|
||||
assert_eq!(args["date"], "2024-12-25");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_glm4_multiple_tools() {
|
||||
let parser = Glm4MoeParser::glm45();
|
||||
|
||||
let input = r#"<tool_call>search
|
||||
<arg_key>query</arg_key>
|
||||
<arg_value>rust tutorials</arg_value>
|
||||
</tool_call>
|
||||
<tool_call>translate
|
||||
<arg_key>text</arg_key>
|
||||
<arg_value>Hello World</arg_value>
|
||||
<arg_key>target_lang</arg_key>
|
||||
<arg_value>zh</arg_value>
|
||||
</tool_call>"#;
|
||||
|
||||
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 2);
|
||||
assert_eq!(normal_text, "");
|
||||
assert_eq!(tools[0].function.name, "search");
|
||||
assert_eq!(tools[1].function.name, "translate");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_glm4_type_conversion() {
|
||||
let parser = Glm4MoeParser::glm45();
|
||||
|
||||
let input = r#"<tool_call>process
|
||||
<arg_key>count</arg_key>
|
||||
<arg_value>42</arg_value>
|
||||
<arg_key>rate</arg_key>
|
||||
<arg_value>1.5</arg_value>
|
||||
<arg_key>enabled</arg_key>
|
||||
<arg_value>true</arg_value>
|
||||
<arg_key>data</arg_key>
|
||||
<arg_value>null</arg_value>
|
||||
<arg_key>text</arg_key>
|
||||
<arg_value>string value</arg_value>
|
||||
</tool_call>"#;
|
||||
|
||||
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
assert_eq!(normal_text, "");
|
||||
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert_eq!(args["count"], 42);
|
||||
assert_eq!(args["rate"], 1.5);
|
||||
assert_eq!(args["enabled"], true);
|
||||
assert_eq!(args["data"], serde_json::Value::Null);
|
||||
assert_eq!(args["text"], "string value");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_glm4_streaming() {
|
||||
let mut parser = Glm4MoeParser::glm45();
|
||||
|
||||
let tools = create_test_tools();
|
||||
|
||||
// Simulate streaming chunks
|
||||
let chunks = vec![
|
||||
"<tool_call>",
|
||||
"get_weather\n",
|
||||
"<arg_key>city</arg_key>\n",
|
||||
"<arg_value>Shanghai</arg_value>\n",
|
||||
"<arg_key>units</arg_key>\n",
|
||||
"<arg_value>celsius</arg_value>\n",
|
||||
"</tool_call>",
|
||||
];
|
||||
|
||||
let mut found_name = false;
|
||||
|
||||
for chunk in chunks {
|
||||
let result = parser.parse_incremental(chunk, &tools).await.unwrap();
|
||||
|
||||
for call in result.calls {
|
||||
if let Some(name) = call.name {
|
||||
assert_eq!(name, "get_weather");
|
||||
found_name = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
assert!(found_name, "Should have found tool name during streaming");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_glm4_format_detection() {
|
||||
let parser = Glm4MoeParser::glm45();
|
||||
|
||||
// Should detect GLM-4 format
|
||||
assert!(parser.has_tool_markers("<tool_call>"));
|
||||
assert!(parser.has_tool_markers("text with <tool_call> marker"));
|
||||
|
||||
// Should not detect other formats
|
||||
assert!(!parser.has_tool_markers("[TOOL_CALLS]"));
|
||||
assert!(!parser.has_tool_markers("<｜tool▁calls▁begin｜>"));
|
||||
assert!(!parser.has_tool_markers("plain text"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_python_literals() {
|
||||
let parser = Glm4MoeParser::glm45();
|
||||
|
||||
let input = r#"<tool_call>test_func
|
||||
<arg_key>bool_true</arg_key>
|
||||
<arg_value>True</arg_value>
|
||||
<arg_key>bool_false</arg_key>
|
||||
<arg_value>False</arg_value>
|
||||
<arg_key>none_val</arg_key>
|
||||
<arg_value>None</arg_value>
|
||||
</tool_call>"#;
|
||||
|
||||
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
assert_eq!(tools[0].function.name, "test_func");
|
||||
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert_eq!(args["bool_true"], true);
|
||||
assert_eq!(args["bool_false"], false);
|
||||
assert_eq!(args["none_val"], serde_json::Value::Null);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_glm4_nested_json_in_arg_values() {
|
||||
let parser = Glm4MoeParser::glm45();
|
||||
|
||||
let input = r#"<tool_call>process
|
||||
<arg_key>data</arg_key>
|
||||
<arg_value>{"nested": {"key": "value"}}</arg_value>
|
||||
<arg_key>list</arg_key>
|
||||
<arg_value>[1, 2, 3]</arg_value>
|
||||
</tool_call>"#;
|
||||
|
||||
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert!(args["data"].is_object());
|
||||
assert!(args["list"].is_array());
|
||||
}
|
||||
@@ -0,0 +1,716 @@
|
||||
//! JSON Parser Integration Tests
|
||||
//!
|
||||
//! Tests for the JSON parser which handles OpenAI, Claude, and generic JSON formats
|
||||
|
||||
use serde_json::json;
|
||||
use smg::tool_parser::{JsonParser, ToolParser};
|
||||
|
||||
use crate::common::{create_test_tools, streaming_helpers::*};
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_simple_json_tool_call() {
|
||||
let parser = JsonParser::new();
|
||||
let input = r#"{"name": "get_weather", "arguments": {"location": "San Francisco"}}"#;
|
||||
|
||||
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
assert_eq!(normal_text, "");
|
||||
assert_eq!(tools[0].function.name, "get_weather");
|
||||
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert_eq!(args["location"], "San Francisco");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_json_array_of_tools() {
|
||||
let parser = JsonParser::new();
|
||||
let input = r#"Hello, here are the results: [
|
||||
{"name": "get_weather", "arguments": {"location": "SF"}},
|
||||
{"name": "search", "arguments": {"query": "news"}}
|
||||
]"#;
|
||||
|
||||
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 2);
|
||||
assert_eq!(normal_text, "Hello, here are the results: ");
|
||||
assert_eq!(tools[0].function.name, "get_weather");
|
||||
assert_eq!(tools[1].function.name, "search");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_json_with_parameters_key() {
|
||||
let parser = JsonParser::new();
|
||||
let input = r#"{"name": "calculate", "parameters": {"x": 10, "y": 20}}"#;
|
||||
|
||||
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
assert_eq!(normal_text, "");
|
||||
assert_eq!(tools[0].function.name, "calculate");
|
||||
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert_eq!(args["x"], 10);
|
||||
assert_eq!(args["y"], 20);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_json_extraction_from_text() {
|
||||
let parser = JsonParser::new();
|
||||
let input = r#"I'll help you with that. {"name": "search", "arguments": {"query": "rust"}} Let me search for that."#;
|
||||
|
||||
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
assert_eq!(
|
||||
normal_text,
|
||||
"I'll help you with that. Let me search for that."
|
||||
);
|
||||
assert_eq!(tools[0].function.name, "search");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_json_with_nested_objects() {
|
||||
let parser = JsonParser::new();
|
||||
let input = r#"{
|
||||
"name": "update_config",
|
||||
"arguments": {
|
||||
"settings": {
|
||||
"theme": "dark",
|
||||
"language": "en",
|
||||
"notifications": {
|
||||
"email": true,
|
||||
"push": false
|
||||
}
|
||||
}
|
||||
}
|
||||
}"#;
|
||||
|
||||
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
assert_eq!(normal_text, "");
|
||||
assert_eq!(tools[0].function.name, "update_config");
|
||||
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert_eq!(args["settings"]["theme"], "dark");
|
||||
assert_eq!(args["settings"]["notifications"]["email"], true);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_json_with_special_characters() {
|
||||
let parser = JsonParser::new();
|
||||
let input = r#"{"name": "echo", "arguments": {"text": "Line 1\nLine 2\tTabbed", "path": "C:\\Users\\test"}}"#;
|
||||
|
||||
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
assert_eq!(normal_text, "");
|
||||
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert_eq!(args["text"], "Line 1\nLine 2\tTabbed");
|
||||
assert_eq!(args["path"], "C:\\Users\\test");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_json_with_unicode() {
|
||||
let parser = JsonParser::new();
|
||||
let input = r#"{"name": "translate", "arguments": {"text": "Hello 世界 🌍", "emoji": "😊"}}"#;
|
||||
|
||||
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
assert_eq!(normal_text, "");
|
||||
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert_eq!(args["text"], "Hello 世界 🌍");
|
||||
assert_eq!(args["emoji"], "😊");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_json_empty_arguments() {
|
||||
let parser = JsonParser::new();
|
||||
let input = r#"{"name": "ping", "arguments": {}}"#;
|
||||
|
||||
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
assert_eq!(normal_text, "");
|
||||
assert_eq!(tools[0].function.name, "ping");
|
||||
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert_eq!(args, json!({}));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_json_invalid_format() {
|
||||
let parser = JsonParser::new();
|
||||
|
||||
// Missing closing brace
|
||||
let input = r#"{"name": "test", "arguments": {"key": "value""#;
|
||||
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 0);
|
||||
assert_eq!(
|
||||
normal_text,
|
||||
"{\"name\": \"test\", \"arguments\": {\"key\": \"value\""
|
||||
);
|
||||
|
||||
// Not JSON at all
|
||||
let input = "This is just plain text";
|
||||
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_json_format_detection() {
|
||||
let parser = JsonParser::new();
|
||||
|
||||
assert!(parser.has_tool_markers(r#"{"name": "test", "arguments": {}}"#));
|
||||
assert!(parser.has_tool_markers(r#"[{"name": "test"}]"#));
|
||||
assert!(!parser.has_tool_markers("plain text"));
|
||||
}
|
||||
|
||||
// Streaming tests for JSON array format
|
||||
#[tokio::test]
|
||||
async fn test_json_array_streaming_required_mode() {
|
||||
use smg::protocols::common::Tool;
|
||||
|
||||
// Test that simulates the exact streaming pattern from required mode
|
||||
let mut parser = JsonParser::new();
|
||||
|
||||
// Define test tools
|
||||
let tools = vec![Tool {
|
||||
tool_type: "function".to_string(),
|
||||
function: smg::protocols::common::Function {
|
||||
name: "get_weather".to_string(),
|
||||
description: Some("Get weather".to_string()),
|
||||
parameters: serde_json::json!({}),
|
||||
strict: None,
|
||||
},
|
||||
}];
|
||||
|
||||
// Simulate the EXACT chunks from the debug log
|
||||
let chunks = vec![
|
||||
"[{",
|
||||
" \"",
|
||||
"name",
|
||||
"\":",
|
||||
" \"",
|
||||
"get",
|
||||
"_weather",
|
||||
"\",",
|
||||
" \"",
|
||||
"parameters",
|
||||
"\":",
|
||||
" {",
|
||||
" \"",
|
||||
"city",
|
||||
"\":",
|
||||
" \"",
|
||||
"Paris",
|
||||
"\"",
|
||||
" }",
|
||||
" }]",
|
||||
];
|
||||
|
||||
let mut all_results = Vec::new();
|
||||
let mut all_normal_text = String::new();
|
||||
|
||||
for chunk in chunks {
|
||||
let result = parser.parse_incremental(chunk, &tools).await.unwrap();
|
||||
all_results.extend(result.calls);
|
||||
all_normal_text.push_str(&result.normal_text);
|
||||
}
|
||||
|
||||
// We should have gotten tool call chunks
|
||||
assert!(
|
||||
!all_results.is_empty(),
|
||||
"Should have emitted tool call chunks"
|
||||
);
|
||||
|
||||
// Should not have emitted any normal text (including the closing ])
|
||||
assert_eq!(
|
||||
all_normal_text, "",
|
||||
"Should not emit normal text for JSON array format"
|
||||
);
|
||||
|
||||
// Check that we got the function name
|
||||
let has_name = all_results
|
||||
.iter()
|
||||
.any(|item| item.name.as_ref().is_some_and(|n| n == "get_weather"));
|
||||
assert!(has_name, "Should have emitted function name");
|
||||
|
||||
// Check that we got the parameters
|
||||
let has_params = all_results.iter().any(|item| !item.parameters.is_empty());
|
||||
assert!(has_params, "Should have emitted parameters");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_json_array_multiple_tools_streaming() {
|
||||
use smg::protocols::common::Tool;
|
||||
|
||||
// Test with multiple tools in array
|
||||
let mut parser = JsonParser::new();
|
||||
|
||||
let tools = vec![
|
||||
Tool {
|
||||
tool_type: "function".to_string(),
|
||||
function: smg::protocols::common::Function {
|
||||
name: "get_weather".to_string(),
|
||||
description: Some("Get weather".to_string()),
|
||||
parameters: serde_json::json!({}),
|
||||
strict: None,
|
||||
},
|
||||
},
|
||||
Tool {
|
||||
tool_type: "function".to_string(),
|
||||
function: smg::protocols::common::Function {
|
||||
name: "get_news".to_string(),
|
||||
description: Some("Get news".to_string()),
|
||||
parameters: serde_json::json!({}),
|
||||
strict: None,
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
// Split into smaller, more realistic chunks
|
||||
let chunks = vec![
|
||||
"[{",
|
||||
"\"name\":",
|
||||
"\"get_weather\"",
|
||||
",\"parameters\":",
|
||||
"{\"city\":",
|
||||
"\"SF\"}",
|
||||
"}",
|
||||
",",
|
||||
"{\"name\":",
|
||||
"\"get_news\"",
|
||||
",\"parameters\":",
|
||||
"{\"topic\":",
|
||||
"\"tech\"}",
|
||||
"}]",
|
||||
];
|
||||
|
||||
let mut all_results = Vec::new();
|
||||
|
||||
for chunk in chunks {
|
||||
let result = parser.parse_incremental(chunk, &tools).await.unwrap();
|
||||
all_results.extend(result.calls);
|
||||
}
|
||||
|
||||
// Should have gotten tool calls for both functions
|
||||
let has_weather = all_results
|
||||
.iter()
|
||||
.any(|item| item.name.as_ref().is_some_and(|n| n == "get_weather"));
|
||||
let has_news = all_results
|
||||
.iter()
|
||||
.any(|item| item.name.as_ref().is_some_and(|n| n == "get_news"));
|
||||
|
||||
assert!(has_weather, "Should have get_weather tool call");
|
||||
assert!(has_news, "Should have get_news tool call");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_json_array_closing_bracket_separate_chunk() {
|
||||
use smg::protocols::common::Tool;
|
||||
|
||||
// Test case where the closing ] comes as a separate chunk
|
||||
let mut parser = JsonParser::new();
|
||||
|
||||
let tools = vec![Tool {
|
||||
tool_type: "function".to_string(),
|
||||
function: smg::protocols::common::Function {
|
||||
name: "get_weather".to_string(),
|
||||
description: Some("Get weather".to_string()),
|
||||
parameters: json!({}),
|
||||
strict: None,
|
||||
},
|
||||
}];
|
||||
|
||||
// Closing ] as separate chunk, followed by normal text
|
||||
let chunks = vec![
|
||||
"[{",
|
||||
"\"",
|
||||
"name",
|
||||
"\":",
|
||||
"\"",
|
||||
"get",
|
||||
"_weather",
|
||||
"\",",
|
||||
"\"",
|
||||
"parameters",
|
||||
"\":",
|
||||
"{",
|
||||
"\"",
|
||||
"city",
|
||||
"\":",
|
||||
"\"",
|
||||
"Paris",
|
||||
"\"",
|
||||
"}",
|
||||
"}",
|
||||
"]",
|
||||
" Here's",
|
||||
" the",
|
||||
" weather",
|
||||
" info",
|
||||
];
|
||||
|
||||
let mut all_normal_text = String::new();
|
||||
|
||||
for chunk in chunks {
|
||||
let result = parser.parse_incremental(chunk, &tools).await.unwrap();
|
||||
all_normal_text.push_str(&result.normal_text);
|
||||
}
|
||||
|
||||
// Should emit only the third chunk as normal text, NOT the ]
|
||||
assert_eq!(
|
||||
all_normal_text, " Here's the weather info",
|
||||
"Should emit only normal text without ], got: '{}'",
|
||||
all_normal_text
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_json_single_object_with_trailing_text() {
|
||||
use smg::protocols::common::Tool;
|
||||
|
||||
// Test single object format (no array) with trailing text
|
||||
let mut parser = JsonParser::new();
|
||||
|
||||
let tools = vec![Tool {
|
||||
tool_type: "function".to_string(),
|
||||
function: smg::protocols::common::Function {
|
||||
name: "get_weather".to_string(),
|
||||
description: Some("Get weather".to_string()),
|
||||
parameters: serde_json::json!({}),
|
||||
strict: None,
|
||||
},
|
||||
}];
|
||||
|
||||
let chunks = vec![
|
||||
"{",
|
||||
"\"",
|
||||
"name",
|
||||
"\":",
|
||||
"\"",
|
||||
"get_weather",
|
||||
"\",",
|
||||
"\"",
|
||||
"parameters",
|
||||
"\":",
|
||||
"{",
|
||||
"\"city",
|
||||
"\":",
|
||||
"\"Paris",
|
||||
"\"}",
|
||||
"}",
|
||||
" Here's",
|
||||
" the",
|
||||
" weather",
|
||||
];
|
||||
|
||||
let mut all_normal_text = String::new();
|
||||
|
||||
for chunk in chunks {
|
||||
let result = parser.parse_incremental(chunk, &tools).await.unwrap();
|
||||
all_normal_text.push_str(&result.normal_text);
|
||||
}
|
||||
|
||||
// Should emit the trailing text as normal_text (no ] to strip for single object)
|
||||
assert_eq!(
|
||||
all_normal_text, " Here's the weather",
|
||||
"Should emit normal text for single object format, got: '{}'",
|
||||
all_normal_text
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_json_single_object_with_bracket_in_text() {
|
||||
use smg::protocols::common::Tool;
|
||||
|
||||
// Test that ] in normal text is NOT stripped for single object format
|
||||
let mut parser = JsonParser::new();
|
||||
|
||||
let tools = vec![Tool {
|
||||
tool_type: "function".to_string(),
|
||||
function: smg::protocols::common::Function {
|
||||
name: "get_weather".to_string(),
|
||||
description: Some("Get weather".to_string()),
|
||||
parameters: serde_json::json!({}),
|
||||
strict: None,
|
||||
},
|
||||
}];
|
||||
|
||||
let chunks = vec![
|
||||
"{",
|
||||
"\"name",
|
||||
"\":",
|
||||
"\"get_weather",
|
||||
"\",",
|
||||
"\"parameters",
|
||||
"\":",
|
||||
"{",
|
||||
"\"city",
|
||||
"\":",
|
||||
"\"Paris",
|
||||
"\"}",
|
||||
"}",
|
||||
"]",
|
||||
" Here's",
|
||||
" the",
|
||||
" weather",
|
||||
];
|
||||
|
||||
let mut all_normal_text = String::new();
|
||||
|
||||
for chunk in chunks {
|
||||
let result = parser.parse_incremental(chunk, &tools).await.unwrap();
|
||||
all_normal_text.push_str(&result.normal_text);
|
||||
}
|
||||
|
||||
// For single object format, ] should NOT be stripped (it's part of normal text)
|
||||
assert_eq!(
|
||||
all_normal_text, "] Here's the weather",
|
||||
"Should preserve ] in normal text for single object format, got: '{}'",
|
||||
all_normal_text
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_json_array_bracket_in_text_after_tools() {
|
||||
use smg::protocols::common::Tool;
|
||||
|
||||
// Test that ] in normal text AFTER array tools is preserved
|
||||
let mut parser = JsonParser::new();
|
||||
|
||||
let tools = vec![Tool {
|
||||
tool_type: "function".to_string(),
|
||||
function: smg::protocols::common::Function {
|
||||
name: "get_weather".to_string(),
|
||||
description: Some("Get weather".to_string()),
|
||||
parameters: serde_json::json!({}),
|
||||
strict: None,
|
||||
},
|
||||
}];
|
||||
|
||||
let chunks = vec![
|
||||
"[",
|
||||
"{",
|
||||
"\"name",
|
||||
"\":",
|
||||
"\"get_weather",
|
||||
"\",",
|
||||
"\"parameters",
|
||||
"\":",
|
||||
"{",
|
||||
"\"city",
|
||||
"\":",
|
||||
"\"Paris",
|
||||
"\"}",
|
||||
"}",
|
||||
"]",
|
||||
" Array",
|
||||
" notation:",
|
||||
" arr",
|
||||
"[",
|
||||
"0",
|
||||
"]",
|
||||
];
|
||||
|
||||
let mut all_normal_text = String::new();
|
||||
|
||||
for chunk in chunks {
|
||||
let result = parser.parse_incremental(chunk, &tools).await.unwrap();
|
||||
all_normal_text.push_str(&result.normal_text);
|
||||
}
|
||||
|
||||
// Should preserve ] in normal text after array tools complete
|
||||
assert_eq!(
|
||||
all_normal_text, " Array notation: arr[0]",
|
||||
"Should preserve ] in normal text after array tools, got: '{}'",
|
||||
all_normal_text
|
||||
);
|
||||
}
|
||||
// =============================================================================
|
||||
// REALISTIC STREAMING TESTS
|
||||
// =============================================================================
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_json_bug_incomplete_tool_name_string() {
|
||||
let tools = create_test_tools();
|
||||
let mut parser = JsonParser::new();
|
||||
|
||||
// This exact sequence triggered the bug:
|
||||
// Parser receives {"name": " and must NOT parse it as empty name
|
||||
let chunks = vec![
|
||||
r#"{"#,
|
||||
r#"""#,
|
||||
r#"name"#,
|
||||
r#"""#,
|
||||
r#":"#,
|
||||
r#" "#,
|
||||
r#"""#, // ← Critical moment: parser has {"name": "
|
||||
// At this point, partial_json should NOT allow incomplete strings
|
||||
// when current_tool_name_sent=false
|
||||
r#"search"#, // Use valid tool name from create_test_tools()
|
||||
r#"""#,
|
||||
r#", "#,
|
||||
r#"""#,
|
||||
r#"arguments"#,
|
||||
r#"""#,
|
||||
r#": {"#,
|
||||
r#"""#,
|
||||
r#"query"#,
|
||||
r#"""#,
|
||||
r#": "#,
|
||||
r#"""#,
|
||||
r#"rust programming"#,
|
||||
r#"""#,
|
||||
r#"}}"#,
|
||||
];
|
||||
|
||||
let mut got_tool_name = false;
|
||||
let mut saw_empty_name = false;
|
||||
|
||||
for chunk in chunks.iter() {
|
||||
let result = parser.parse_incremental(chunk, &tools).await.unwrap();
|
||||
|
||||
for call in result.calls {
|
||||
if let Some(name) = &call.name {
|
||||
if name.is_empty() {
|
||||
saw_empty_name = true;
|
||||
}
|
||||
if name == "search" {
|
||||
got_tool_name = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
assert!(
|
||||
!saw_empty_name,
|
||||
"Parser should NEVER return empty tool name"
|
||||
);
|
||||
assert!(got_tool_name, "Should have parsed tool name correctly");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_json_realistic_chunks_simple_tool() {
|
||||
let tools = create_test_tools();
|
||||
let mut parser = JsonParser::new();
|
||||
|
||||
let input = r#"{"name": "get_weather", "arguments": {"city": "Paris"}}"#;
|
||||
let chunks = create_realistic_chunks(input);
|
||||
|
||||
assert!(chunks.len() > 10, "Should have many small chunks");
|
||||
|
||||
let mut got_tool_name = false;
|
||||
|
||||
for chunk in chunks {
|
||||
let result = parser.parse_incremental(&chunk, &tools).await.unwrap();
|
||||
for call in result.calls {
|
||||
if let Some(name) = call.name {
|
||||
assert_eq!(name, "get_weather");
|
||||
got_tool_name = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
assert!(got_tool_name, "Should have parsed tool name");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_json_strategic_chunks_with_quotes() {
|
||||
let tools = create_test_tools();
|
||||
let mut parser = JsonParser::new();
|
||||
|
||||
let input = r#"{"name": "search", "arguments": {"query": "rust programming"}}"#;
|
||||
let chunks = create_strategic_chunks(input);
|
||||
|
||||
// Strategic chunks break after quotes and colons
|
||||
assert!(chunks.iter().any(|c| c.ends_with('"')));
|
||||
|
||||
let mut got_tool_name = false;
|
||||
|
||||
for chunk in chunks {
|
||||
let result = parser.parse_incremental(&chunk, &tools).await.unwrap();
|
||||
for call in result.calls {
|
||||
if call.name.is_some() {
|
||||
got_tool_name = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
assert!(got_tool_name, "Should have parsed tool name");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_json_incremental_arguments_streaming() {
|
||||
let tools = create_test_tools();
|
||||
let mut parser = JsonParser::new();
|
||||
|
||||
let input = r#"{"name": "search", "arguments": {"query": "test", "limit": 10}}"#;
|
||||
let chunks = create_realistic_chunks(input);
|
||||
|
||||
let mut tool_name_sent = false;
|
||||
let mut got_arguments = false;
|
||||
|
||||
for chunk in chunks {
|
||||
let result = parser.parse_incremental(&chunk, &tools).await.unwrap();
|
||||
for call in result.calls {
|
||||
if call.name.is_some() {
|
||||
tool_name_sent = true;
|
||||
}
|
||||
if tool_name_sent && !call.parameters.is_empty() {
|
||||
got_arguments = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
assert!(tool_name_sent, "Should have sent tool name");
|
||||
assert!(got_arguments, "Should have sent arguments");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_json_very_long_url_in_arguments() {
|
||||
let tools = create_test_tools();
|
||||
let mut parser = JsonParser::new();
|
||||
|
||||
// Simulate long URL arriving in many chunks
|
||||
let long_url = "https://example.com/very/long/path/".to_string() + &"segment/".repeat(50);
|
||||
let input = format!(
|
||||
r#"{{"name": "search", "arguments": {{"query": "{}"}}}}"#,
|
||||
long_url
|
||||
);
|
||||
let chunks = create_realistic_chunks(&input);
|
||||
|
||||
assert!(chunks.len() > 100, "Long URL should create many chunks");
|
||||
|
||||
let mut got_tool_name = false;
|
||||
|
||||
for chunk in chunks {
|
||||
let result = parser.parse_incremental(&chunk, &tools).await.unwrap();
|
||||
for call in result.calls {
|
||||
if call.name.is_some() {
|
||||
got_tool_name = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
assert!(got_tool_name, "Should have parsed tool name");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_json_unicode() {
|
||||
let tools = create_test_tools();
|
||||
let mut parser = JsonParser::new();
|
||||
|
||||
let input = r#"{"name": "search", "arguments": {"query": "Hello 世界 🌍"}}"#;
|
||||
let chunks = create_realistic_chunks(input);
|
||||
|
||||
let mut got_tool_name = false;
|
||||
|
||||
for chunk in chunks {
|
||||
let result = parser.parse_incremental(&chunk, &tools).await.unwrap();
|
||||
for call in result.calls {
|
||||
if call.name.is_some() {
|
||||
got_tool_name = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
assert!(got_tool_name, "Should have parsed with unicode");
|
||||
}
|
||||
@@ -0,0 +1,156 @@
|
||||
//! Kimi K2 Parser Integration Tests
|
||||
|
||||
use smg::tool_parser::{KimiK2Parser, ToolParser};
|
||||
|
||||
use crate::common::create_test_tools;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_kimik2_complete_parsing() {
|
||||
let parser = KimiK2Parser::new();
|
||||
|
||||
let input = r#"Let me help you with that.
|
||||
<|tool_calls_section_begin|>
|
||||
<|tool_call_begin|>functions.get_weather:0<|tool_call_argument_begin|>{"location": "Tokyo", "units": "celsius"}<|tool_call_end|>
|
||||
<|tool_calls_section_end|>
|
||||
The weather in Tokyo is..."#;
|
||||
|
||||
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
assert_eq!(normal_text, "Let me help you with that.\n");
|
||||
assert_eq!(tools[0].function.name, "get_weather");
|
||||
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert_eq!(args["location"], "Tokyo");
|
||||
assert_eq!(args["units"], "celsius");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_kimik2_multiple_tools() {
|
||||
let parser = KimiK2Parser::new();
|
||||
|
||||
let input = r#"<|tool_calls_section_begin|>
|
||||
<|tool_call_begin|>functions.search:0<|tool_call_argument_begin|>{"query": "rust tutorials"}<|tool_call_end|>
|
||||
<|tool_call_begin|>functions.translate:1<|tool_call_argument_begin|>{"text": "Hello", "to": "ja"}<|tool_call_end|>
|
||||
<|tool_calls_section_end|>"#;
|
||||
|
||||
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 2);
|
||||
assert_eq!(normal_text, "");
|
||||
assert_eq!(tools[0].function.name, "search");
|
||||
assert_eq!(tools[1].function.name, "translate");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_kimik2_with_whitespace() {
|
||||
let parser = KimiK2Parser::new();
|
||||
|
||||
let input = r#"<|tool_calls_section_begin|>
|
||||
<|tool_call_begin|> functions.test:0 <|tool_call_argument_begin|> {"key": "value", "num": 42} <|tool_call_end|>
|
||||
<|tool_calls_section_end|>"#;
|
||||
|
||||
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
assert_eq!(normal_text, "");
|
||||
assert_eq!(tools[0].function.name, "test");
|
||||
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert_eq!(args["key"], "value");
|
||||
assert_eq!(args["num"], 42);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_kimik2_streaming() {
|
||||
let tools = create_test_tools();
|
||||
|
||||
let mut parser = KimiK2Parser::new();
|
||||
|
||||
// Simulate streaming chunks
|
||||
let chunks = vec![
|
||||
"<|tool_calls_section_begin|>\n",
|
||||
"<|tool_call_begin|>functions.",
|
||||
"calculate:0",
|
||||
"<|tool_call_argument_begin|>",
|
||||
r#"{"x": 10, "#,
|
||||
r#""y": 20}"#,
|
||||
"<|tool_call_end|>\n",
|
||||
"<|tool_calls_section_end|>",
|
||||
];
|
||||
|
||||
let mut found_name = false;
|
||||
|
||||
for chunk in chunks {
|
||||
let result = parser.parse_incremental(chunk, &tools).await.unwrap();
|
||||
|
||||
for call in result.calls {
|
||||
if let Some(name) = call.name {
|
||||
assert_eq!(name, "calculate");
|
||||
found_name = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
assert!(found_name, "Should have found tool name during streaming");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_kimik2_format_detection() {
|
||||
let parser = KimiK2Parser::new();
|
||||
|
||||
// Should detect Kimi K2 format
|
||||
assert!(parser.has_tool_markers("<|tool_calls_section_begin|>"));
|
||||
assert!(parser.has_tool_markers("text with <|tool_calls_section_begin|> marker"));
|
||||
|
||||
// Should not detect other formats
|
||||
assert!(!parser.has_tool_markers("[TOOL_CALLS]"));
|
||||
assert!(!parser.has_tool_markers("<tool_call>"));
|
||||
assert!(!parser.has_tool_markers("plain text"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_kimik2_sequential_indices() {
|
||||
let parser = KimiK2Parser::new();
|
||||
|
||||
let input = r#"<|tool_calls_section_begin|>
|
||||
<|tool_call_begin|>functions.first:0<|tool_call_argument_begin|>{"param": "a"}<|tool_call_end|>
|
||||
<|tool_call_begin|>functions.second:1<|tool_call_argument_begin|>{"param": "b"}<|tool_call_end|>
|
||||
<|tool_call_begin|>functions.third:2<|tool_call_argument_begin|>{"param": "c"}<|tool_call_end|>
|
||||
<|tool_calls_section_end|>"#;
|
||||
|
||||
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 3);
|
||||
assert_eq!(normal_text, "");
|
||||
assert_eq!(tools[0].function.name, "first");
|
||||
assert_eq!(tools[1].function.name, "second");
|
||||
assert_eq!(tools[2].function.name, "third");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_function_index_extraction() {
|
||||
let parser = KimiK2Parser::new();
|
||||
|
||||
let input = r#"Text before tool calls.
|
||||
<|tool_calls_section_begin|>
|
||||
<|tool_call_begin|>functions.search:0<|tool_call_argument_begin|>{"query": "rust"}<|tool_call_end|>
|
||||
<|tool_call_begin|>functions.calc:1<|tool_call_argument_begin|>{"x": 10}<|tool_call_end|>
|
||||
<|tool_calls_section_end|>"#;
|
||||
|
||||
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 2);
|
||||
assert_eq!(normal_text, "Text before tool calls.\n");
|
||||
assert_eq!(tools[0].function.name, "search");
|
||||
assert_eq!(tools[1].function.name, "calc");
|
||||
// TODO: Verify indices are preserved: 0 and 1
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_namespace_extraction() {
|
||||
let parser = KimiK2Parser::new();
|
||||
|
||||
let input = r#"<|tool_calls_section_begin|>
|
||||
<|tool_call_begin|>api.tools.search:0<|tool_call_argument_begin|>{"q": "test"}<|tool_call_end|>
|
||||
<|tool_calls_section_end|>"#;
|
||||
|
||||
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
assert_eq!(tools[0].function.name, "api.tools.search"); // Includes full namespace
|
||||
}
|
||||
@@ -0,0 +1,454 @@
|
||||
//! Llama Parser Integration Tests
|
||||
//!
|
||||
//! Tests for the Llama parser which handles <|python_tag|> format and plain JSON
|
||||
|
||||
use smg::tool_parser::{LlamaParser, ToolParser};
|
||||
|
||||
use crate::common::{create_test_tools, streaming_helpers::*};
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_llama_python_tag_format() {
|
||||
let parser = LlamaParser::new();
|
||||
let input = r#"Here are some results: <|python_tag|>{"name": "search", "parameters": {"query": "weather"}}"#;
|
||||
|
||||
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
assert_eq!(tools[0].function.name, "search");
|
||||
assert_eq!(normal_text, "Here are some results: ");
|
||||
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert_eq!(args["query"], "weather");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_llama_with_semicolon_separation() {
|
||||
let parser = LlamaParser::new();
|
||||
|
||||
let input = r#"<|python_tag|>{"name": "tool1", "parameters": {}};{"name": "tool2", "parameters": {"y": 2}}"#;
|
||||
|
||||
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 2);
|
||||
assert_eq!(tools[0].function.name, "tool1");
|
||||
assert_eq!(tools[1].function.name, "tool2");
|
||||
assert_eq!(normal_text, "");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_llama_no_tool_calls() {
|
||||
let parser = LlamaParser::new();
|
||||
|
||||
let input = "This is just plain text with no tool calls";
|
||||
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 0);
|
||||
assert_eq!(normal_text, input);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_llama_plain_json_fallback() {
|
||||
let parser = LlamaParser::new();
|
||||
let input = r#"{"name": "calculate", "parameters": {"x": 5, "y": 10}}"#;
|
||||
|
||||
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
assert_eq!(tools[0].function.name, "calculate");
|
||||
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert_eq!(args["x"], 5);
|
||||
assert_eq!(args["y"], 10);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_llama_with_text_before() {
|
||||
let parser = LlamaParser::new();
|
||||
let input = r#"Let me help you with that. <|python_tag|>{"name": "get_time", "parameters": {"timezone": "UTC"}}"#;
|
||||
|
||||
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
assert_eq!(normal_text, "Let me help you with that. ");
|
||||
assert_eq!(tools[0].function.name, "get_time");
|
||||
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert_eq!(args["timezone"], "UTC");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_llama_with_nested_json() {
|
||||
let parser = LlamaParser::new();
|
||||
let input = r#"<|python_tag|>{
|
||||
"name": "update_settings",
|
||||
"parameters": {
|
||||
"preferences": {
|
||||
"theme": "dark",
|
||||
"language": "en"
|
||||
},
|
||||
"notifications": true
|
||||
}
|
||||
}"#;
|
||||
|
||||
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
assert_eq!(tools[0].function.name, "update_settings");
|
||||
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert_eq!(args["preferences"]["theme"], "dark");
|
||||
assert_eq!(args["notifications"], true);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_llama_empty_arguments() {
|
||||
let parser = LlamaParser::new();
|
||||
|
||||
// With python_tag
|
||||
let input = r#"<|python_tag|>{"name": "ping", "parameters": {}}"#;
|
||||
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
assert_eq!(tools[0].function.name, "ping");
|
||||
|
||||
// Plain JSON
|
||||
let input = r#"{"name": "ping", "parameters": {}}"#;
|
||||
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
assert_eq!(tools[0].function.name, "ping");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_llama_format_detection() {
|
||||
let parser = LlamaParser::new();
|
||||
|
||||
assert!(parser.has_tool_markers(r#"<|python_tag|>{"name": "test"}"#));
|
||||
assert!(parser.has_tool_markers(r#"{"name": "test", "parameters": {}}"#));
|
||||
assert!(!parser.has_tool_markers("plain text"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_llama_invalid_json_after_tag() {
|
||||
let parser = LlamaParser::new();
|
||||
|
||||
let input = r#"<|python_tag|>{"name": invalid}"#;
|
||||
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 0);
|
||||
assert_eq!(normal_text, "<|python_tag|>{\"name\": invalid}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_llama_real_world_output() {
|
||||
let parser = LlamaParser::new();
|
||||
|
||||
// Actual output from Llama 3.2 model - simplified for testing
|
||||
let input = r#"I'll search for that information for you.
|
||||
|
||||
<|python_tag|>{"name": "web_search", "parameters": {"query": "Llama 3.2 model capabilities", "num_results": 5, "search_type": "recent"}}"#;
|
||||
|
||||
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
assert_eq!(tools[0].function.name, "web_search");
|
||||
|
||||
let formatted_input = r#"<|python_tag|>{
|
||||
"name": "get_current_time",
|
||||
"parameters": {
|
||||
"timezone": "America/New_York",
|
||||
"format": "ISO8601"
|
||||
}
|
||||
}"#;
|
||||
|
||||
let (_normal_text, tools2) = parser.parse_complete(formatted_input).await.unwrap();
|
||||
assert_eq!(tools2.len(), 1);
|
||||
assert_eq!(tools2[0].function.name, "get_current_time");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_single_json() {
|
||||
let parser = LlamaParser::new();
|
||||
let text = r#"{"name": "get_weather", "parameters": {"city": "Paris"}}"#;
|
||||
|
||||
let (_normal_text, tools) = parser.parse_complete(text).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
assert_eq!(tools[0].function.name, "get_weather");
|
||||
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert_eq!(args["city"], "Paris");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_multiple_json_with_separator() {
|
||||
let parser = LlamaParser::new();
|
||||
let text = r#"<|python_tag|>{"name": "get_weather", "parameters": {"city": "Paris"}};{"name": "get_tourist_attractions", "parameters": {"city": "Paris"}}"#;
|
||||
|
||||
let (_normal_text, tools) = parser.parse_complete(text).await.unwrap();
|
||||
// Note: Current implementation may only parse the first one due to semicolon handling
|
||||
assert!(!tools.is_empty());
|
||||
assert_eq!(tools[0].function.name, "get_weather");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_json_with_trailing_text() {
|
||||
let parser = LlamaParser::new();
|
||||
// Valid JSON with trailing text - LlamaParser doesn't support this mixed format
|
||||
let text = r#"{"name": "get_weather", "parameters": {}} Some follow-up text"#;
|
||||
|
||||
let (normal_text, tools) = parser.parse_complete(text).await.unwrap();
|
||||
// LlamaParser expects pure JSON or <|python_tag|> format, not JSON with trailing text
|
||||
// So this returns as normal text
|
||||
assert_eq!(tools.len(), 0);
|
||||
assert_eq!(normal_text, text);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_invalid_then_valid_json() {
|
||||
let parser = LlamaParser::new();
|
||||
let text =
|
||||
r#"{"name": "get_weather", "parameters": {{"name": "get_weather", "parameters": {}}"#;
|
||||
|
||||
let (_normal_text, tools) = parser.parse_complete(text).await.unwrap();
|
||||
// Should parse at least one valid JSON
|
||||
if !tools.is_empty() {
|
||||
assert_eq!(tools[0].function.name, "get_weather");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_plain_text_only() {
|
||||
let parser = LlamaParser::new();
|
||||
let text = "This is just plain explanation text.";
|
||||
|
||||
let (_normal_text, tools) = parser.parse_complete(text).await.unwrap();
|
||||
assert_eq!(tools.len(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_with_python_tag_prefix() {
|
||||
let parser = LlamaParser::new();
|
||||
let text = r#"Some intro. <|python_tag|>{"name": "get_weather", "parameters": {}}"#;
|
||||
|
||||
let (_normal_text, tools) = parser.parse_complete(text).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
assert_eq!(tools[0].function.name, "get_weather");
|
||||
}
|
||||
|
||||
// STREAMING TESTS
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_llama_streaming_simple() {
|
||||
let tools = create_test_tools();
|
||||
|
||||
let mut parser = LlamaParser::new();
|
||||
|
||||
// Send complete JSON at once
|
||||
let full_json = r#"<|python_tag|>{"name": "search", "parameters": {"query": "weather"}}"#;
|
||||
|
||||
let result = parser.parse_incremental(full_json, &tools).await.unwrap();
|
||||
|
||||
assert!(
|
||||
!result.calls.is_empty(),
|
||||
"Expected tool call for complete JSON input"
|
||||
);
|
||||
assert_eq!(result.calls[0].name.as_ref().unwrap(), "search");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_llama_streaming_partial() {
|
||||
let tools = create_test_tools();
|
||||
|
||||
let mut parser = LlamaParser::new();
|
||||
|
||||
// Stream in chunks
|
||||
let chunks = vec![
|
||||
r#"<|python"#,
|
||||
r#"_tag|>{"name": "#,
|
||||
r#""calculate", "#,
|
||||
r#""parameters": {"x": 10}"#,
|
||||
r#"}"#,
|
||||
];
|
||||
|
||||
let mut got_complete = false;
|
||||
|
||||
for chunk in chunks {
|
||||
let result = parser.parse_incremental(chunk, &tools).await.unwrap();
|
||||
if !result.calls.is_empty() {
|
||||
if let Some(name) = &result.calls[0].name {
|
||||
assert_eq!(name, "calculate");
|
||||
got_complete = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
assert!(got_complete, "Should have completed parsing");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_llama_streaming_plain_json() {
|
||||
let tools = create_test_tools();
|
||||
|
||||
let mut parser = LlamaParser::new();
|
||||
|
||||
// Stream plain JSON without python_tag
|
||||
let chunks = vec![
|
||||
r#"{"name": "#,
|
||||
r#""search", "#,
|
||||
r#""parameters": "#,
|
||||
r#"{"query": "#,
|
||||
r#""test"}}"#,
|
||||
];
|
||||
|
||||
let mut got_complete = false;
|
||||
|
||||
for chunk in chunks {
|
||||
let result = parser.parse_incremental(chunk, &tools).await.unwrap();
|
||||
if !result.calls.is_empty() {
|
||||
if let Some(name) = &result.calls[0].name {
|
||||
assert_eq!(name, "search");
|
||||
got_complete = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
assert!(got_complete, "Should have completed parsing");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_llama_streaming_with_text_before() {
|
||||
let tools = create_test_tools();
|
||||
|
||||
let mut parser = LlamaParser::new();
|
||||
|
||||
let chunks = vec![
|
||||
r#"Let me help you. "#,
|
||||
r#"<|python_tag|>"#,
|
||||
r#"{"name": "get_time","#,
|
||||
r#" "parameters": {"#,
|
||||
r#""timezone": "UTC"}}"#,
|
||||
];
|
||||
|
||||
let mut got_complete = false;
|
||||
|
||||
for chunk in chunks {
|
||||
let result = parser.parse_incremental(chunk, &tools).await.unwrap();
|
||||
if !result.calls.is_empty() {
|
||||
if let Some(name) = &result.calls[0].name {
|
||||
assert_eq!(name, "get_time");
|
||||
got_complete = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
assert!(got_complete, "Should have completed parsing");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_llama_streaming_multiple_tools() {
|
||||
let tools = create_test_tools();
|
||||
|
||||
let mut parser = LlamaParser::new();
|
||||
|
||||
let text =
|
||||
r#"<|python_tag|>{"name": "func1", "parameters": {}};{"name": "func2", "parameters": {}}"#;
|
||||
|
||||
let result = parser.parse_incremental(text, &tools).await.unwrap();
|
||||
|
||||
// Should get first tool complete
|
||||
assert!(
|
||||
!result.calls.is_empty(),
|
||||
"Expected first tool to be complete"
|
||||
);
|
||||
if let Some(name) = &result.calls[0].name {
|
||||
assert_eq!(name, "func1");
|
||||
}
|
||||
|
||||
// Process remaining buffer to get second tool
|
||||
let result2 = parser.parse_incremental("", &tools).await.unwrap();
|
||||
if !result2.calls.is_empty() {
|
||||
if let Some(name) = &result2.calls[0].name {
|
||||
assert_eq!(name, "func2");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_llama_streaming_multiple_tools_chunked() {
|
||||
let mut parser = LlamaParser::new();
|
||||
|
||||
let tools = create_test_tools();
|
||||
|
||||
// First chunk - incomplete first JSON
|
||||
let chunk1 = r#"<|python_tag|>{"name": "get_weather", "parameters""#;
|
||||
let result1 = parser.parse_incremental(chunk1, &tools).await.unwrap();
|
||||
if !result1.calls.is_empty() {
|
||||
if let Some(name) = &result1.calls[0].name {
|
||||
assert_eq!(name, "get_weather");
|
||||
}
|
||||
}
|
||||
|
||||
// Second chunk - complete first JSON and separator
|
||||
let chunk2 = r#": {"city": "Paris"}};{"name": "#;
|
||||
let result2 = parser.parse_incremental(chunk2, &tools).await.unwrap();
|
||||
|
||||
// Should get parameters for first tool (name already sent in result1)
|
||||
if !result2.calls.is_empty() {
|
||||
let args: serde_json::Value = serde_json::from_str(&result2.calls[0].parameters).unwrap();
|
||||
assert_eq!(args["city"], "Paris");
|
||||
}
|
||||
|
||||
let chunk3 = r#""get_time", "parameters": {"timezone": "UTC"}}"#;
|
||||
let result3 = parser.parse_incremental(chunk3, &tools).await.unwrap();
|
||||
if !result3.calls.is_empty() {
|
||||
if let Some(name) = &result3.calls[0].name {
|
||||
assert_eq!(name, "get_time");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// REALISTIC STREAMING TESTS
|
||||
// =============================================================================
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_llama_realistic_chunks_with_python_tag() {
|
||||
let tools = create_test_tools();
|
||||
let mut parser = LlamaParser::new();
|
||||
|
||||
let input = r#"<|python_tag|>{"name": "calculate", "parameters": {"x": 10, "y": 20}}"#;
|
||||
let chunks = create_realistic_chunks(input);
|
||||
|
||||
assert!(chunks.len() > 15, "Should have many small chunks");
|
||||
|
||||
let mut got_tool_name = false;
|
||||
|
||||
for chunk in chunks {
|
||||
let result = parser.parse_incremental(&chunk, &tools).await.unwrap();
|
||||
for call in result.calls {
|
||||
if let Some(name) = call.name {
|
||||
assert_eq!(name, "calculate");
|
||||
got_tool_name = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
assert!(got_tool_name, "Should have parsed tool name");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_llama_python_tag_arrives_in_parts() {
|
||||
let tools = create_test_tools();
|
||||
let mut parser = LlamaParser::new();
|
||||
|
||||
// Python tag itself arrives in small chunks
|
||||
let chunks = vec![
|
||||
"<|p", "yth", "on_", "tag", "|>{", r#"""#, "na", r#"me""#, ": ", r#"""#, "sea", "rch",
|
||||
r#"""#, ", ", r#"""#, "par", "ame", "ter", "s", r#"""#, ": {", r#"""#, "q", r#"""#, ": ",
|
||||
r#"""#, "tes", "t", r#"""#, "}}",
|
||||
];
|
||||
|
||||
let mut got_tool_name = false;
|
||||
|
||||
for chunk in chunks {
|
||||
let result = parser.parse_incremental(chunk, &tools).await.unwrap();
|
||||
for call in result.calls {
|
||||
if let Some(name) = call.name {
|
||||
assert_eq!(name, "search");
|
||||
got_tool_name = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
assert!(got_tool_name, "Should have parsed tool name");
|
||||
}
|
||||
@@ -0,0 +1,779 @@
|
||||
//! MiniMax M2 Parser Integration Tests
|
||||
|
||||
use smg::tool_parser::{MinimaxM2Parser, ToolParser};
|
||||
|
||||
use crate::common::create_test_tools;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_minimax_complete_parsing() {
|
||||
let parser = MinimaxM2Parser::new();
|
||||
|
||||
let input = r#"Let me search for that.
|
||||
<minimax:tool_call>
|
||||
<invoke name="get_weather">
|
||||
<parameter name="city">Beijing</parameter>
|
||||
<parameter name="date">2024-12-25</parameter>
|
||||
</invoke>
|
||||
</minimax:tool_call>
|
||||
The weather will be..."#;
|
||||
|
||||
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
assert_eq!(normal_text, "Let me search for that.\n");
|
||||
assert_eq!(tools[0].function.name, "get_weather");
|
||||
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert_eq!(args["city"], "Beijing");
|
||||
assert_eq!(args["date"], "2024-12-25");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_minimax_multiple_tools() {
|
||||
let parser = MinimaxM2Parser::new();
|
||||
|
||||
let input = r#"<minimax:tool_call>
|
||||
<invoke name="search">
|
||||
<parameter name="query">rust tutorials</parameter>
|
||||
</invoke>
|
||||
</minimax:tool_call>
|
||||
<minimax:tool_call>
|
||||
<invoke name="translate">
|
||||
<parameter name="text">Hello World</parameter>
|
||||
<parameter name="target_lang">zh</parameter>
|
||||
</invoke>
|
||||
</minimax:tool_call>"#;
|
||||
|
||||
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 2);
|
||||
assert_eq!(normal_text, "");
|
||||
assert_eq!(tools[0].function.name, "search");
|
||||
assert_eq!(tools[1].function.name, "translate");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_minimax_type_conversion() {
|
||||
let parser = MinimaxM2Parser::new();
|
||||
|
||||
let input = r#"<minimax:tool_call>
|
||||
<invoke name="process">
|
||||
<parameter name="count">42</parameter>
|
||||
<parameter name="rate">1.5</parameter>
|
||||
<parameter name="enabled">true</parameter>
|
||||
<parameter name="data">null</parameter>
|
||||
<parameter name="text">string value</parameter>
|
||||
</invoke>
|
||||
</minimax:tool_call>"#;
|
||||
|
||||
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
assert_eq!(normal_text, "");
|
||||
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert_eq!(args["count"], 42);
|
||||
assert_eq!(args["rate"], 1.5);
|
||||
assert_eq!(args["enabled"], true);
|
||||
assert_eq!(args["data"], serde_json::Value::Null);
|
||||
assert_eq!(args["text"], "string value");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_minimax_streaming_basic() {
|
||||
let mut parser = MinimaxM2Parser::new();
|
||||
|
||||
let tools = create_test_tools();
|
||||
|
||||
// Simulate streaming chunks
|
||||
let chunks = vec![
|
||||
"<minimax:tool_call>",
|
||||
r#"<invoke name="get_weather">"#,
|
||||
r#"<parameter name="city">Shanghai</parameter>"#,
|
||||
r#"<parameter name="units">celsius</parameter>"#,
|
||||
"</invoke>",
|
||||
"</minimax:tool_call>",
|
||||
];
|
||||
|
||||
let mut found_name = false;
|
||||
let mut found_params = false;
|
||||
|
||||
for chunk in chunks {
|
||||
let result = parser.parse_incremental(chunk, &tools).await.unwrap();
|
||||
|
||||
for call in result.calls {
|
||||
if let Some(name) = call.name {
|
||||
assert_eq!(name, "get_weather");
|
||||
found_name = true;
|
||||
}
|
||||
if !call.parameters.is_empty() {
|
||||
found_params = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
assert!(found_name, "Should have found tool name during streaming");
|
||||
assert!(found_params, "Should have streamed parameters");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_minimax_format_detection() {
|
||||
let parser = MinimaxM2Parser::new();
|
||||
|
||||
// Should detect MiniMax format
|
||||
assert!(parser.has_tool_markers("<minimax:tool_call>"));
|
||||
assert!(parser.has_tool_markers("text with <minimax:tool_call> marker"));
|
||||
|
||||
// Should not detect other formats
|
||||
assert!(!parser.has_tool_markers("<tool_call>")); // GLM4 format
|
||||
assert!(!parser.has_tool_markers("[TOOL_CALLS]"));
|
||||
assert!(!parser.has_tool_markers("<｜tool▁calls▁begin｜>"));
|
||||
assert!(!parser.has_tool_markers("plain text"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_minimax_python_literals() {
|
||||
let parser = MinimaxM2Parser::new();
|
||||
|
||||
let input = r#"<minimax:tool_call>
|
||||
<invoke name="test_func">
|
||||
<parameter name="bool_true">True</parameter>
|
||||
<parameter name="bool_false">False</parameter>
|
||||
<parameter name="none_val">None</parameter>
|
||||
</invoke>
|
||||
</minimax:tool_call>"#;
|
||||
|
||||
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
assert_eq!(tools[0].function.name, "test_func");
|
||||
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert_eq!(args["bool_true"], true);
|
||||
assert_eq!(args["bool_false"], false);
|
||||
assert_eq!(args["none_val"], serde_json::Value::Null);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_minimax_nested_json_in_parameters() {
|
||||
let parser = MinimaxM2Parser::new();
|
||||
|
||||
let input = r#"<minimax:tool_call>
|
||||
<invoke name="process">
|
||||
<parameter name="data">{"nested": {"key": "value"}}</parameter>
|
||||
<parameter name="list">[1, 2, 3]</parameter>
|
||||
</invoke>
|
||||
</minimax:tool_call>"#;
|
||||
|
||||
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
// JSON-like strings are kept as strings, not parsed as JSON
|
||||
// This matches the behavior of other parsers like GLM4 MOE
|
||||
assert!(args["data"].is_string());
|
||||
assert_eq!(args["data"], r#"{"nested": {"key": "value"}}"#);
|
||||
assert!(args["list"].is_string());
|
||||
assert_eq!(args["list"], "[1, 2, 3]");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_minimax_xml_entities() {
|
||||
let parser = MinimaxM2Parser::new();
|
||||
|
||||
let input = r#"<minimax:tool_call>
|
||||
<invoke name="process">
|
||||
<parameter name="html"><div>content</div></parameter>
|
||||
<parameter name="text">Quote: "hello"</parameter>
|
||||
<parameter name="code">if (a && b) { }</parameter>
|
||||
</invoke>
|
||||
</minimax:tool_call>"#;
|
||||
|
||||
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert_eq!(args["html"], "<div>content</div>");
|
||||
assert_eq!(args["text"], "Quote: \"hello\"");
|
||||
assert_eq!(args["code"], "if (a && b) { }");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_minimax_streaming_partial_tags() {
|
||||
let mut parser = MinimaxM2Parser::new();
|
||||
let tools = create_test_tools();
|
||||
|
||||
// Chunks split mid-tag
|
||||
let chunks = vec![
|
||||
"<minimax:tool_c",
|
||||
"all><invoke na",
|
||||
r#"me="get_weather"><param"#,
|
||||
r#"eter name="city">Bei"#,
|
||||
"jing</parameter></inv",
|
||||
"oke></minimax:tool_call>",
|
||||
];
|
||||
|
||||
let mut found_name = false;
|
||||
let mut buffer = String::new();
|
||||
|
||||
for chunk in chunks {
|
||||
let result = parser.parse_incremental(chunk, &tools).await.unwrap();
|
||||
|
||||
buffer.push_str(&result.normal_text);
|
||||
|
||||
for call in result.calls {
|
||||
if let Some(name) = call.name {
|
||||
assert_eq!(name, "get_weather");
|
||||
found_name = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
assert!(
|
||||
found_name,
|
||||
"Should have parsed function name from partial chunks"
|
||||
);
|
||||
assert_eq!(buffer, "");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_minimax_streaming_incremental_json() {
|
||||
let mut parser = MinimaxM2Parser::new();
|
||||
let tools = create_test_tools();
|
||||
|
||||
let chunks = vec![
|
||||
"<minimax:tool_call>",
|
||||
r#"<invoke name="get_weather">"#,
|
||||
r#"<parameter name="city">Paris</parameter>"#,
|
||||
r#"<parameter name="units">metric</parameter>"#,
|
||||
"</invoke></minimax:tool_call>",
|
||||
];
|
||||
|
||||
let mut json_fragments = Vec::new();
|
||||
let mut found_function = false;
|
||||
|
||||
for chunk in chunks {
|
||||
let result = parser.parse_incremental(chunk, &tools).await.unwrap();
|
||||
|
||||
for call in result.calls {
|
||||
if let Some(_name) = call.name {
|
||||
found_function = true;
|
||||
}
|
||||
if !call.parameters.is_empty() {
|
||||
json_fragments.push(call.parameters.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
assert!(found_function);
|
||||
|
||||
// Verify JSON was built incrementally
|
||||
assert!(!json_fragments.is_empty());
|
||||
|
||||
// First fragment should start with opening brace
|
||||
if let Some(first) = json_fragments.first() {
|
||||
assert!(
|
||||
first.starts_with('{'),
|
||||
"First JSON fragment should start with '{{': {}",
|
||||
first
|
||||
);
|
||||
}
|
||||
|
||||
// Last fragment should be closing brace
|
||||
if let Some(last) = json_fragments.last() {
|
||||
assert!(
|
||||
last.contains('}'),
|
||||
"Last JSON fragment should contain '}}': {}",
|
||||
last
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_minimax_multiple_tools_boundary() {
|
||||
let mut parser = MinimaxM2Parser::new();
|
||||
let tools = create_test_tools();
|
||||
|
||||
// Tool boundary at chunk boundary
|
||||
let chunks = vec![
|
||||
r#"<minimax:tool_call><invoke name="get_weather"><parameter name="city">Tokyo</parameter></invoke></minimax:tool_call>"#,
|
||||
r#"<minimax:tool_call><invoke name="search"><parameter name="query">weather forecast</parameter></invoke></minimax:tool_call>"#,
|
||||
];
|
||||
|
||||
let mut tool_names = Vec::new();
|
||||
|
||||
for chunk in chunks {
|
||||
let result = parser.parse_incremental(chunk, &tools).await.unwrap();
|
||||
|
||||
for call in result.calls {
|
||||
if let Some(name) = call.name {
|
||||
tool_names.push(name);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
assert_eq!(tool_names.len(), 2);
|
||||
assert_eq!(tool_names[0], "get_weather");
|
||||
assert_eq!(tool_names[1], "search");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_minimax_invalid_function_name() {
|
||||
let mut parser = MinimaxM2Parser::new();
|
||||
let tools = create_test_tools();
|
||||
|
||||
let chunks = vec![
|
||||
"<minimax:tool_call>",
|
||||
r#"<invoke name="invalid_function">"#,
|
||||
r#"<parameter name="param">value</parameter>"#,
|
||||
"</invoke></minimax:tool_call>",
|
||||
];
|
||||
|
||||
let mut found_invalid = false;
|
||||
|
||||
for chunk in chunks {
|
||||
let result = parser.parse_incremental(chunk, &tools).await.unwrap();
|
||||
|
||||
// Invalid function should be skipped
|
||||
for call in result.calls {
|
||||
if let Some(name) = call.name {
|
||||
if name == "invalid_function" {
|
||||
found_invalid = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
assert!(!found_invalid, "Invalid function should not be parsed");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_minimax_empty_parameters() {
|
||||
let parser = MinimaxM2Parser::new();
|
||||
|
||||
let input = r#"<minimax:tool_call>
|
||||
<invoke name="simple_func">
|
||||
</invoke>
|
||||
</minimax:tool_call>"#;
|
||||
|
||||
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
assert_eq!(tools[0].function.name, "simple_func");
|
||||
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert_eq!(args, serde_json::json!({}));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_minimax_multiline_parameter_values() {
|
||||
let parser = MinimaxM2Parser::new();
|
||||
|
||||
let input = r#"<minimax:tool_call>
|
||||
<invoke name="process">
|
||||
<parameter name="multiline">line1
|
||||
line2
|
||||
line3</parameter>
|
||||
<parameter name="unicode">你好世界 🌍</parameter>
|
||||
</invoke>
|
||||
</minimax:tool_call>"#;
|
||||
|
||||
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert_eq!(args["multiline"], "line1\nline2\nline3");
|
||||
assert_eq!(args["unicode"], "你好世界 🌍");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_minimax_nested_xml_like_content() {
|
||||
let parser = MinimaxM2Parser::new();
|
||||
|
||||
let input = r#"<minimax:tool_call>
|
||||
<invoke name="process">
|
||||
<parameter name="template"><html><body>Hello</body></html></parameter>
|
||||
<parameter name="config">{"key": "<value>nested</value>"}</parameter>
|
||||
</invoke>
|
||||
</minimax:tool_call>"#;
|
||||
|
||||
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert_eq!(args["template"], "<html><body>Hello</body></html>");
|
||||
|
||||
// The nested JSON with XML-like content
|
||||
let config =
|
||||
serde_json::from_str::<serde_json::Value>(args["config"].as_str().unwrap()).unwrap();
|
||||
assert_eq!(config["key"], "<value>nested</value>");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_minimax_streaming_state_reset() {
|
||||
let mut parser = MinimaxM2Parser::new();
|
||||
let tools = create_test_tools();
|
||||
|
||||
// First tool
|
||||
let chunks1 = vec![
|
||||
r#"<minimax:tool_call><invoke name="get_weather">"#,
|
||||
r#"<parameter name="city">London</parameter>"#,
|
||||
"</invoke></minimax:tool_call>",
|
||||
];
|
||||
|
||||
for chunk in chunks1 {
|
||||
parser.parse_incremental(chunk, &tools).await.unwrap();
|
||||
}
|
||||
|
||||
// Second tool - state should be reset
|
||||
let chunks2 = vec![
|
||||
r#"<minimax:tool_call><invoke name="search">"#,
|
||||
r#"<parameter name="query">rust</parameter>"#,
|
||||
"</invoke></minimax:tool_call>",
|
||||
];
|
||||
|
||||
let mut second_tool_name = None;
|
||||
for chunk in chunks2 {
|
||||
let result = parser.parse_incremental(chunk, &tools).await.unwrap();
|
||||
for call in result.calls {
|
||||
if let Some(name) = call.name {
|
||||
second_tool_name = Some(name);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
assert_eq!(second_tool_name, Some("search".to_string()));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_minimax_many_parameters() {
|
||||
let parser = MinimaxM2Parser::new();
|
||||
|
||||
let mut params_xml = String::new();
|
||||
for i in 1..=20 {
|
||||
params_xml.push_str(&format!(
|
||||
r#"<parameter name="param{}">value{}</parameter>
|
||||
"#,
|
||||
i, i
|
||||
));
|
||||
}
|
||||
|
||||
let input = format!(
|
||||
r#"<minimax:tool_call>
|
||||
<invoke name="complex_func">
|
||||
{}
|
||||
</invoke>
|
||||
</minimax:tool_call>"#,
|
||||
params_xml
|
||||
);
|
||||
|
||||
let (_normal_text, tools) = parser.parse_complete(&input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
assert_eq!(tools[0].function.name, "complex_func");
|
||||
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
|
||||
// Verify all 20 parameters are parsed
|
||||
for i in 1..=20 {
|
||||
let key = format!("param{}", i);
|
||||
let expected_value = format!("value{}", i);
|
||||
assert_eq!(args[key], expected_value);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_minimax_character_by_character_streaming() {
|
||||
// Test character-by-character streaming to simulate real-world streaming
|
||||
let mut parser = MinimaxM2Parser::new();
|
||||
let tools = create_test_tools();
|
||||
|
||||
let complete_text = r#"Let me help you. <minimax:tool_call>
|
||||
<invoke name="get_weather">
|
||||
<parameter name="city">Seattle</parameter>
|
||||
<parameter name="units">celsius</parameter>
|
||||
</invoke>
|
||||
</minimax:tool_call> Here are the results."#;
|
||||
|
||||
let mut content_collected = String::new();
|
||||
let mut tool_name_found = false;
|
||||
let mut parameters_found = Vec::new();
|
||||
|
||||
// Stream character by character - feed only one character at a time
|
||||
for i in 0..complete_text.len() {
|
||||
let delta = &complete_text[i..i + 1];
|
||||
let result = parser.parse_incremental(delta, &tools).await.unwrap();
|
||||
content_collected.push_str(&result.normal_text);
|
||||
|
||||
for call in result.calls {
|
||||
if let Some(name) = call.name {
|
||||
assert_eq!(name, "get_weather");
|
||||
tool_name_found = true;
|
||||
}
|
||||
if !call.parameters.is_empty() && !parameters_found.contains(&call.parameters) {
|
||||
parameters_found.push(call.parameters.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
assert!(
|
||||
tool_name_found,
|
||||
"Should find tool name during character-by-character streaming"
|
||||
);
|
||||
assert!(
|
||||
!parameters_found.is_empty(),
|
||||
"Should find parameters during streaming"
|
||||
);
|
||||
|
||||
// Should have initial content and final content
|
||||
assert!(content_collected.contains("Let me help you."));
|
||||
assert!(content_collected.contains("Here are the results."));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_minimax_content_before_and_after_tool_calls() {
|
||||
let parser = MinimaxM2Parser::new();
|
||||
|
||||
let input = r#"I'll analyze the weather for you now.
|
||||
<minimax:tool_call>
|
||||
<invoke name="get_weather">
|
||||
<parameter name="city">Boston</parameter>
|
||||
<parameter name="state">MA</parameter>
|
||||
</invoke>
|
||||
</minimax:tool_call>
|
||||
Based on the analysis, here's what I found."#;
|
||||
|
||||
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
|
||||
// Verify tool extraction
|
||||
assert_eq!(tools.len(), 1);
|
||||
assert_eq!(tools[0].function.name, "get_weather");
|
||||
|
||||
// Verify content preservation (only text before tool call is returned)
|
||||
assert!(normal_text.contains("I'll analyze the weather for you now."));
|
||||
// Text after tool call is not included in parse_complete
|
||||
assert!(!normal_text.contains("Based on the analysis, here's what I found."));
|
||||
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert_eq!(args["city"], "Boston");
|
||||
assert_eq!(args["state"], "MA");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_minimax_incomplete_tool_call() {
|
||||
let parser = MinimaxM2Parser::new();
|
||||
|
||||
// Incomplete tool call - missing closing tag
|
||||
let input = r#"<minimax:tool_call>
|
||||
<invoke name="get_weather">
|
||||
<parameter name="city">Chicago</parameter>"#;
|
||||
|
||||
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
|
||||
// Should not extract incomplete tool calls
|
||||
assert_eq!(tools.len(), 0);
|
||||
assert_eq!(normal_text, input); // Should return as normal text
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_minimax_malformed_invoke_tag() {
|
||||
let parser = MinimaxM2Parser::new();
|
||||
|
||||
// Malformed invoke tag - missing name attribute
|
||||
let input = r#"<minimax:tool_call>
|
||||
<invoke>
|
||||
<parameter name="city">Miami</parameter>
|
||||
</invoke>
|
||||
</minimax:tool_call>"#;
|
||||
|
||||
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
|
||||
// Should not extract tool calls with malformed invoke tags
|
||||
assert_eq!(tools.len(), 0);
|
||||
assert_eq!(normal_text, input);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_minimax_streaming_with_invalid_function_progressive() {
|
||||
let mut parser = MinimaxM2Parser::new();
|
||||
let tools = create_test_tools();
|
||||
|
||||
// Progressive chunks building an invalid function call
|
||||
let chunks = vec![
|
||||
"<minimax:tool_call>",
|
||||
r#"<invoke name="invalid_function">"#,
|
||||
r#"<parameter name="test">value</parameter>"#,
|
||||
"</invoke>",
|
||||
"</minimax:tool_call>",
|
||||
];
|
||||
|
||||
let mut all_normal_text = String::new();
|
||||
let mut found_valid_tool = false;
|
||||
|
||||
for chunk in chunks {
|
||||
let result = parser.parse_incremental(chunk, &tools).await.unwrap();
|
||||
all_normal_text.push_str(&result.normal_text);
|
||||
|
||||
for call in result.calls {
|
||||
if let Some(name) = call.name {
|
||||
// Should not get here for invalid function
|
||||
if tools.iter().any(|t| t.function.name == name) {
|
||||
found_valid_tool = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
assert!(
|
||||
!found_valid_tool,
|
||||
"Invalid function should not be parsed as tool call"
|
||||
);
|
||||
// The invalid tool call should be returned as normal text
|
||||
assert!(all_normal_text.contains("invalid_function"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_minimax_rapid_streaming_bursts() {
|
||||
// Test handling of rapid streaming bursts (multiple chunks at once)
|
||||
let mut parser = MinimaxM2Parser::new();
|
||||
let tools = create_test_tools();
|
||||
|
||||
let chunks = vec![
|
||||
"<minimax:tool_call><invoke name=\"search\"><parameter name=\"query\">",
|
||||
"rust programming",
|
||||
"</parameter></invoke></minimax:tool_call>",
|
||||
];
|
||||
|
||||
let mut found_function = false;
|
||||
let mut parameters = Vec::new();
|
||||
|
||||
for chunk in chunks {
|
||||
let result = parser.parse_incremental(chunk, &tools).await.unwrap();
|
||||
|
||||
for call in result.calls {
|
||||
if let Some(name) = call.name {
|
||||
assert_eq!(name, "search");
|
||||
found_function = true;
|
||||
}
|
||||
if !call.parameters.is_empty() {
|
||||
parameters.push(call.parameters.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
assert!(found_function);
|
||||
|
||||
// Verify that parameters were streamed correctly
|
||||
let final_params = parameters.join("");
|
||||
assert!(final_params.contains("rust programming"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_minimax_special_characters_in_values() {
|
||||
let parser = MinimaxM2Parser::new();
|
||||
|
||||
let input = r#"<minimax:tool_call>
|
||||
<invoke name="process">
|
||||
<parameter name="text">Special chars: @#$%^&*()</parameter>
|
||||
<parameter name="emoji">🦀 Rust 🚀</parameter>
|
||||
<parameter name="quotes">"double" and 'single' quotes</parameter>
|
||||
</invoke>
|
||||
</minimax:tool_call>"#;
|
||||
|
||||
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert_eq!(args["text"], "Special chars: @#$%^&*()");
|
||||
assert_eq!(args["emoji"], "🦀 Rust 🚀");
|
||||
assert_eq!(args["quotes"], "\"double\" and 'single' quotes");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_minimax_whitespace_handling() {
|
||||
let parser = MinimaxM2Parser::new();
|
||||
|
||||
// Test with various whitespace scenarios
|
||||
let input = r#"<minimax:tool_call>
|
||||
<invoke name="process">
|
||||
<parameter name="trimmed"> spaces around </parameter>
|
||||
<parameter name="newlines">
|
||||
Line 1
|
||||
Line 2
|
||||
</parameter>
|
||||
<parameter name="tabs"> tab separated </parameter>
|
||||
</invoke>
|
||||
</minimax:tool_call>"#;
|
||||
|
||||
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
// Values should preserve internal whitespace but may trim edges based on parser design
|
||||
assert!(args["newlines"].as_str().unwrap().contains("Line 1"));
|
||||
assert!(args["newlines"].as_str().unwrap().contains("Line 2"));
|
||||
assert_eq!(args["tabs"], "\ttab\tseparated\t");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_minimax_no_tools() {
|
||||
// Test input with no tool calls at all
|
||||
let parser = MinimaxM2Parser::new();
|
||||
|
||||
let input = r#"This is just a normal response without any tool calls.
|
||||
I can provide information directly without using any tools.
|
||||
Even if I mention function names like get_weather or search,
|
||||
they are not actual tool calls unless properly formatted."#;
|
||||
|
||||
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
|
||||
// No tools should be extracted
|
||||
assert_eq!(
|
||||
tools.len(),
|
||||
0,
|
||||
"Should not extract any tools from plain text"
|
||||
);
|
||||
|
||||
// All content should be returned as normal text
|
||||
assert_eq!(
|
||||
normal_text, input,
|
||||
"All content should be returned as normal text when no tools present"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_minimax_invalid_json_in_parameters() {
|
||||
// Test handling of invalid JSON in parameter values
|
||||
let parser = MinimaxM2Parser::new();
|
||||
|
||||
let input = r#"<minimax:tool_call>
|
||||
<invoke name="process">
|
||||
<parameter name="valid">{"key": "value"}</parameter>
|
||||
<parameter name="invalid">{invalid json: no quotes}</parameter>
|
||||
<parameter name="broken">[1, 2, unclosed</parameter>
|
||||
<parameter name="mixed">Some text {"partial": json} more text</parameter>
|
||||
</invoke>
|
||||
</minimax:tool_call>"#;
|
||||
|
||||
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
|
||||
// Tool should still be extracted despite invalid JSON in parameters
|
||||
assert_eq!(
|
||||
tools.len(),
|
||||
1,
|
||||
"Should extract tool even with invalid JSON in parameters"
|
||||
);
|
||||
assert_eq!(tools[0].function.name, "process");
|
||||
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
|
||||
// Parameters are stored as strings, not parsed as JSON
|
||||
// Even invalid JSON should be preserved as string values
|
||||
assert!(args["valid"].is_string());
|
||||
assert_eq!(args["valid"], r#"{"key": "value"}"#);
|
||||
|
||||
assert!(args["invalid"].is_string());
|
||||
assert_eq!(args["invalid"], "{invalid json: no quotes}");
|
||||
|
||||
assert!(args["broken"].is_string());
|
||||
assert_eq!(args["broken"], "[1, 2, unclosed");
|
||||
|
||||
assert!(args["mixed"].is_string());
|
||||
assert_eq!(args["mixed"], r#"Some text {"partial": json} more text"#);
|
||||
|
||||
assert_eq!(normal_text, "");
|
||||
}
|
||||
@@ -0,0 +1,274 @@
|
||||
//! Mistral Parser Integration Tests
|
||||
//!
|
||||
//! Tests for the Mistral parser which handles [TOOL_CALLS] format
|
||||
|
||||
use serde_json::json;
|
||||
use smg::tool_parser::{MistralParser, ToolParser};
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_mistral_single_tool() {
|
||||
let parser = MistralParser::new();
|
||||
let input = r#"Let me search for that.
|
||||
[TOOL_CALLS] [{"name": "search_web", "arguments": {"query": "latest news", "max_results": 5}}]"#;
|
||||
|
||||
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
assert_eq!(normal_text, "Let me search for that.\n");
|
||||
assert_eq!(tools[0].function.name, "search_web");
|
||||
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert_eq!(args["query"], "latest news");
|
||||
assert_eq!(args["max_results"], 5);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_mistral_multiple_tools() {
|
||||
let parser = MistralParser::new();
|
||||
let input = r#"I'll help you with both tasks.
|
||||
[TOOL_CALLS] [
|
||||
{"name": "get_weather", "arguments": {"city": "Tokyo", "units": "celsius"}},
|
||||
{"name": "search_news", "arguments": {"query": "AI developments", "limit": 10}}
|
||||
]"#;
|
||||
|
||||
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 2);
|
||||
assert_eq!(normal_text, "I'll help you with both tasks.\n");
|
||||
|
||||
assert_eq!(tools[0].function.name, "get_weather");
|
||||
let args0: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert_eq!(args0["city"], "Tokyo");
|
||||
|
||||
assert_eq!(tools[1].function.name, "search_news");
|
||||
let args1: serde_json::Value = serde_json::from_str(&tools[1].function.arguments).unwrap();
|
||||
assert_eq!(args1["query"], "AI developments");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_mistral_nested_json() {
|
||||
let parser = MistralParser::new();
|
||||
let input = r#"Processing complex data.
|
||||
[TOOL_CALLS] [{"name": "process_data", "arguments": {"config": {"nested": {"value": [1, 2, 3]}}, "enabled": true}}]"#;
|
||||
|
||||
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
assert_eq!(normal_text, "Processing complex data.\n");
|
||||
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert_eq!(args["config"]["nested"]["value"], json!([1, 2, 3]));
|
||||
assert_eq!(args["enabled"], true);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_mistral_with_text_after() {
|
||||
let parser = MistralParser::new();
|
||||
let input = r#"[TOOL_CALLS] [{"name": "test", "arguments": {}}]
|
||||
|
||||
And here's some text after the tool call that should be ignored."#;
|
||||
|
||||
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
assert_eq!(tools[0].function.name, "test");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_mistral_empty_arguments() {
|
||||
let parser = MistralParser::new();
|
||||
let input = r#"[TOOL_CALLS] [{"name": "ping", "arguments": {}}]"#;
|
||||
|
||||
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
assert_eq!(tools[0].function.name, "ping");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_mistral_with_brackets_in_strings() {
|
||||
let parser = MistralParser::new();
|
||||
let input = r#"[TOOL_CALLS] [{"name": "echo", "arguments": {"text": "Array notation: arr[0] = value[1]"}}]"#;
|
||||
|
||||
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert_eq!(args["text"], "Array notation: arr[0] = value[1]");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_mistral_format_detection() {
|
||||
let parser = MistralParser::new();
|
||||
|
||||
assert!(parser.has_tool_markers("[TOOL_CALLS] ["));
|
||||
assert!(parser.has_tool_markers("Some text [TOOL_CALLS] ["));
|
||||
assert!(!parser.has_tool_markers("Just plain text"));
|
||||
assert!(!parser.has_tool_markers("[{\"name\": \"test\"}]")); // JSON array without TOOL_CALLS
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_mistral_malformed_json() {
|
||||
let parser = MistralParser::new();
|
||||
|
||||
// Missing closing bracket
|
||||
let input = r#"[TOOL_CALLS] [{"name": "test", "arguments": {}"#;
|
||||
if let Ok((_normal_text, tools)) = parser.parse_complete(input).await {
|
||||
assert_eq!(tools.len(), 0);
|
||||
}
|
||||
// Error is also acceptable for malformed input
|
||||
|
||||
// Invalid JSON inside
|
||||
let input = r#"[TOOL_CALLS] [{"name": invalid}]"#;
|
||||
if let Ok((_normal_text, tools)) = parser.parse_complete(input).await {
|
||||
assert_eq!(tools.len(), 0);
|
||||
}
|
||||
// Error is also acceptable for malformed input
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_mistral_real_world_output() {
|
||||
let parser = MistralParser::new();
|
||||
|
||||
// Actual output from Mistral model
|
||||
let input = r#"I'll search for information about Rust programming and check the weather in San Francisco.
|
||||
|
||||
[TOOL_CALLS] [
|
||||
{
|
||||
"name": "web_search",
|
||||
"arguments": {
|
||||
"query": "Rust programming language features 2024",
|
||||
"max_results": 3,
|
||||
"include_snippets": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "get_weather",
|
||||
"arguments": {
|
||||
"location": "San Francisco, CA",
|
||||
"units": "fahrenheit",
|
||||
"include_forecast": false
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
Let me execute these searches for you."#;
|
||||
|
||||
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 2);
|
||||
assert_eq!(normal_text, "I'll search for information about Rust programming and check the weather in San Francisco.\n\n");
|
||||
assert_eq!(tools[0].function.name, "web_search");
|
||||
assert_eq!(tools[1].function.name, "get_weather");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_mistral_streaming_closing_bracket() {
|
||||
use smg::protocols::common::Tool;
|
||||
|
||||
// Test that closing ] is stripped for Mistral array format
|
||||
let mut parser = MistralParser::new();
|
||||
|
||||
let tools = vec![Tool {
|
||||
tool_type: "function".to_string(),
|
||||
function: smg::protocols::common::Function {
|
||||
name: "get_weather".to_string(),
|
||||
description: Some("Get weather".to_string()),
|
||||
parameters: json!({}),
|
||||
strict: None,
|
||||
},
|
||||
}];
|
||||
|
||||
let chunks = vec![
|
||||
"[TOOL_CALLS] ",
|
||||
"[{",
|
||||
"\"",
|
||||
"name",
|
||||
"\":",
|
||||
"\"",
|
||||
"get",
|
||||
"_weather",
|
||||
"\",",
|
||||
"\"",
|
||||
"arguments",
|
||||
"\":",
|
||||
"{",
|
||||
"\"",
|
||||
"city",
|
||||
"\":",
|
||||
"\"",
|
||||
"Paris",
|
||||
"\"",
|
||||
"}",
|
||||
"}",
|
||||
"]",
|
||||
" Here's",
|
||||
" the weather",
|
||||
" info",
|
||||
];
|
||||
|
||||
let mut all_normal_text = String::new();
|
||||
|
||||
for chunk in chunks {
|
||||
let result = parser.parse_incremental(chunk, &tools).await.unwrap();
|
||||
all_normal_text.push_str(&result.normal_text);
|
||||
}
|
||||
|
||||
// Should emit only the third chunk as normal text, NOT the ]
|
||||
assert_eq!(
|
||||
all_normal_text, " Here's the weather info",
|
||||
"Should not emit ] for Mistral array format, got: '{}'",
|
||||
all_normal_text
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_mistral_streaming_bracket_in_text_after_tools() {
|
||||
use smg::protocols::common::Tool;
|
||||
|
||||
// Test that ] in normal text AFTER tool calls is preserved
|
||||
let mut parser = MistralParser::new();
|
||||
|
||||
let tools = vec![Tool {
|
||||
tool_type: "function".to_string(),
|
||||
function: smg::protocols::common::Function {
|
||||
name: "get_weather".to_string(),
|
||||
description: Some("Get weather".to_string()),
|
||||
parameters: json!({}),
|
||||
strict: None,
|
||||
},
|
||||
}];
|
||||
|
||||
let chunks = vec![
|
||||
"[TOOL_CALLS] ",
|
||||
"[",
|
||||
"{",
|
||||
"\"name",
|
||||
"\":",
|
||||
"\"get_weather",
|
||||
"\",",
|
||||
"\"arguments",
|
||||
"\":",
|
||||
"{\"",
|
||||
"city",
|
||||
"\":",
|
||||
"\"Paris",
|
||||
"\"}",
|
||||
"}",
|
||||
"]",
|
||||
" Array",
|
||||
" notation:",
|
||||
" arr",
|
||||
"[",
|
||||
"0",
|
||||
"]",
|
||||
];
|
||||
|
||||
let mut all_normal_text = String::new();
|
||||
|
||||
for chunk in chunks {
|
||||
let result = parser.parse_incremental(chunk, &tools).await.unwrap();
|
||||
all_normal_text.push_str(&result.normal_text);
|
||||
}
|
||||
|
||||
// Should preserve ] in normal text after tools complete
|
||||
assert_eq!(
|
||||
all_normal_text, " Array notation: arr[0]",
|
||||
"Should preserve ] in normal text after tools, got: '{}'",
|
||||
all_normal_text
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,290 @@
|
||||
//! Mixed Format and Additional Edge Case Tests
|
||||
//!
|
||||
//! Tests for edge cases across parsers and mixed format scenarios
|
||||
|
||||
use serde_json::json;
|
||||
use smg::tool_parser::{
|
||||
JsonParser, LlamaParser, MistralParser, PythonicParser, QwenParser, ToolParser,
|
||||
};
|
||||
|
||||
use crate::common::create_test_tools;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_mixed_formats_in_text() {
|
||||
let json_parser = JsonParser::new();
|
||||
let input = r#"
|
||||
Some text with [TOOL_CALLS] marker that shouldn't trigger.
|
||||
Also has <tool_call> tags and [function()] syntax.
|
||||
But here's the actual JSON: {"name": "test", "arguments": {}}
|
||||
"#;
|
||||
|
||||
let (_normal_text, tools) = json_parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
assert_eq!(tools[0].function.name, "test");
|
||||
|
||||
// Mistral parser should ignore JSON and other formats
|
||||
let mistral_parser = MistralParser::new();
|
||||
let input = r#"
|
||||
{"name": "fake"} [function()] <tool_call>
|
||||
[TOOL_CALLS] [{"name": "real", "arguments": {}}]
|
||||
"#;
|
||||
|
||||
let (_normal_text, tools) = mistral_parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
assert_eq!(tools[0].function.name, "real");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_format_markers_in_string_content() {
|
||||
let pythonic_parser = PythonicParser::new();
|
||||
let input = r#"[echo(text="Use [TOOL_CALLS] and <tool_call> in text")]"#;
|
||||
|
||||
let (_normal_text, tools) = pythonic_parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert_eq!(args["text"], "Use [TOOL_CALLS] and <tool_call> in text");
|
||||
|
||||
let qwen_parser = QwenParser::new();
|
||||
let input = r#"<tool_call>
|
||||
{"name": "log", "arguments": {"msg": "Found [function()] pattern"}}
|
||||
</tool_call>"#;
|
||||
|
||||
let (_normal_text, tools) = qwen_parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert_eq!(args["msg"], "Found [function()] pattern");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_deeply_nested_json_structures() {
|
||||
let json_parser = JsonParser::new();
|
||||
|
||||
let input = r#"{
|
||||
"name": "deep_process",
|
||||
"arguments": {
|
||||
"level1": {
|
||||
"level2": {
|
||||
"level3": {
|
||||
"level4": {
|
||||
"level5": {
|
||||
"data": [1, 2, [3, [4, 5]]]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}"#;
|
||||
|
||||
let (_normal_text, tools) = json_parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
assert_eq!(tools[0].function.name, "deep_process");
|
||||
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert!(args["level1"]["level2"]["level3"]["level4"]["level5"]["data"].is_array());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_multiple_sequential_calls_different_formats() {
|
||||
// Simulate a scenario where different parts of text have different formats
|
||||
// (though each parser will only recognize its own format)
|
||||
|
||||
let llama_parser = LlamaParser::new();
|
||||
|
||||
// Llama parser currently only returns the first tool found
|
||||
let input = r#"First call: <|python_tag|>{"name": "call1", "arguments": {}}"#;
|
||||
|
||||
let (_normal_text, tools) = llama_parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
assert_eq!(tools[0].function.name, "call1");
|
||||
|
||||
let input2 = r#"{"name": "call2", "arguments": {"x": 1}}"#;
|
||||
let (_normal_text2, tools2) = llama_parser.parse_complete(input2).await.unwrap();
|
||||
assert_eq!(tools2.len(), 1);
|
||||
assert_eq!(tools2[0].function.name, "call2");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_empty_and_whitespace_variations() {
|
||||
let json_parser = JsonParser::new();
|
||||
|
||||
// Various whitespace scenarios
|
||||
let cases = vec![
|
||||
r#" {"name":"compact","arguments":{}} "#,
|
||||
r#"
|
||||
|
||||
{"name": "spaced", "arguments": {}}
|
||||
|
||||
"#,
|
||||
r#" {"name": "tabbed", "arguments": {}} "#, // tabs
|
||||
];
|
||||
|
||||
for input in cases {
|
||||
let (_normal_text, tools) = json_parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1, "Should parse regardless of whitespace");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_special_json_values() {
|
||||
let json_parser = JsonParser::new();
|
||||
|
||||
let input = r#"{
|
||||
"name": "test_special",
|
||||
"arguments": {
|
||||
"float_e": 1.23e10,
|
||||
"float_neg_e": 1.23e-10,
|
||||
"hex_like": "0x1234",
|
||||
"very_long_num": 99999999999999999999,
|
||||
"special_strings": ["", " ", "\u0000", "\u001f"],
|
||||
"escaped": "\\n\\r\\t\\\"\\\\",
|
||||
"unicode": "\u4e2d\u6587"
|
||||
}
|
||||
}"#;
|
||||
|
||||
let (_normal_text, tools) = json_parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
assert_eq!(tools[0].function.name, "test_special");
|
||||
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert!(args["special_strings"].is_array());
|
||||
assert!(args["escaped"].is_string());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_parser_recovery_after_invalid_input() {
|
||||
let mut parser = JsonParser::new();
|
||||
let tools = create_test_tools();
|
||||
|
||||
// Send invalid JSON first
|
||||
let _ = parser.parse_incremental(r#"{"broken": "#, &tools).await;
|
||||
|
||||
// Create a new parser instance for clean state
|
||||
let mut parser2 = JsonParser::new();
|
||||
let result = parser2
|
||||
.parse_incremental(r#"{"name": "valid", "arguments": {}}"#, &tools)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
if !result.calls.is_empty() {
|
||||
if let Some(name) = &result.calls[0].name {
|
||||
assert_eq!(name, "valid");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_boundary_cases_for_extraction() {
|
||||
let json_parser = JsonParser::new();
|
||||
|
||||
// JSON at the very beginning
|
||||
let input = r#"{"name": "start", "arguments": {}} and then text"#;
|
||||
let (_normal_text, tools) = json_parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
assert_eq!(tools[0].function.name, "start");
|
||||
|
||||
// JSON at the very end
|
||||
let input = r#"Some text first {"name": "end", "arguments": {}}"#;
|
||||
let (_normal_text, tools) = json_parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
assert_eq!(tools[0].function.name, "end");
|
||||
|
||||
// Multiple JSON objects in text (should find first valid one)
|
||||
let input =
|
||||
r#"Text {"name": "first", "arguments": {}} more {"name": "second", "arguments": {}}"#;
|
||||
let (_normal_text, tools) = json_parser.parse_complete(input).await.unwrap();
|
||||
assert!(!tools.is_empty());
|
||||
assert_eq!(tools[0].function.name, "first");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_pythonic_edge_cases() {
|
||||
let parser = PythonicParser::new();
|
||||
|
||||
// Function name with underscores and numbers
|
||||
let input = r#"[func_name_2(param_1="value")]"#;
|
||||
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
assert_eq!(tools[0].function.name, "func_name_2");
|
||||
|
||||
// Empty string argument
|
||||
let input = r#"[process(text="")]"#;
|
||||
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert_eq!(args["text"], "");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_mistral_with_pretty_json() {
|
||||
let parser = MistralParser::new();
|
||||
|
||||
// Pretty-printed JSON in Mistral format
|
||||
let input = r#"[TOOL_CALLS] [
|
||||
{
|
||||
"name": "formatted",
|
||||
"arguments": {
|
||||
"nested": {
|
||||
"key": "value"
|
||||
},
|
||||
"array": [
|
||||
1,
|
||||
2,
|
||||
3
|
||||
]
|
||||
}
|
||||
}
|
||||
]"#;
|
||||
|
||||
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
assert_eq!(tools[0].function.name, "formatted");
|
||||
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert_eq!(args["nested"]["key"], "value");
|
||||
assert_eq!(args["array"], json!([1, 2, 3]));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_qwen_with_cdata_like_content() {
|
||||
let parser = QwenParser::new();
|
||||
|
||||
// Note: QwenParser expects exactly "<tool_call>\n" with the newline
|
||||
let input = r#"<tool_call>
|
||||
{"name": "process", "arguments": {"xml": "<![CDATA[some data]]>"}}
|
||||
</tool_call>"#;
|
||||
|
||||
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
assert_eq!(tools[0].function.name, "process");
|
||||
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert_eq!(args["xml"], "<![CDATA[some data]]>");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_extremely_long_function_names() {
|
||||
let parser = PythonicParser::new();
|
||||
|
||||
let long_name = "very_long_function_name_that_might_appear_in_generated_code_somewhere";
|
||||
let input = format!(r#"[{}(param="value")]"#, long_name);
|
||||
|
||||
let (_normal_text, tools) = parser.parse_complete(&input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
assert_eq!(tools[0].function.name, long_name);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_json_with_duplicate_keys() {
|
||||
let parser = JsonParser::new();
|
||||
|
||||
// JSON with duplicate keys (last one should win per JSON spec)
|
||||
let input = r#"{"name": "test", "arguments": {"key": "first", "key": "second"}}"#;
|
||||
|
||||
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
// JSON parsers typically keep the last value for duplicate keys
|
||||
assert_eq!(args["key"], "second");
|
||||
}
|
||||
@@ -0,0 +1,156 @@
|
||||
//! Partial JSON Parser Tests
|
||||
//!
|
||||
//! Tests for the partial JSON parser with allow_partial_strings flag behavior
|
||||
|
||||
use smg::tool_parser::partial_json::PartialJson;
|
||||
|
||||
#[test]
|
||||
fn test_partial_string_flag_disallows_incomplete_strings() {
|
||||
// Test case from the bug report: {"name": "
|
||||
// With allow_partial_strings=false, should return {} (stop before incomplete string)
|
||||
let parser = PartialJson::new(32, true);
|
||||
let input = r#"{"name": ""#;
|
||||
|
||||
let result = parser.parse_value(input, false);
|
||||
assert!(result.is_ok());
|
||||
|
||||
let (obj, consumed) = result.unwrap();
|
||||
|
||||
// Should parse just the opening brace and stop at the incomplete string
|
||||
assert!(obj.is_object());
|
||||
let obj_map = obj.as_object().unwrap();
|
||||
|
||||
// Should have empty object (stopped before parsing incomplete "name" key)
|
||||
assert!(
|
||||
obj_map.is_empty() || !obj_map.contains_key("name"),
|
||||
"Should not parse incomplete string key, got: {:?}",
|
||||
obj_map
|
||||
);
|
||||
|
||||
// Should consume characters up to the incomplete string
|
||||
assert!(consumed <= input.len());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_partial_string_flag_allows_incomplete_strings() {
|
||||
// Test case: {"name": "
|
||||
// With allow_partial_strings=true, should parse the incomplete string
|
||||
let parser = PartialJson::new(32, true);
|
||||
let input = r#"{"name": ""#;
|
||||
|
||||
let result = parser.parse_value(input, true);
|
||||
assert!(result.is_ok());
|
||||
|
||||
let (obj, consumed) = result.unwrap();
|
||||
|
||||
// Should parse the object with incomplete string value
|
||||
assert!(obj.is_object());
|
||||
let obj_map = obj.as_object().unwrap();
|
||||
|
||||
// With allow_partial_strings=true, should parse "name" key with empty string value
|
||||
assert!(
|
||||
obj_map.contains_key("name"),
|
||||
"Should parse incomplete string with allow_partial_strings=true"
|
||||
);
|
||||
|
||||
assert_eq!(consumed, input.len());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_partial_string_flag_complete_json() {
|
||||
// Test case: {"name": "test"}
|
||||
// Both flags should parse complete JSON the same way
|
||||
let input = r#"{"name": "test"}"#;
|
||||
|
||||
let parser = PartialJson::new(32, true);
|
||||
let result1 = parser.parse_value(input, false);
|
||||
assert!(result1.is_ok());
|
||||
let (obj1, consumed1) = result1.unwrap();
|
||||
|
||||
let result2 = parser.parse_value(input, true);
|
||||
assert!(result2.is_ok());
|
||||
let (obj2, consumed2) = result2.unwrap();
|
||||
|
||||
// Both should parse the same complete JSON
|
||||
assert_eq!(obj1, obj2);
|
||||
assert_eq!(consumed1, consumed2);
|
||||
assert_eq!(consumed1, input.len());
|
||||
|
||||
// Check the parsed value
|
||||
assert!(obj1.is_object());
|
||||
let obj_map = obj1.as_object().unwrap();
|
||||
assert_eq!(obj_map.get("name").and_then(|v| v.as_str()), Some("test"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_backward_compatibility_default() {
|
||||
// Test that default PartialJson still allows partial strings (backward compatible)
|
||||
let parser = PartialJson::default();
|
||||
let input = r#"{"name": ""#;
|
||||
|
||||
let result = parser.parse_value(input, true);
|
||||
assert!(result.is_ok());
|
||||
|
||||
let (obj, _) = result.unwrap();
|
||||
assert!(obj.is_object());
|
||||
|
||||
// Default behavior should allow partial strings
|
||||
let obj_map = obj.as_object().unwrap();
|
||||
assert!(
|
||||
obj_map.contains_key("name"),
|
||||
"Default should allow partial strings for backward compatibility"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_partial_string_in_nested_object() {
|
||||
// Test case: {"tool": {"name": "
|
||||
let parser = PartialJson::new(32, true);
|
||||
let input = r#"{"tool": {"name": ""#;
|
||||
|
||||
let result = parser.parse_value(input, false);
|
||||
assert!(result.is_ok());
|
||||
|
||||
let (obj, _) = result.unwrap();
|
||||
assert!(obj.is_object());
|
||||
|
||||
// With allow_partial_strings=false, should stop before incomplete nested string
|
||||
let obj_map = obj.as_object().unwrap();
|
||||
if let Some(tool) = obj_map.get("tool") {
|
||||
if let Some(tool_map) = tool.as_object() {
|
||||
assert!(
|
||||
!tool_map.contains_key("name")
|
||||
|| tool_map.get("name").and_then(|v| v.as_str()).is_none(),
|
||||
"Should not parse incomplete nested string"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bug_fix_exact_scenario() {
|
||||
// This test verifies the exact bug scenario from the issue:
|
||||
// buffer = "{\"name\": \""
|
||||
// flags = Allow.ALL & ~Allow.STR
|
||||
// Python returns: Parsed object: {}, consumed length: 10
|
||||
|
||||
let parser = PartialJson::new(32, true);
|
||||
let input = r#"{"name": ""#;
|
||||
|
||||
let result = parser.parse_value(input, false);
|
||||
assert!(result.is_ok());
|
||||
|
||||
let (obj, consumed) = result.unwrap();
|
||||
|
||||
// Should return empty object (not {"name": null} or {"name": ""})
|
||||
assert!(obj.is_object());
|
||||
let obj_map = obj.as_object().unwrap();
|
||||
assert!(
|
||||
obj_map.is_empty(),
|
||||
"Expected empty object, got: {:?}. This matches Python behavior with Allow.ALL & ~Allow.STR",
|
||||
obj_map
|
||||
);
|
||||
|
||||
// Should consume all characters (10 bytes)
|
||||
assert_eq!(consumed, 10, "Should consume all 10 characters");
|
||||
}
|
||||
@@ -0,0 +1,517 @@
|
||||
//! Pythonic Parser Integration Tests
|
||||
//!
|
||||
//! Tests for the Pythonic parser which handles Python function call syntax
|
||||
|
||||
use serde_json::json;
|
||||
use smg::tool_parser::{PythonicParser, ToolParser};
|
||||
|
||||
use crate::common::create_test_tools;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_pythonic_single_function() {
|
||||
let parser = PythonicParser::new();
|
||||
let input = r#"[get_weather(city="London", units="celsius")]"#;
|
||||
|
||||
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
assert_eq!(tools[0].function.name, "get_weather");
|
||||
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert_eq!(args["city"], "London");
|
||||
assert_eq!(args["units"], "celsius");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_pythonic_multiple_functions() {
|
||||
let parser = PythonicParser::new();
|
||||
let input =
|
||||
r#"[search_web(query="Rust programming", max_results=5), get_time(timezone="UTC")]"#;
|
||||
|
||||
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 2);
|
||||
assert_eq!(tools[0].function.name, "search_web");
|
||||
assert_eq!(tools[1].function.name, "get_time");
|
||||
|
||||
let args0: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert_eq!(args0["query"], "Rust programming");
|
||||
assert_eq!(args0["max_results"], 5);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_pythonic_with_python_literals() {
|
||||
let parser = PythonicParser::new();
|
||||
let input = r#"[configure(enabled=True, disabled=False, optional=None)]"#;
|
||||
|
||||
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert_eq!(args["enabled"], true);
|
||||
assert_eq!(args["disabled"], false);
|
||||
assert_eq!(args["optional"], json!(null));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_pythonic_with_lists_and_dicts() {
|
||||
let parser = PythonicParser::new();
|
||||
let input =
|
||||
r#"[process_data(items=[1, 2, 3], config={"key": "value", "nested": {"deep": True}})]"#;
|
||||
|
||||
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert_eq!(args["items"], json!([1, 2, 3]));
|
||||
assert_eq!(args["config"]["key"], "value");
|
||||
assert_eq!(args["config"]["nested"]["deep"], true);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_pythonic_with_special_tokens() {
|
||||
let parser = PythonicParser::new();
|
||||
|
||||
// Llama 4 sometimes outputs these tokens
|
||||
let input = r#"<|python_start|>[calculate(x=10, y=20)]<|python_end|>"#;
|
||||
|
||||
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
assert_eq!(tools[0].function.name, "calculate");
|
||||
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert_eq!(args["x"], 10);
|
||||
assert_eq!(args["y"], 20);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_pythonic_with_nested_parentheses() {
|
||||
let parser = PythonicParser::new();
|
||||
let input = r#"[math_eval(expression="(2 + 3) * (4 - 1)", round_to=2)]"#;
|
||||
|
||||
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert_eq!(args["expression"], "(2 + 3) * (4 - 1)");
|
||||
assert_eq!(args["round_to"], 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_pythonic_with_escaped_quotes() {
|
||||
let parser = PythonicParser::new();
|
||||
let input = r#"[echo(text="She said \"Hello\" to him")]"#;
|
||||
|
||||
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert_eq!(args["text"], "She said \"Hello\" to him");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_pythonic_empty_arguments() {
|
||||
let parser = PythonicParser::new();
|
||||
let input = r#"[ping()]"#;
|
||||
|
||||
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
assert_eq!(tools[0].function.name, "ping");
|
||||
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert_eq!(args, json!({}));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_pythonic_format_detection() {
|
||||
let parser = PythonicParser::new();
|
||||
|
||||
assert!(!parser.has_tool_markers("[function_name(")); // Incomplete
|
||||
assert!(parser.has_tool_markers("[get_weather(city=\"NYC\")]"));
|
||||
assert!(!parser.has_tool_markers("Just plain text"));
|
||||
assert!(!parser.has_tool_markers("{\"name\": \"test\"}")); // JSON
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_pythonic_invalid_syntax() {
|
||||
let parser = PythonicParser::new();
|
||||
|
||||
// Missing closing bracket
|
||||
let input = r#"[function(arg=value"#;
|
||||
if let Ok((_normal_text, tools)) = parser.parse_complete(input).await {
|
||||
assert_eq!(tools.len(), 0);
|
||||
}
|
||||
// Error is also acceptable for invalid syntax
|
||||
|
||||
// Invalid Python syntax - empty parameter name
|
||||
// Note: The parser currently accepts this invalid syntax and returns a result
|
||||
// This is a known limitation of the current implementation
|
||||
let input = r#"[function(=value)]"#;
|
||||
if let Ok((_normal_text, tools)) = parser.parse_complete(input).await {
|
||||
// The parser incorrectly accepts this, returning 1 result
|
||||
// We'll accept this behavior for now but note it's not ideal
|
||||
assert!(tools.len() <= 1, "Should parse at most one function");
|
||||
}
|
||||
// Error would be the correct behavior
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_pythonic_real_world_llama4() {
|
||||
let parser = PythonicParser::new();
|
||||
|
||||
// Actual output from Llama 4 model
|
||||
let input = r#"I'll help you with multiple tasks. Let me search for information and perform calculations.
|
||||
|
||||
[web_search(query="latest Rust features", max_results=3, safe_search=True),
|
||||
calculate(expression="42 * 3.14159", precision=2),
|
||||
get_weather(city="San Francisco", units="fahrenheit", include_forecast=False)]
|
||||
|
||||
These functions will provide the information you need."#;
|
||||
|
||||
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 3);
|
||||
assert_eq!(normal_text, "I'll help you with multiple tasks. Let me search for information and perform calculations.\n\n\n\nThese functions will provide the information you need.");
|
||||
assert_eq!(tools[0].function.name, "web_search");
|
||||
assert_eq!(tools[1].function.name, "calculate");
|
||||
assert_eq!(tools[2].function.name, "get_weather");
|
||||
|
||||
let args0: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert_eq!(args0["query"], "latest Rust features");
|
||||
assert_eq!(args0["safe_search"], true);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_pythonic_nested_brackets_in_lists() {
|
||||
let parser = PythonicParser::new();
|
||||
|
||||
let input = r#"[process_matrix(data=[[1, 2], [3, 4]], labels=["row[0]", "row[1]"])]"#;
|
||||
|
||||
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
assert_eq!(tools[0].function.name, "process_matrix");
|
||||
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert_eq!(args["data"], json!([[1, 2], [3, 4]]));
|
||||
assert_eq!(args["labels"], json!(["row[0]", "row[1]"]));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_pythonic_nested_brackets_in_dicts() {
|
||||
let parser = PythonicParser::new();
|
||||
|
||||
let input =
|
||||
r#"[analyze(config={"patterns": ["[a-z]+", "[0-9]+"], "nested": {"list": [1, [2, 3]]}})]"#;
|
||||
|
||||
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
assert_eq!(tools[0].function.name, "analyze");
|
||||
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert_eq!(args["config"]["patterns"], json!(["[a-z]+", "[0-9]+"]));
|
||||
assert_eq!(args["config"]["nested"]["list"], json!([1, [2, 3]]));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_pythonic_mixed_quotes() {
|
||||
let parser = PythonicParser::new();
|
||||
|
||||
let input = r#"[format_text(single='Hello', double="World", mixed="It's \"quoted\"")]"#;
|
||||
|
||||
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
assert_eq!(tools[0].function.name, "format_text");
|
||||
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert_eq!(args["single"], "Hello");
|
||||
assert_eq!(args["double"], "World");
|
||||
assert_eq!(args["mixed"], "It's \"quoted\"");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_pythonic_complex_nesting() {
|
||||
let parser = PythonicParser::new();
|
||||
|
||||
let input = r#"[transform(
|
||||
matrix=[[1, [2, 3]], [4, [5, [6, 7]]]],
|
||||
operations=[{"type": "scale", "factor": [2, 3]}, {"type": "rotate", "angle": 90}],
|
||||
metadata={"tags": ["nested[0]", "nested[1]"], "config": {"depth": [1, 2, 3]}}
|
||||
)]"#;
|
||||
|
||||
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
assert_eq!(tools[0].function.name, "transform");
|
||||
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert!(args["matrix"].is_array());
|
||||
assert!(args["operations"].is_array());
|
||||
assert_eq!(args["operations"][0]["type"], "scale");
|
||||
assert_eq!(args["metadata"]["config"]["depth"], json!([1, 2, 3]));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_parse_streaming_no_brackets() {
|
||||
let mut parser = PythonicParser::new();
|
||||
|
||||
let tools = create_test_tools();
|
||||
|
||||
let text = "This is just normal text without any tool calls.";
|
||||
let result = parser.parse_incremental(text, &tools).await.unwrap();
|
||||
|
||||
// Expected - no tool calls found
|
||||
assert!(result.calls.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_parse_streaming_complete_tool_call() {
|
||||
let mut parser = PythonicParser::new();
|
||||
|
||||
let tools = create_test_tools();
|
||||
|
||||
let text = "Here's a tool call: [get_weather(location='New York', unit='celsius')]";
|
||||
let result = parser.parse_incremental(text, &tools).await.unwrap();
|
||||
|
||||
assert!(!result.calls.is_empty(), "Should parse complete tool call");
|
||||
assert_eq!(result.calls[0].name.as_ref().unwrap(), "get_weather");
|
||||
let args: serde_json::Value = serde_json::from_str(&result.calls[0].parameters).unwrap();
|
||||
assert_eq!(args["location"], "New York");
|
||||
assert_eq!(args["unit"], "celsius");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_parse_streaming_text_before_tool_call() {
|
||||
let mut parser = PythonicParser::new();
|
||||
|
||||
let tools = create_test_tools();
|
||||
|
||||
let text = "This is some text before [get_weather(location='London')]";
|
||||
let result = parser.parse_incremental(text, &tools).await.unwrap();
|
||||
|
||||
assert!(!result.calls.is_empty(), "Should parse tool call");
|
||||
assert_eq!(result.calls[0].name.as_ref().unwrap(), "get_weather");
|
||||
let args: serde_json::Value = serde_json::from_str(&result.calls[0].parameters).unwrap();
|
||||
assert_eq!(args["location"], "London");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_parse_streaming_partial_tool_call() {
|
||||
let mut parser = PythonicParser::new();
|
||||
|
||||
let tools = create_test_tools();
|
||||
|
||||
// First chunk with opening bracket but no closing bracket
|
||||
let text1 = "Let me check the weather: [get_weather(location=";
|
||||
let result1 = parser.parse_incremental(text1, &tools).await.unwrap();
|
||||
|
||||
// First chunk should be incomplete
|
||||
assert!(
|
||||
result1.calls.is_empty(),
|
||||
"First chunk should not return tool call"
|
||||
);
|
||||
|
||||
// Second chunk completing the tool call
|
||||
let text2 = "'Paris')]";
|
||||
let result2 = parser.parse_incremental(text2, &tools).await.unwrap();
|
||||
|
||||
assert!(
|
||||
!result2.calls.is_empty(),
|
||||
"Second chunk should complete tool call"
|
||||
);
|
||||
assert_eq!(result2.calls[0].name.as_ref().unwrap(), "get_weather");
|
||||
let args: serde_json::Value = serde_json::from_str(&result2.calls[0].parameters).unwrap();
|
||||
assert_eq!(args["location"], "Paris");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_parse_streaming_bracket_without_text_before() {
|
||||
let mut parser = PythonicParser::new();
|
||||
|
||||
let tools = create_test_tools();
|
||||
|
||||
let text = "[search(query='python programming')]";
|
||||
let result = parser.parse_incremental(text, &tools).await.unwrap();
|
||||
|
||||
assert!(!result.calls.is_empty(), "Should parse tool call");
|
||||
assert_eq!(result.calls[0].name.as_ref().unwrap(), "search");
|
||||
let args: serde_json::Value = serde_json::from_str(&result.calls[0].parameters).unwrap();
|
||||
assert_eq!(args["query"], "python programming");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_parse_streaming_text_after_tool_call() {
|
||||
let mut parser = PythonicParser::new();
|
||||
|
||||
let tools = create_test_tools();
|
||||
|
||||
// First chunk with complete tool call and some text after
|
||||
let text = "[get_weather(location='Tokyo')] Here's the forecast:";
|
||||
let result = parser.parse_incremental(text, &tools).await.unwrap();
|
||||
|
||||
assert!(!result.calls.is_empty(), "Should parse tool call");
|
||||
assert_eq!(result.calls[0].name.as_ref().unwrap(), "get_weather");
|
||||
// Text after tool call is handled by parser internally
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_parse_streaming_multiple_tool_calls() {
|
||||
let mut parser = PythonicParser::new();
|
||||
|
||||
let tools = create_test_tools();
|
||||
|
||||
let text = "[get_weather(location='Berlin'), search(query='restaurants')]";
|
||||
|
||||
// Current implementation may handle this as a single parse
|
||||
let result = parser.parse_incremental(text, &tools).await.unwrap();
|
||||
|
||||
// The parser should handle multiple tools in one bracket pair
|
||||
// This test is flexible about the implementation behavior
|
||||
if !result.calls.is_empty() {
|
||||
// Parser found at least one tool
|
||||
assert!(result.calls[0].name.is_some());
|
||||
}
|
||||
// Also acceptable if parser returns empty waiting for more context
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_parse_streaming_opening_bracket_only() {
|
||||
let mut parser = PythonicParser::new();
|
||||
|
||||
let tools = create_test_tools();
|
||||
|
||||
let text = "Let's try this: [";
|
||||
let result = parser.parse_incremental(text, &tools).await.unwrap();
|
||||
|
||||
// Should be incomplete - no complete tool call
|
||||
assert!(
|
||||
result.calls.is_empty(),
|
||||
"Should not return tool call for partial bracket"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_parse_streaming_nested_brackets() {
|
||||
let mut parser = PythonicParser::new();
|
||||
|
||||
let tools = create_test_tools();
|
||||
|
||||
let text = "[get_weather(location='New York', unit='celsius', data=[1, 2, 3])]";
|
||||
let result = parser.parse_incremental(text, &tools).await.unwrap();
|
||||
|
||||
assert!(
|
||||
!result.calls.is_empty(),
|
||||
"Should parse tool call with nested brackets"
|
||||
);
|
||||
assert_eq!(result.calls[0].name.as_ref().unwrap(), "get_weather");
|
||||
let args: serde_json::Value = serde_json::from_str(&result.calls[0].parameters).unwrap();
|
||||
assert_eq!(args["location"], "New York");
|
||||
assert_eq!(args["unit"], "celsius");
|
||||
assert_eq!(args["data"], json!([1, 2, 3]));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_parse_streaming_nested_brackets_dict() {
|
||||
let mut parser = PythonicParser::new();
|
||||
let tools = create_test_tools();
|
||||
|
||||
let text = r#"[search(query='test', config={'options': [1, 2], 'nested': {'key': 'value'}})]"#;
|
||||
let result = parser.parse_incremental(text, &tools).await.unwrap();
|
||||
|
||||
assert!(
|
||||
!result.calls.is_empty(),
|
||||
"Should parse tool call with nested dict"
|
||||
);
|
||||
assert_eq!(result.calls[0].name.as_ref().unwrap(), "search");
|
||||
let args: serde_json::Value = serde_json::from_str(&result.calls[0].parameters).unwrap();
|
||||
assert_eq!(args["query"], "test");
|
||||
assert_eq!(args["config"]["options"], json!([1, 2]));
|
||||
assert_eq!(args["config"]["nested"]["key"], "value");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_parse_streaming_multiple_tools_with_nested_brackets() {
|
||||
let mut parser = PythonicParser::new();
|
||||
|
||||
let tools = create_test_tools();
|
||||
|
||||
let text =
|
||||
"[get_weather(location='Paris', data=[10, 20]), search(query='test', filters=['a', 'b'])]";
|
||||
let result = parser.parse_incremental(text, &tools).await.unwrap();
|
||||
|
||||
// Should parse tools successfully
|
||||
if !result.calls.is_empty() {
|
||||
// At least gets the first tool
|
||||
assert!(result.calls[0].name.is_some());
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_parse_streaming_partial_nested_brackets() {
|
||||
let mut parser = PythonicParser::new();
|
||||
|
||||
let tools = create_test_tools();
|
||||
|
||||
// First chunk with nested brackets but incomplete
|
||||
let text1 = "Here's a call: [get_weather(location='Tokyo', data=[1, 2";
|
||||
let result1 = parser.parse_incremental(text1, &tools).await.unwrap();
|
||||
|
||||
// First chunk should be incomplete
|
||||
assert!(result1.calls.is_empty(), "First chunk should not complete");
|
||||
|
||||
// Second chunk completing the nested brackets
|
||||
let text2 = ", 3])]";
|
||||
let result2 = parser.parse_incremental(text2, &tools).await.unwrap();
|
||||
|
||||
assert!(
|
||||
!result2.calls.is_empty(),
|
||||
"Second chunk should complete tool call"
|
||||
);
|
||||
assert_eq!(result2.calls[0].name.as_ref().unwrap(), "get_weather");
|
||||
let args: serde_json::Value = serde_json::from_str(&result2.calls[0].parameters).unwrap();
|
||||
assert_eq!(args["location"], "Tokyo");
|
||||
assert_eq!(args["data"], json!([1, 2, 3]));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_parse_streaming_with_python_start_and_end_token() {
|
||||
let mut parser = PythonicParser::new();
|
||||
|
||||
let tools = create_test_tools();
|
||||
|
||||
let chunks = vec![
|
||||
"Here's a call: ",
|
||||
"<|python_",
|
||||
"start|>[get_weather(location=",
|
||||
"'Tokyo', data=[1, 2",
|
||||
", 3])]<|python_end|>",
|
||||
];
|
||||
|
||||
let mut got_tool = false;
|
||||
|
||||
for chunk in chunks {
|
||||
let result = parser.parse_incremental(chunk, &tools).await.unwrap();
|
||||
if !result.calls.is_empty() {
|
||||
if let Some(name) = &result.calls[0].name {
|
||||
assert_eq!(name, "get_weather");
|
||||
let args: serde_json::Value =
|
||||
serde_json::from_str(&result.calls[0].parameters).unwrap();
|
||||
assert_eq!(args["location"], "Tokyo");
|
||||
assert_eq!(args["data"], json!([1, 2, 3]));
|
||||
got_tool = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
assert!(got_tool, "Should have parsed the tool call");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_detect_and_parse_with_python_start_and_end_token() {
|
||||
let parser = PythonicParser::new();
|
||||
|
||||
let text = "User wants to get the weather in Mars. <|python_start|>[get_weather(location='Mars', unit='celsius')]<|python_end|> In this way we will get the weather in Mars.";
|
||||
let (_normal_text, tools) = parser.parse_complete(text).await.unwrap();
|
||||
|
||||
assert_eq!(tools.len(), 1);
|
||||
assert_eq!(tools[0].function.name, "get_weather");
|
||||
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert_eq!(args["location"], "Mars");
|
||||
assert_eq!(args["unit"], "celsius");
|
||||
}
|
||||
@@ -0,0 +1,306 @@
|
||||
//! Qwen Parser Integration Tests
|
||||
//!
|
||||
//! Tests for the Qwen parser which handles <tool_call>...</tool_call> format
|
||||
|
||||
use serde_json::json;
|
||||
use smg::tool_parser::{QwenParser, ToolParser};
|
||||
|
||||
use crate::common::{create_test_tools, streaming_helpers::*};
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_qwen_single_tool() {
|
||||
let parser = QwenParser::new();
|
||||
let input = r#"<tool_call>
|
||||
{"name": "get_weather", "arguments": {"city": "Beijing", "units": "celsius"}}
|
||||
</tool_call>"#;
|
||||
|
||||
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
assert_eq!(tools[0].function.name, "get_weather");
|
||||
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert_eq!(args["city"], "Beijing");
|
||||
assert_eq!(args["units"], "celsius");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_qwen_multiple_sequential_tools() {
|
||||
let parser = QwenParser::new();
|
||||
let input = r#"Let me help you with that.
|
||||
<tool_call>
|
||||
{"name": "search", "arguments": {"query": "Qwen model"}}
|
||||
</tool_call>
|
||||
<tool_call>
|
||||
{"name": "translate", "arguments": {"text": "Hello", "to": "zh"}}
|
||||
</tool_call>"#;
|
||||
|
||||
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 2);
|
||||
assert_eq!(normal_text, "Let me help you with that.\n");
|
||||
assert_eq!(tools[0].function.name, "search");
|
||||
assert_eq!(tools[1].function.name, "translate");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_qwen_pretty_printed_json() {
|
||||
let parser = QwenParser::new();
|
||||
let input = r#"<tool_call>
|
||||
{
|
||||
"name": "create_document",
|
||||
"arguments": {
|
||||
"title": "Test Document",
|
||||
"content": "This is a test",
|
||||
"metadata": {
|
||||
"author": "Qwen",
|
||||
"tags": ["test", "example"]
|
||||
}
|
||||
}
|
||||
}
|
||||
</tool_call>"#;
|
||||
|
||||
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
assert_eq!(tools[0].function.name, "create_document");
|
||||
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert_eq!(args["metadata"]["author"], "Qwen");
|
||||
assert_eq!(args["metadata"]["tags"], json!(["test", "example"]));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_qwen_with_text_between() {
|
||||
let parser = QwenParser::new();
|
||||
let input = r#"First, let me search for information.
|
||||
<tool_call>
|
||||
{"name": "search", "arguments": {"query": "test"}}
|
||||
</tool_call>
|
||||
|
||||
Now I'll translate something.
|
||||
|
||||
<tool_call>
|
||||
{"name": "translate", "arguments": {"text": "world", "to": "es"}}
|
||||
</tool_call>
|
||||
Done!"#;
|
||||
|
||||
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 2);
|
||||
assert_eq!(normal_text, "First, let me search for information.\n");
|
||||
assert_eq!(tools[0].function.name, "search");
|
||||
assert_eq!(tools[1].function.name, "translate");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_qwen_empty_arguments() {
|
||||
let parser = QwenParser::new();
|
||||
let input = r#"<tool_call>
|
||||
{"name": "get_time", "arguments": {}}
|
||||
</tool_call>"#;
|
||||
|
||||
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
assert_eq!(tools[0].function.name, "get_time");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_qwen_with_newlines_in_strings() {
|
||||
let parser = QwenParser::new();
|
||||
let input = r#"<tool_call>
|
||||
{"name": "write_file", "arguments": {"content": "Line 1\nLine 2\nLine 3", "path": "/tmp/test.txt"}}
|
||||
</tool_call>"#;
|
||||
|
||||
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert_eq!(args["content"], "Line 1\nLine 2\nLine 3");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_qwen_format_detection() {
|
||||
let parser = QwenParser::new();
|
||||
|
||||
assert!(parser.has_tool_markers("<tool_call>"));
|
||||
assert!(parser.has_tool_markers("Some text <tool_call>\n{"));
|
||||
assert!(!parser.has_tool_markers("Just plain text"));
|
||||
assert!(!parser.has_tool_markers("{\"name\": \"test\"}")); // Plain JSON
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_qwen_incomplete_tags() {
|
||||
let parser = QwenParser::new();
|
||||
|
||||
// Missing closing tag
|
||||
let input = r#"<tool_call>
|
||||
{"name": "test", "arguments": {}}"#;
|
||||
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 0);
|
||||
|
||||
// Missing opening tag
|
||||
let input = r#"{"name": "test", "arguments": {}}
|
||||
</tool_call>"#;
|
||||
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_qwen_real_world_output() {
|
||||
let parser = QwenParser::new();
|
||||
|
||||
// Actual output from Qwen model
|
||||
let input = r#"I'll help you search for information and perform calculations.
|
||||
|
||||
<tool_call>
|
||||
{
|
||||
"name": "web_search",
|
||||
"arguments": {
|
||||
"query": "quantum computing breakthroughs 2024",
|
||||
"language": "en",
|
||||
"region": "us",
|
||||
"safe_search": true
|
||||
}
|
||||
}
|
||||
</tool_call>
|
||||
|
||||
Let me also calculate something for you:
|
||||
|
||||
<tool_call>
|
||||
{
|
||||
"name": "calculator",
|
||||
"arguments": {
|
||||
"expression": "sqrt(144) + 3^2",
|
||||
"precision": 2
|
||||
}
|
||||
}
|
||||
</tool_call>
|
||||
|
||||
These tools will provide the information you need."#;
|
||||
|
||||
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 2);
|
||||
assert_eq!(
|
||||
normal_text,
|
||||
"I'll help you search for information and perform calculations.\n\n"
|
||||
);
|
||||
assert_eq!(tools[0].function.name, "web_search");
|
||||
assert_eq!(tools[1].function.name, "calculator");
|
||||
|
||||
let args0: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert_eq!(args0["query"], "quantum computing breakthroughs 2024");
|
||||
assert_eq!(args0["safe_search"], true);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_buffer_drain_optimization() {
|
||||
let mut parser = QwenParser::new();
|
||||
|
||||
let tools = create_test_tools();
|
||||
|
||||
// First chunk - incomplete tool call
|
||||
let chunk1 = "<tool_call>\n{\"name\": \"test1\", ";
|
||||
let _result = parser.parse_incremental(chunk1, &tools).await.unwrap();
|
||||
// The important thing is buffer accumulation works
|
||||
|
||||
// Complete first tool and start second
|
||||
let chunk2 = "\"arguments\": {}}\n</tool_call><tool_call>\n{\"name\": \"test2\", ";
|
||||
let result = parser.parse_incremental(chunk2, &tools).await.unwrap();
|
||||
|
||||
if !result.calls.is_empty() {
|
||||
if let Some(_name) = &result.calls[0].name {
|
||||
assert_eq!(result.calls[0].name.as_ref().unwrap(), "test1");
|
||||
// After consuming the first tool, buffer is managed internally
|
||||
}
|
||||
}
|
||||
|
||||
// Complete the second tool
|
||||
let chunk3 = "\"arguments\": {\"x\": 1}}\n</tool_call>";
|
||||
let result = parser.parse_incremental(chunk3, &tools).await.unwrap();
|
||||
|
||||
if !result.calls.is_empty() {
|
||||
if let Some(_name) = &result.calls[0].name {
|
||||
assert_eq!(result.calls[0].name.as_ref().unwrap(), "test2");
|
||||
// Buffer is managed internally
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_buffer_efficiency_with_multiple_tools() {
|
||||
let mut parser = QwenParser::new();
|
||||
|
||||
let tools = create_test_tools();
|
||||
|
||||
// Send multiple complete tools at once
|
||||
let input = r#"<tool_call>
|
||||
{"name": "tool1", "arguments": {"a": 1}}
|
||||
</tool_call><tool_call>
|
||||
{"name": "tool2", "arguments": {"b": 2}}
|
||||
</tool_call><tool_call>
|
||||
{"name": "tool3", "arguments": {"c": 3}}
|
||||
</tool_call>"#;
|
||||
|
||||
// This should efficiently process tools using drain() without creating new strings
|
||||
let result = parser.parse_incremental(input, &tools).await.unwrap();
|
||||
|
||||
// In Phase 2, this will likely parse only the first tool
|
||||
// The important thing is that drain() doesn't cause any issues
|
||||
if !result.calls.is_empty() {
|
||||
if let Some(name) = &result.calls[0].name {
|
||||
assert!(["tool1", "tool2", "tool3"].contains(&name.as_str()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// REALISTIC STREAMING TESTS
|
||||
// =============================================================================
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_qwen_realistic_chunks_with_xml_tags() {
|
||||
let tools = create_test_tools();
|
||||
let mut parser = QwenParser::new();
|
||||
|
||||
let input = "<tool_call>\n{\"name\": \"get_weather\", \"arguments\": {\"city\": \"Tokyo\"}}\n</tool_call>";
|
||||
let chunks = create_realistic_chunks(input);
|
||||
|
||||
assert!(chunks.len() > 20, "Should have many small chunks");
|
||||
|
||||
let mut got_tool_name = false;
|
||||
|
||||
for chunk in chunks {
|
||||
let result = parser.parse_incremental(&chunk, &tools).await.unwrap();
|
||||
for call in result.calls {
|
||||
if let Some(name) = call.name {
|
||||
assert_eq!(name, "get_weather");
|
||||
got_tool_name = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
assert!(got_tool_name, "Should have parsed tool name");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_qwen_xml_tag_arrives_in_parts() {
|
||||
let tools = create_test_tools();
|
||||
let mut parser = QwenParser::new();
|
||||
|
||||
let chunks = vec![
|
||||
"<to", "ol_", "cal", "l>\n", "{", r#"""#, "na", "me", r#"""#, ": ", r#"""#, "tra", "nsl",
|
||||
"ate", r#"""#, ", ", r#"""#, "arg", "ume", "nts", r#"""#, ": {", r#"""#, "tex", "t",
|
||||
r#"""#, ": ", r#"""#, "hel", "lo", r#"""#, "}}\n", "</t", "ool", "_ca", "ll>",
|
||||
];
|
||||
|
||||
let mut got_tool_name = false;
|
||||
|
||||
for chunk in chunks {
|
||||
let result = parser.parse_incremental(chunk, &tools).await.unwrap();
|
||||
for call in result.calls {
|
||||
if let Some(name) = call.name {
|
||||
assert_eq!(name, "translate");
|
||||
got_tool_name = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
assert!(got_tool_name, "Should have parsed tool name");
|
||||
}
|
||||
@@ -0,0 +1,237 @@
|
||||
//! Step3 Parser Integration Tests
|
||||
|
||||
use smg::tool_parser::{Step3Parser, ToolParser};
|
||||
|
||||
use crate::common::create_test_tools;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_step3_complete_parsing() {
|
||||
let parser = Step3Parser::new();
|
||||
|
||||
let input = r#"Let me help you.
|
||||
<｜tool_calls_begin｜>
|
||||
<｜tool_call_begin｜>function<｜tool_sep｜><steptml:invoke name="search">
|
||||
<steptml:parameter name="query">rust programming</steptml:parameter>
|
||||
<steptml:parameter name="limit">10</steptml:parameter>
|
||||
</steptml:invoke><｜tool_call_end｜>
|
||||
<｜tool_calls_end｜>
|
||||
Here are the results..."#;
|
||||
|
||||
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
assert_eq!(normal_text, "Let me help you.\n");
|
||||
assert_eq!(tools[0].function.name, "search");
|
||||
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert_eq!(args["query"], "rust programming");
|
||||
assert_eq!(args["limit"], 10);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_step3_multiple_tools() {
|
||||
let parser = Step3Parser::new();
|
||||
|
||||
let input = r#"<｜tool_calls_begin｜>
|
||||
<｜tool_call_begin｜>function<｜tool_sep｜><steptml:invoke name="get_weather">
|
||||
<steptml:parameter name="location">Tokyo</steptml:parameter>
|
||||
</steptml:invoke><｜tool_call_end｜>
|
||||
<｜tool_call_begin｜>function<｜tool_sep｜><steptml:invoke name="get_news">
|
||||
<steptml:parameter name="category">tech</steptml:parameter>
|
||||
<steptml:parameter name="limit">5</steptml:parameter>
|
||||
</steptml:invoke><｜tool_call_end｜>
|
||||
<｜tool_calls_end｜>"#;
|
||||
|
||||
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 2);
|
||||
assert_eq!(tools[0].function.name, "get_weather");
|
||||
assert_eq!(tools[1].function.name, "get_news");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_step3_type_conversion() {
|
||||
let parser = Step3Parser::new();
|
||||
|
||||
let input = r#"<｜tool_calls_begin｜>
|
||||
<｜tool_call_begin｜>function<｜tool_sep｜><steptml:invoke name="process">
|
||||
<steptml:parameter name="count">100</steptml:parameter>
|
||||
<steptml:parameter name="rate">2.5</steptml:parameter>
|
||||
<steptml:parameter name="active">true</steptml:parameter>
|
||||
<steptml:parameter name="optional">null</steptml:parameter>
|
||||
<steptml:parameter name="text">hello world</steptml:parameter>
|
||||
</steptml:invoke><｜tool_call_end｜>
|
||||
<｜tool_calls_end｜>"#;
|
||||
|
||||
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert_eq!(args["count"], 100);
|
||||
assert_eq!(args["rate"], 2.5);
|
||||
assert_eq!(args["active"], true);
|
||||
assert_eq!(args["optional"], serde_json::Value::Null);
|
||||
assert_eq!(args["text"], "hello world");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_step3_streaming() {
|
||||
let mut parser = Step3Parser::new();
|
||||
|
||||
let tools = create_test_tools();
|
||||
|
||||
// Simulate streaming chunks
|
||||
let chunks = vec![
|
||||
"<｜tool_calls_begin｜>\n",
|
||||
"<｜tool_call_begin｜>function",
|
||||
"<｜tool_sep｜><steptml:invoke name=\"calc\">",
|
||||
"\n<steptml:parameter name=\"x\">10</steptml:parameter>",
|
||||
"\n<steptml:parameter name=\"y\">20</steptml:parameter>",
|
||||
"\n</steptml:invoke><｜tool_call_end｜>",
|
||||
"\n<｜tool_calls_end｜>",
|
||||
];
|
||||
|
||||
let mut found_complete = false;
|
||||
|
||||
for chunk in chunks {
|
||||
let result = parser.parse_incremental(chunk, &tools).await.unwrap();
|
||||
|
||||
if !result.calls.is_empty() {
|
||||
if let Some(name) = &result.calls[0].name {
|
||||
assert_eq!(name, "calc");
|
||||
found_complete = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
assert!(found_complete);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_step3_format_detection() {
|
||||
let parser = Step3Parser::new();
|
||||
|
||||
// Should detect Step3 format
|
||||
assert!(parser.has_tool_markers("<｜tool_calls_begin｜>"));
|
||||
assert!(parser.has_tool_markers("text with <｜tool_calls_begin｜> marker"));
|
||||
|
||||
// Should not detect other formats
|
||||
assert!(!parser.has_tool_markers("[TOOL_CALLS]"));
|
||||
assert!(!parser.has_tool_markers("<tool_call>"));
|
||||
assert!(!parser.has_tool_markers("plain text"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_step3_nested_steptml() {
|
||||
let parser = Step3Parser::new();
|
||||
|
||||
let input = r#"<｜tool_calls_begin｜>
|
||||
<｜tool_call_begin｜>function<｜tool_sep｜><steptml:invoke name="config">
|
||||
<steptml:parameter name="settings">{"nested": {"key": "value"}}</steptml:parameter>
|
||||
<steptml:parameter name="array">[1, 2, 3]</steptml:parameter>
|
||||
</steptml:invoke><｜tool_call_end｜>
|
||||
<｜tool_calls_end｜>"#;
|
||||
|
||||
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
assert_eq!(tools[0].function.name, "config");
|
||||
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert!(args["settings"].is_object());
|
||||
assert!(args["array"].is_array());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_step3_python_literals() {
|
||||
let parser = Step3Parser::new();
|
||||
|
||||
let input = r#"<｜tool_calls_begin｜>
|
||||
<｜tool_call_begin｜>function<｜tool_sep｜><steptml:invoke name="test">
|
||||
<steptml:parameter name="bool_true">True</steptml:parameter>
|
||||
<steptml:parameter name="bool_false">False</steptml:parameter>
|
||||
<steptml:parameter name="none_value">None</steptml:parameter>
|
||||
</steptml:invoke><｜tool_call_end｜>
|
||||
<｜tool_calls_end｜>"#;
|
||||
|
||||
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert_eq!(args["bool_true"], true);
|
||||
assert_eq!(args["bool_false"], false);
|
||||
assert_eq!(args["none_value"], serde_json::Value::Null);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_steptml_format() {
|
||||
let parser = Step3Parser::new();
|
||||
|
||||
let input = r#"Text before.
|
||||
<｜tool_calls_begin｜>
|
||||
<｜tool_call_begin｜>function<｜tool_sep｜><steptml:invoke name="search">
|
||||
<steptml:parameter name="query">rust lang</steptml:parameter>
|
||||
<steptml:parameter name="limit">10</steptml:parameter>
|
||||
</steptml:invoke><｜tool_call_end｜>
|
||||
<｜tool_calls_end｜>Text after."#;
|
||||
|
||||
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
assert_eq!(normal_text, "Text before.\n");
|
||||
assert_eq!(tools[0].function.name, "search");
|
||||
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert_eq!(args["query"], "rust lang");
|
||||
assert_eq!(args["limit"], 10);
|
||||
// TODO: Verify normal text extraction
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_json_parameter_values() {
|
||||
let parser = Step3Parser::new();
|
||||
|
||||
let input = r#"<｜tool_calls_begin｜>
|
||||
<｜tool_call_begin｜>function<｜tool_sep｜><steptml:invoke name="config">
|
||||
<steptml:parameter name="settings">{"nested": {"value": true}}</steptml:parameter>
|
||||
<steptml:parameter name="items">[1, 2, 3]</steptml:parameter>
|
||||
</steptml:invoke><｜tool_call_end｜>
|
||||
<｜tool_calls_end｜>"#;
|
||||
|
||||
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert!(args["settings"].is_object());
|
||||
assert!(args["items"].is_array());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_step3_parameter_with_angle_brackets() {
|
||||
let parser = Step3Parser::new();
|
||||
|
||||
let input = r#"<｜tool_calls_begin｜>
|
||||
<｜tool_call_begin｜>function<｜tool_sep｜><steptml:invoke name="compare">
|
||||
<steptml:parameter name="expression">a < b && b > c</steptml:parameter>
|
||||
<steptml:parameter name="context">comparison test</steptml:parameter>
|
||||
</steptml:invoke><｜tool_call_end｜>
|
||||
<｜tool_calls_end｜>"#;
|
||||
|
||||
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 1);
|
||||
assert_eq!(tools[0].function.name, "compare");
|
||||
|
||||
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
|
||||
assert_eq!(args["expression"], "a < b && b > c");
|
||||
assert_eq!(args["context"], "comparison test");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_step3_empty_function_name() {
|
||||
let parser = Step3Parser::new();
|
||||
|
||||
let input = r#"<｜tool_calls_begin｜>
|
||||
<｜tool_call_begin｜>function<｜tool_sep｜><steptml:invoke name="">
|
||||
<steptml:parameter name="param">value</steptml:parameter>
|
||||
</steptml:invoke><｜tool_call_end｜>
|
||||
<｜tool_calls_end｜>"#;
|
||||
|
||||
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
|
||||
assert_eq!(tools.len(), 0); // Should reject empty function name
|
||||
}
|
||||
Reference in New Issue
Block a user