diff --git a/sgl-router/src/tool_parser/factory.rs b/sgl-router/src/tool_parser/factory.rs index 3f3a6fba4..c22b2f8c6 100644 --- a/sgl-router/src/tool_parser/factory.rs +++ b/sgl-router/src/tool_parser/factory.rs @@ -9,8 +9,8 @@ use tokio::sync::Mutex; use crate::tool_parser::{ parsers::{ - DeepSeekParser, Glm4MoeParser, JsonParser, KimiK2Parser, LlamaParser, MistralParser, - PassthroughParser, PythonicParser, QwenParser, Step3Parser, + DeepSeekParser, Glm4MoeParser, JsonParser, KimiK2Parser, LlamaParser, MinimaxM2Parser, + MistralParser, PassthroughParser, PythonicParser, QwenParser, Step3Parser, }, traits::ToolParser, }; @@ -242,6 +242,7 @@ impl ParserFactory { registry.register_parser("glm4_moe", || Box::new(Glm4MoeParser::new())); registry.register_parser("step3", || Box::new(Step3Parser::new())); registry.register_parser("kimik2", || Box::new(KimiK2Parser::new())); + registry.register_parser("minimax_m2", || Box::new(MinimaxM2Parser::new())); // Register default model mappings Self::register_default_mappings(®istry); @@ -293,6 +294,10 @@ impl ParserFactory { registry.map_model("Kimi-K2*", "kimik2"); registry.map_model("moonshot*/Kimi-K2*", "kimik2"); + // MiniMax models + registry.map_model("minimax*", "minimax_m2"); + registry.map_model("MiniMax*", "minimax_m2"); + // Other models registry.map_model("gemini-*", "json"); registry.map_model("palm-*", "json"); diff --git a/sgl-router/src/tool_parser/mod.rs b/sgl-router/src/tool_parser/mod.rs index da7a703a4..7cfd1ffc4 100644 --- a/sgl-router/src/tool_parser/mod.rs +++ b/sgl-router/src/tool_parser/mod.rs @@ -20,8 +20,8 @@ pub use errors::{ParserError, ParserResult}; pub use factory::{ParserFactory, ParserRegistry, PooledParser}; // Re-export parsers for convenience pub use parsers::{ - DeepSeekParser, Glm4MoeParser, JsonParser, KimiK2Parser, LlamaParser, MistralParser, - PythonicParser, QwenParser, Step3Parser, + DeepSeekParser, Glm4MoeParser, JsonParser, KimiK2Parser, LlamaParser, MinimaxM2Parser, + MistralParser, PythonicParser, QwenParser, Step3Parser, }; pub use traits::{PartialJsonParser, ToolParser}; pub use types::{FunctionCall, PartialToolCall, StreamingParseResult, ToolCall}; diff --git a/sgl-router/src/tool_parser/parsers/minimax_m2.rs b/sgl-router/src/tool_parser/parsers/minimax_m2.rs new file mode 100644 index 000000000..eff92ba67 --- /dev/null +++ b/sgl-router/src/tool_parser/parsers/minimax_m2.rs @@ -0,0 +1,563 @@ +use std::{collections::HashMap, fmt::Write as FmtWrite}; + +use async_trait::async_trait; +use regex::Regex; +use serde_json::Value; + +use crate::{ + protocols::common::Tool, + tool_parser::{ + errors::{ParserError, ParserResult}, + parsers::helpers, + traits::ToolParser, + types::{FunctionCall, StreamingParseResult, ToolCall, ToolCallItem}, + }, +}; + +/// MiniMax M2 format parser for tool calls +/// +/// Implements the MiniMax-M2 model's XML-based tool calling format as specified in the +/// official chat template. The M2 model uses a structured XML format for function invocations +/// to ensure reliable parsing and execution. +/// +/// ## Format Reference +/// - HuggingFace Model: https://huggingface.co/MiniMaxAI/MiniMax-M2 +/// - Chat Template: https://huggingface.co/MiniMaxAI/MiniMax-M2?chat_template=default +/// +/// ## Tool Call Structure +/// ```xml +/// +/// +/// value1 +/// value2 +/// +/// +/// ``` +/// +/// ## Key Features +/// - **Namespaced XML tags**: Uses `minimax:` namespace to avoid conflicts +/// - **Structured invocation**: Functions wrapped in `` tags +/// - **Named parameters**: Each parameter uses `value` +/// - **Incremental streaming**: Converts XML to JSON progressively during streaming +/// - **XML entity decoding**: Handles encoded entities (`<`, `>`, etc.) in parameter values +pub struct MinimaxM2Parser { + // Regex patterns + tool_call_extractor: Regex, + invoke_extractor: Regex, + param_extractor: Regex, + + // Streaming state + buffer: String, + prev_tool_call_arr: Vec, + current_tool_id: i32, + streamed_args_for_tool: Vec, + current_function_name: String, + current_parameters: HashMap, + in_tool_call: bool, + function_name_sent: bool, + waiting_for_tool_call_end: bool, + + // Token configuration + tool_call_start_token: &'static str, + tool_call_end_token: &'static str, + invoke_end_token: &'static str, +} + +impl MinimaxM2Parser { + /// Parse a value from string with consistent logic + #[inline] + fn parse_value(text: &str) -> Value { + // Try parsing as common literals first + match text { + "true" | "True" => return Value::Bool(true), + "false" | "False" => return Value::Bool(false), + "null" | "None" => return Value::Null, + _ => {} + } + + // Try parsing as number + if let Ok(num) = text.parse::() { + return Value::Number(num.into()); + } + + if let Ok(num) = text.parse::() { + if let Some(n) = serde_json::Number::from_f64(num) { + return Value::Number(n); + } + } + + // Default to string + Value::String(text.to_string()) + } + + /// Create a new MiniMax M2 parser + pub fn new() -> Self { + // Use (?s) flag for DOTALL mode to handle newlines + let tool_call_pattern = r"(?s).*?"; + let tool_call_extractor = Regex::new(tool_call_pattern).expect("Valid regex pattern"); + + let invoke_pattern = r#"(?s)(.*?)"#; + let invoke_extractor = Regex::new(invoke_pattern).expect("Valid regex pattern"); + + let param_pattern = r#"(?s)(.*?)"#; + let param_extractor = Regex::new(param_pattern).expect("Valid regex pattern"); + + Self { + tool_call_extractor, + invoke_extractor, + param_extractor, + buffer: String::new(), + prev_tool_call_arr: Vec::new(), + current_tool_id: -1, + streamed_args_for_tool: Vec::new(), + current_function_name: String::new(), + current_parameters: HashMap::new(), + in_tool_call: false, + function_name_sent: false, + waiting_for_tool_call_end: false, + tool_call_start_token: "", + tool_call_end_token: "", + invoke_end_token: "", + } + } + + /// Parse parameters from parameter tags + fn parse_parameters(&self, params_text: &str) -> ParserResult> { + let mut parameters = serde_json::Map::new(); + + for capture in self.param_extractor.captures_iter(params_text) { + let key = capture.get(1).map_or("", |m| m.as_str()).trim(); + let value_str = capture.get(2).map_or("", |m| m.as_str()); + + // Decode XML entities and parse value + let decoded_value = self.decode_xml_entities(value_str); + + // Note: We keep JSON-like strings as strings (not parsed JSON) + // This matches the behavior of other parsers like GLM4 MOE + let value = Self::parse_value(&decoded_value); + + parameters.insert(key.to_string(), value); + } + + Ok(parameters) + } + + /// Decode common XML entities + fn decode_xml_entities(&self, text: &str) -> String { + text.replace("<", "<") + .replace(">", ">") + .replace("&", "&") + .replace(""", "\"") + .replace("'", "'") + } + + /// Parse a single tool call block + fn parse_tool_call(&self, block: &str) -> ParserResult> { + if let Some(captures) = self.invoke_extractor.captures(block) { + // Get function name from invoke tag attribute + let func_name = captures.get(1).map_or("", |m| m.as_str()).trim(); + + // Get parameters text + let params_text = captures.get(2).map_or("", |m| m.as_str()); + + // Parse parameters + let parameters = self.parse_parameters(params_text)?; + + let arguments_str = serde_json::to_string(¶meters) + .map_err(|e| ParserError::ParsingFailed(e.to_string()))?; + + Ok(Some(ToolCall { + function: FunctionCall { + name: func_name.to_string(), + arguments: arguments_str, + }, + })) + } else { + Ok(None) + } + } + + /// Parse all tool calls from text and return first valid position + fn parse_tool_calls_from_text( + &self, + text: &str, + ) -> ParserResult<(Vec, Option)> { + let mut tools = Vec::new(); + let mut first_valid_pos = None; + + for mat in self.tool_call_extractor.find_iter(text) { + match self.parse_tool_call(mat.as_str()) { + Ok(Some(tool)) => { + if first_valid_pos.is_none() { + first_valid_pos = Some(mat.start()); + } + tools.push(tool); + } + Ok(None) => continue, + Err(e) => { + tracing::warn!("Failed to parse tool call: {}", e); + continue; + } + } + } + + Ok((tools, first_valid_pos)) + } + + /// Parse and stream parameters incrementally + fn parse_and_stream_parameters(&mut self, text: &str, _tools: &[Tool]) -> Vec { + let mut calls = Vec::new(); + + // Find all complete parameter patterns in the buffer + let param_matches: Vec<_> = self + .param_extractor + .captures_iter(text) + .map(|cap| { + let name = cap.get(1).map_or("", |m| m.as_str()).trim().to_string(); + let value_str = cap.get(2).map_or("", |m| m.as_str()); + let decoded = self.decode_xml_entities(value_str); + + // Try parsing as JSON first (for nested objects/arrays) + let value = if decoded.starts_with('{') || decoded.starts_with('[') { + if let Ok(json_val) = serde_json::from_str::(&decoded) { + json_val + } else { + Self::parse_value(&decoded) + } + } else { + Self::parse_value(&decoded) + }; + + (name, value) + }) + .collect(); + + // Build new parameters map + let mut new_params = HashMap::new(); + for (name, value) in param_matches { + new_params.insert(name, value); + } + + // If we have new parameters that weren't in current_parameters, stream them + if !new_params.is_empty() && new_params != self.current_parameters { + let tool_id = self.current_tool_id as usize; + + // Ensure we have enough capacity + while self.streamed_args_for_tool.len() <= tool_id { + self.streamed_args_for_tool.push(String::new()); + } + + // Build incremental JSON with single allocation + if self.current_parameters.is_empty() { + // First parameters - start JSON object but don't close it + let mut json_fragment = String::with_capacity(256); + json_fragment.push('{'); + + let mut first = true; + for (key, value) in &new_params { + if !first { + json_fragment.push_str(", "); + } + write!( + &mut json_fragment, + "{}: {}", + serde_json::to_string(key).unwrap(), + serde_json::to_string(value).unwrap() + ) + .unwrap(); + first = false; + } + + calls.push(ToolCallItem { + tool_index: tool_id, + name: None, + parameters: json_fragment.clone(), + }); + + self.streamed_args_for_tool[tool_id] = json_fragment; + } else { + // Additional parameters - add them incrementally + let new_keys: Vec<_> = new_params + .keys() + .filter(|k| !self.current_parameters.contains_key(*k)) + .collect(); + + if !new_keys.is_empty() { + let mut json_fragment = String::with_capacity(128); + + for key in new_keys { + let value = &new_params[key]; + write!( + &mut json_fragment, + ", {}: {}", + serde_json::to_string(key).unwrap(), + serde_json::to_string(value).unwrap() + ) + .unwrap(); + } + + calls.push(ToolCallItem { + tool_index: tool_id, + name: None, + parameters: json_fragment.clone(), + }); + + self.streamed_args_for_tool[tool_id].push_str(&json_fragment); + } + } + + // Update current parameters + self.current_parameters = new_params; + + // Update prev_tool_call_arr + while self.prev_tool_call_arr.len() <= tool_id { + self.prev_tool_call_arr.push(Value::Null); + } + self.prev_tool_call_arr[tool_id] = serde_json::json!({ + "name": self.current_function_name, + "arguments": self.current_parameters, + }); + } + + calls + } +} + +impl Default for MinimaxM2Parser { + fn default() -> Self { + Self::new() + } +} + +#[async_trait] +impl ToolParser for MinimaxM2Parser { + async fn parse_complete(&self, text: &str) -> ParserResult<(String, Vec)> { + // Check if text contains MiniMax M2 format + if !self.has_tool_markers(text) { + return Ok((text.to_string(), vec![])); + } + + // Parse all tool calls and get first valid position + let (tools, first_valid_tool_pos) = self.parse_tool_calls_from_text(text)?; + + // If no tools were successfully parsed, return entire text as fallback + if tools.is_empty() { + return Ok((text.to_string(), vec![])); + } + + // Determine what text to return as normal_text + let normal_text = if let Some(pos) = first_valid_tool_pos { + // Return text up to the first valid tool call + text[..pos].to_string() + } else { + // No valid tool calls found, return entire text + text.to_string() + }; + + Ok((normal_text, tools)) + } + + async fn parse_incremental( + &mut self, + chunk: &str, + tools: &[Tool], + ) -> ParserResult { + self.buffer.push_str(chunk); + let mut normal_text = String::new(); + let mut calls = Vec::new(); + + // Build tool indices for validation + let tool_indices = helpers::get_tool_indices(tools); + + loop { + // If we're waiting for the tool call end tag, check for it first + if self.waiting_for_tool_call_end { + if let Some(end_pos) = self.buffer.find(self.tool_call_end_token) { + // Complete tool call found + self.buffer = + self.buffer[end_pos + self.tool_call_end_token.len()..].to_string(); + self.in_tool_call = false; + self.waiting_for_tool_call_end = false; + self.function_name_sent = false; + self.current_function_name.clear(); + self.current_parameters.clear(); + self.current_tool_id += 1; + continue; + } else { + // End tag not complete yet, wait for more text + break; + } + } + + // If we're not in a tool call and don't see a start token, return normal text + if !self.in_tool_call && !self.buffer.contains(self.tool_call_start_token) { + // Check if buffer might contain a partial start token at the end + if let Some(partial_len) = + helpers::ends_with_partial_token(&self.buffer, self.tool_call_start_token) + { + // Return everything except the potential partial token + let end = self.buffer.len() - partial_len; + normal_text = self.buffer[..end].to_string(); + self.buffer = self.buffer[end..].to_string(); + } else { + // No partial token, return all as normal text + normal_text = self.buffer.clone(); + self.buffer.clear(); + } + break; + } + + // Look for tool call start + if !self.in_tool_call { + if let Some(start) = self.buffer.find(self.tool_call_start_token) { + normal_text = self.buffer[..start].to_string(); + self.buffer = + self.buffer[start + self.tool_call_start_token.len()..].to_string(); + + self.in_tool_call = true; + self.function_name_sent = false; + self.current_function_name.clear(); + self.current_parameters.clear(); + + continue; + } else { + // No start token found + break; + } + } + + // We're in a tool call, try to parse function name if not sent yet + if !self.function_name_sent { + // Use regex to extract function name from pattern + // Check if we have enough text to match the invoke pattern + if let Some(captures) = self.invoke_extractor.captures(&self.buffer) { + let function_name = captures + .get(1) + .map_or("", |m| m.as_str()) + .trim() + .to_string(); + + // Validate function name + if tool_indices.contains_key(&function_name) { + self.current_function_name = function_name.clone(); + self.function_name_sent = true; + + // Initialize tool call tracking + if self.current_tool_id == -1 { + self.current_tool_id = 0; + } + + // Ensure tracking arrays are large enough + helpers::ensure_capacity( + self.current_tool_id, + &mut self.prev_tool_call_arr, + &mut self.streamed_args_for_tool, + ); + + // Send tool name with empty parameters + calls.push(ToolCallItem { + tool_index: self.current_tool_id as usize, + name: Some(function_name), + parameters: String::new(), + }); + + // Find the position after the opening invoke tag (after the >) + // We only want to remove up to the opening tag, not the full match + if let Some(pos) = self.buffer.find('>') { + self.buffer = self.buffer[pos + 1..].to_string(); + } + continue; + } else { + // Invalid function name, reset state + tracing::warn!("Invalid function name: {}", function_name); + self.in_tool_call = false; + normal_text.push_str(&self.buffer); + self.buffer.clear(); + break; + } + } + // No complete invoke pattern found yet, wait for more text + break; + } + + // Parse parameters incrementally + if self.function_name_sent { + // Process parameters and get any calls to emit + // Note: We need to be careful here - parse_and_stream_parameters needs + // to work with the buffer but we can't pass &self.buffer directly + // due to borrow checker. Instead, we'll refactor slightly. + // For now, keep the clone but mark it as a TODO for future optimization + let buffer_copy = self.buffer.clone(); // TODO: Optimize this + let parameter_calls = self.parse_and_stream_parameters(&buffer_copy, tools); + calls.extend(parameter_calls); + + // Check if tool call is complete ( found) + if let Some(invoke_end) = self.buffer.find(self.invoke_end_token) { + // Add closing brace to complete the JSON object + let tool_id = self.current_tool_id as usize; + if tool_id < self.streamed_args_for_tool.len() { + let current_streamed = &self.streamed_args_for_tool[tool_id]; + if !current_streamed.is_empty() && !current_streamed.ends_with('}') { + // Count opening and closing braces to check if JSON is complete + let open_braces = current_streamed.matches('{').count(); + let close_braces = current_streamed.matches('}').count(); + if open_braces > close_braces { + calls.push(ToolCallItem { + tool_index: tool_id, + name: None, + parameters: "}".to_string(), + }); + self.streamed_args_for_tool[tool_id].push('}'); + } + } + } + + // Move buffer past the + self.buffer = + self.buffer[invoke_end + self.invoke_end_token.len()..].to_string(); + + // Check if we have the closing + if let Some(end_pos) = self.buffer.find(self.tool_call_end_token) { + // Complete tool call found + self.buffer = + self.buffer[end_pos + self.tool_call_end_token.len()..].to_string(); + self.in_tool_call = false; + self.function_name_sent = false; + self.current_function_name.clear(); + self.current_parameters.clear(); + self.current_tool_id += 1; + continue; + } else { + // End tag not complete yet, mark that we're waiting for it + self.waiting_for_tool_call_end = true; + break; + } + } + // Tool call not complete yet, wait for more text + break; + } + } + + Ok(StreamingParseResult { normal_text, calls }) + } + + fn has_tool_markers(&self, text: &str) -> bool { + text.contains(self.tool_call_start_token) + } + + fn get_unstreamed_tool_args(&self) -> Option> { + helpers::get_unstreamed_args(&self.prev_tool_call_arr, &self.streamed_args_for_tool) + } + + fn reset(&mut self) { + self.buffer.clear(); + self.prev_tool_call_arr.clear(); + self.current_tool_id = -1; + self.streamed_args_for_tool.clear(); + self.current_function_name.clear(); + self.current_parameters.clear(); + self.in_tool_call = false; + self.function_name_sent = false; + self.waiting_for_tool_call_end = false; + } +} diff --git a/sgl-router/src/tool_parser/parsers/mod.rs b/sgl-router/src/tool_parser/parsers/mod.rs index befde0172..5b28d0b4b 100644 --- a/sgl-router/src/tool_parser/parsers/mod.rs +++ b/sgl-router/src/tool_parser/parsers/mod.rs @@ -8,6 +8,7 @@ pub mod glm4_moe; pub mod json; pub mod kimik2; pub mod llama; +pub mod minimax_m2; pub mod mistral; pub mod passthrough; pub mod pythonic; @@ -23,6 +24,7 @@ pub use glm4_moe::Glm4MoeParser; pub use json::JsonParser; pub use kimik2::KimiK2Parser; pub use llama::LlamaParser; +pub use minimax_m2::MinimaxM2Parser; pub use mistral::MistralParser; pub use passthrough::PassthroughParser; pub use pythonic::PythonicParser; diff --git a/sgl-router/tests/tool_parser_minimax_m2.rs b/sgl-router/tests/tool_parser_minimax_m2.rs new file mode 100644 index 000000000..a6b296ec5 --- /dev/null +++ b/sgl-router/tests/tool_parser_minimax_m2.rs @@ -0,0 +1,780 @@ +//! MiniMax M2 Parser Integration Tests + +use sglang_router_rs::tool_parser::{MinimaxM2Parser, ToolParser}; + +mod common; +use common::create_test_tools; + +#[tokio::test] +async fn test_minimax_complete_parsing() { + let parser = MinimaxM2Parser::new(); + + let input = r#"Let me search for that. + + +Beijing +2024-12-25 + + +The weather will be..."#; + + let (normal_text, tools) = parser.parse_complete(input).await.unwrap(); + assert_eq!(tools.len(), 1); + assert_eq!(normal_text, "Let me search for that.\n"); + assert_eq!(tools[0].function.name, "get_weather"); + + let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap(); + assert_eq!(args["city"], "Beijing"); + assert_eq!(args["date"], "2024-12-25"); +} + +#[tokio::test] +async fn test_minimax_multiple_tools() { + let parser = MinimaxM2Parser::new(); + + let input = r#" + +rust tutorials + + + + +Hello World +zh + +"#; + + let (normal_text, tools) = parser.parse_complete(input).await.unwrap(); + assert_eq!(tools.len(), 2); + assert_eq!(normal_text, ""); + assert_eq!(tools[0].function.name, "search"); + assert_eq!(tools[1].function.name, "translate"); +} + +#[tokio::test] +async fn test_minimax_type_conversion() { + let parser = MinimaxM2Parser::new(); + + let input = r#" + +42 +1.5 +true +null +string value + +"#; + + let (normal_text, tools) = parser.parse_complete(input).await.unwrap(); + assert_eq!(tools.len(), 1); + assert_eq!(normal_text, ""); + + let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap(); + assert_eq!(args["count"], 42); + assert_eq!(args["rate"], 1.5); + assert_eq!(args["enabled"], true); + assert_eq!(args["data"], serde_json::Value::Null); + assert_eq!(args["text"], "string value"); +} + +#[tokio::test] +async fn test_minimax_streaming_basic() { + let mut parser = MinimaxM2Parser::new(); + + let tools = create_test_tools(); + + // Simulate streaming chunks + let chunks = vec![ + "", + r#""#, + r#"Shanghai"#, + r#"celsius"#, + "", + "", + ]; + + let mut found_name = false; + let mut found_params = false; + + for chunk in chunks { + let result = parser.parse_incremental(chunk, &tools).await.unwrap(); + + for call in result.calls { + if let Some(name) = call.name { + assert_eq!(name, "get_weather"); + found_name = true; + } + if !call.parameters.is_empty() { + found_params = true; + } + } + } + + assert!(found_name, "Should have found tool name during streaming"); + assert!(found_params, "Should have streamed parameters"); +} + +#[test] +fn test_minimax_format_detection() { + let parser = MinimaxM2Parser::new(); + + // Should detect MiniMax format + assert!(parser.has_tool_markers("")); + assert!(parser.has_tool_markers("text with marker")); + + // Should not detect other formats + assert!(!parser.has_tool_markers("")); // GLM4 format + assert!(!parser.has_tool_markers("[TOOL_CALLS]")); + assert!(!parser.has_tool_markers("<|tool▁calls▁begin|>")); + assert!(!parser.has_tool_markers("plain text")); +} + +#[tokio::test] +async fn test_minimax_python_literals() { + let parser = MinimaxM2Parser::new(); + + let input = r#" + +True +False +None + +"#; + + let (_normal_text, tools) = parser.parse_complete(input).await.unwrap(); + assert_eq!(tools.len(), 1); + assert_eq!(tools[0].function.name, "test_func"); + + let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap(); + assert_eq!(args["bool_true"], true); + assert_eq!(args["bool_false"], false); + assert_eq!(args["none_val"], serde_json::Value::Null); +} + +#[tokio::test] +async fn test_minimax_nested_json_in_parameters() { + let parser = MinimaxM2Parser::new(); + + let input = r#" + +{"nested": {"key": "value"}} +[1, 2, 3] + +"#; + + let (_normal_text, tools) = parser.parse_complete(input).await.unwrap(); + assert_eq!(tools.len(), 1); + + let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap(); + // JSON-like strings are kept as strings, not parsed as JSON + // This matches the behavior of other parsers like GLM4 MOE + assert!(args["data"].is_string()); + assert_eq!(args["data"], r#"{"nested": {"key": "value"}}"#); + assert!(args["list"].is_string()); + assert_eq!(args["list"], "[1, 2, 3]"); +} + +#[tokio::test] +async fn test_minimax_xml_entities() { + let parser = MinimaxM2Parser::new(); + + let input = r#" + +<div>content</div> +Quote: "hello" +if (a && b) { } + +"#; + + let (_normal_text, tools) = parser.parse_complete(input).await.unwrap(); + assert_eq!(tools.len(), 1); + + let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap(); + assert_eq!(args["html"], "
content
"); + assert_eq!(args["text"], "Quote: \"hello\""); + assert_eq!(args["code"], "if (a && b) { }"); +} + +#[tokio::test] +async fn test_minimax_streaming_partial_tags() { + let mut parser = MinimaxM2Parser::new(); + let tools = create_test_tools(); + + // Chunks split mid-tag + let chunks = vec![ + "Bei"#, + "jing
", + ]; + + let mut found_name = false; + let mut buffer = String::new(); + + for chunk in chunks { + let result = parser.parse_incremental(chunk, &tools).await.unwrap(); + + buffer.push_str(&result.normal_text); + + for call in result.calls { + if let Some(name) = call.name { + assert_eq!(name, "get_weather"); + found_name = true; + } + } + } + + assert!( + found_name, + "Should have parsed function name from partial chunks" + ); + assert_eq!(buffer, ""); +} + +#[tokio::test] +async fn test_minimax_streaming_incremental_json() { + let mut parser = MinimaxM2Parser::new(); + let tools = create_test_tools(); + + let chunks = vec![ + "", + r#""#, + r#"Paris"#, + r#"metric"#, + "", + ]; + + let mut json_fragments = Vec::new(); + let mut found_function = false; + + for chunk in chunks { + let result = parser.parse_incremental(chunk, &tools).await.unwrap(); + + for call in result.calls { + if let Some(_name) = call.name { + found_function = true; + } + if !call.parameters.is_empty() { + json_fragments.push(call.parameters.clone()); + } + } + } + + assert!(found_function); + + // Verify JSON was built incrementally + assert!(!json_fragments.is_empty()); + + // First fragment should start with opening brace + if let Some(first) = json_fragments.first() { + assert!( + first.starts_with('{'), + "First JSON fragment should start with '{{': {}", + first + ); + } + + // Last fragment should be closing brace + if let Some(last) = json_fragments.last() { + assert!( + last.contains('}'), + "Last JSON fragment should contain '}}': {}", + last + ); + } +} + +#[tokio::test] +async fn test_minimax_multiple_tools_boundary() { + let mut parser = MinimaxM2Parser::new(); + let tools = create_test_tools(); + + // Tool boundary at chunk boundary + let chunks = vec![ + r#"Tokyo"#, + r#"weather forecast"#, + ]; + + let mut tool_names = Vec::new(); + + for chunk in chunks { + let result = parser.parse_incremental(chunk, &tools).await.unwrap(); + + for call in result.calls { + if let Some(name) = call.name { + tool_names.push(name); + } + } + } + + assert_eq!(tool_names.len(), 2); + assert_eq!(tool_names[0], "get_weather"); + assert_eq!(tool_names[1], "search"); +} + +#[tokio::test] +async fn test_minimax_invalid_function_name() { + let mut parser = MinimaxM2Parser::new(); + let tools = create_test_tools(); + + let chunks = vec![ + "", + r#""#, + r#"value"#, + "", + ]; + + let mut found_invalid = false; + + for chunk in chunks { + let result = parser.parse_incremental(chunk, &tools).await.unwrap(); + + // Invalid function should be skipped + for call in result.calls { + if let Some(name) = call.name { + if name == "invalid_function" { + found_invalid = true; + } + } + } + } + + assert!(!found_invalid, "Invalid function should not be parsed"); +} + +#[tokio::test] +async fn test_minimax_empty_parameters() { + let parser = MinimaxM2Parser::new(); + + let input = r#" + + +"#; + + let (_normal_text, tools) = parser.parse_complete(input).await.unwrap(); + assert_eq!(tools.len(), 1); + assert_eq!(tools[0].function.name, "simple_func"); + + let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap(); + assert_eq!(args, serde_json::json!({})); +} + +#[tokio::test] +async fn test_minimax_multiline_parameter_values() { + let parser = MinimaxM2Parser::new(); + + let input = r#" + +line1 +line2 +line3 +你好世界 🌍 + +"#; + + let (_normal_text, tools) = parser.parse_complete(input).await.unwrap(); + assert_eq!(tools.len(), 1); + + let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap(); + assert_eq!(args["multiline"], "line1\nline2\nline3"); + assert_eq!(args["unicode"], "你好世界 🌍"); +} + +#[tokio::test] +async fn test_minimax_nested_xml_like_content() { + let parser = MinimaxM2Parser::new(); + + let input = r#" + +Hello +{"key": "nested"} + +"#; + + let (_normal_text, tools) = parser.parse_complete(input).await.unwrap(); + assert_eq!(tools.len(), 1); + + let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap(); + assert_eq!(args["template"], "Hello"); + + // The nested JSON with XML-like content + let config = + serde_json::from_str::(args["config"].as_str().unwrap()).unwrap(); + assert_eq!(config["key"], "nested"); +} + +#[tokio::test] +async fn test_minimax_streaming_state_reset() { + let mut parser = MinimaxM2Parser::new(); + let tools = create_test_tools(); + + // First tool + let chunks1 = vec![ + r#""#, + r#"London"#, + "", + ]; + + for chunk in chunks1 { + parser.parse_incremental(chunk, &tools).await.unwrap(); + } + + // Second tool - state should be reset + let chunks2 = vec![ + r#""#, + r#"rust"#, + "", + ]; + + let mut second_tool_name = None; + for chunk in chunks2 { + let result = parser.parse_incremental(chunk, &tools).await.unwrap(); + for call in result.calls { + if let Some(name) = call.name { + second_tool_name = Some(name); + } + } + } + + assert_eq!(second_tool_name, Some("search".to_string())); +} + +#[tokio::test] +async fn test_minimax_many_parameters() { + let parser = MinimaxM2Parser::new(); + + let mut params_xml = String::new(); + for i in 1..=20 { + params_xml.push_str(&format!( + r#"value{} +"#, + i, i + )); + } + + let input = format!( + r#" + +{} + +"#, + params_xml + ); + + let (_normal_text, tools) = parser.parse_complete(&input).await.unwrap(); + assert_eq!(tools.len(), 1); + assert_eq!(tools[0].function.name, "complex_func"); + + let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap(); + + // Verify all 20 parameters are parsed + for i in 1..=20 { + let key = format!("param{}", i); + let expected_value = format!("value{}", i); + assert_eq!(args[key], expected_value); + } +} + +#[tokio::test] +async fn test_minimax_character_by_character_streaming() { + // Test character-by-character streaming to simulate real-world streaming + let mut parser = MinimaxM2Parser::new(); + let tools = create_test_tools(); + + let complete_text = r#"Let me help you. + +Seattle +celsius + + Here are the results."#; + + let mut content_collected = String::new(); + let mut tool_name_found = false; + let mut parameters_found = Vec::new(); + + // Stream character by character - feed only one character at a time + for i in 0..complete_text.len() { + let delta = &complete_text[i..i + 1]; + let result = parser.parse_incremental(delta, &tools).await.unwrap(); + content_collected.push_str(&result.normal_text); + + for call in result.calls { + if let Some(name) = call.name { + assert_eq!(name, "get_weather"); + tool_name_found = true; + } + if !call.parameters.is_empty() && !parameters_found.contains(&call.parameters) { + parameters_found.push(call.parameters.clone()); + } + } + } + + assert!( + tool_name_found, + "Should find tool name during character-by-character streaming" + ); + assert!( + !parameters_found.is_empty(), + "Should find parameters during streaming" + ); + + // Should have initial content and final content + assert!(content_collected.contains("Let me help you.")); + assert!(content_collected.contains("Here are the results.")); +} + +#[tokio::test] +async fn test_minimax_content_before_and_after_tool_calls() { + let parser = MinimaxM2Parser::new(); + + let input = r#"I'll analyze the weather for you now. + + +Boston +MA + + +Based on the analysis, here's what I found."#; + + let (normal_text, tools) = parser.parse_complete(input).await.unwrap(); + + // Verify tool extraction + assert_eq!(tools.len(), 1); + assert_eq!(tools[0].function.name, "get_weather"); + + // Verify content preservation (only text before tool call is returned) + assert!(normal_text.contains("I'll analyze the weather for you now.")); + // Text after tool call is not included in parse_complete + assert!(!normal_text.contains("Based on the analysis, here's what I found.")); + + let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap(); + assert_eq!(args["city"], "Boston"); + assert_eq!(args["state"], "MA"); +} + +#[tokio::test] +async fn test_minimax_incomplete_tool_call() { + let parser = MinimaxM2Parser::new(); + + // Incomplete tool call - missing closing tag + let input = r#" + +Chicago"#; + + let (normal_text, tools) = parser.parse_complete(input).await.unwrap(); + + // Should not extract incomplete tool calls + assert_eq!(tools.len(), 0); + assert_eq!(normal_text, input); // Should return as normal text +} + +#[tokio::test] +async fn test_minimax_malformed_invoke_tag() { + let parser = MinimaxM2Parser::new(); + + // Malformed invoke tag - missing name attribute + let input = r#" + +Miami + +"#; + + let (normal_text, tools) = parser.parse_complete(input).await.unwrap(); + + // Should not extract tool calls with malformed invoke tags + assert_eq!(tools.len(), 0); + assert_eq!(normal_text, input); +} + +#[tokio::test] +async fn test_minimax_streaming_with_invalid_function_progressive() { + let mut parser = MinimaxM2Parser::new(); + let tools = create_test_tools(); + + // Progressive chunks building an invalid function call + let chunks = vec![ + "", + r#""#, + r#"value"#, + "", + "", + ]; + + let mut all_normal_text = String::new(); + let mut found_valid_tool = false; + + for chunk in chunks { + let result = parser.parse_incremental(chunk, &tools).await.unwrap(); + all_normal_text.push_str(&result.normal_text); + + for call in result.calls { + if let Some(name) = call.name { + // Should not get here for invalid function + if tools.iter().any(|t| t.function.name == name) { + found_valid_tool = true; + } + } + } + } + + assert!( + !found_valid_tool, + "Invalid function should not be parsed as tool call" + ); + // The invalid tool call should be returned as normal text + assert!(all_normal_text.contains("invalid_function")); +} + +#[tokio::test] +async fn test_minimax_rapid_streaming_bursts() { + // Test handling of rapid streaming bursts (multiple chunks at once) + let mut parser = MinimaxM2Parser::new(); + let tools = create_test_tools(); + + let chunks = vec![ + "", + "rust programming", + "", + ]; + + let mut found_function = false; + let mut parameters = Vec::new(); + + for chunk in chunks { + let result = parser.parse_incremental(chunk, &tools).await.unwrap(); + + for call in result.calls { + if let Some(name) = call.name { + assert_eq!(name, "search"); + found_function = true; + } + if !call.parameters.is_empty() { + parameters.push(call.parameters.clone()); + } + } + } + + assert!(found_function); + + // Verify that parameters were streamed correctly + let final_params = parameters.join(""); + assert!(final_params.contains("rust programming")); +} + +#[tokio::test] +async fn test_minimax_special_characters_in_values() { + let parser = MinimaxM2Parser::new(); + + let input = r#" + +Special chars: @#$%^&*() +🦀 Rust 🚀 +"double" and 'single' quotes + +"#; + + let (_normal_text, tools) = parser.parse_complete(input).await.unwrap(); + assert_eq!(tools.len(), 1); + + let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap(); + assert_eq!(args["text"], "Special chars: @#$%^&*()"); + assert_eq!(args["emoji"], "🦀 Rust 🚀"); + assert_eq!(args["quotes"], "\"double\" and 'single' quotes"); +} + +#[tokio::test] +async fn test_minimax_whitespace_handling() { + let parser = MinimaxM2Parser::new(); + + // Test with various whitespace scenarios + let input = r#" + + spaces around + + Line 1 + Line 2 + + tab separated + +"#; + + let (_normal_text, tools) = parser.parse_complete(input).await.unwrap(); + assert_eq!(tools.len(), 1); + + let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap(); + // Values should preserve internal whitespace but may trim edges based on parser design + assert!(args["newlines"].as_str().unwrap().contains("Line 1")); + assert!(args["newlines"].as_str().unwrap().contains("Line 2")); + assert_eq!(args["tabs"], "\ttab\tseparated\t"); +} + +#[tokio::test] +async fn test_minimax_no_tools() { + // Test input with no tool calls at all + let parser = MinimaxM2Parser::new(); + + let input = r#"This is just a normal response without any tool calls. +I can provide information directly without using any tools. +Even if I mention function names like get_weather or search, +they are not actual tool calls unless properly formatted."#; + + let (normal_text, tools) = parser.parse_complete(input).await.unwrap(); + + // No tools should be extracted + assert_eq!( + tools.len(), + 0, + "Should not extract any tools from plain text" + ); + + // All content should be returned as normal text + assert_eq!( + normal_text, input, + "All content should be returned as normal text when no tools present" + ); +} + +#[tokio::test] +async fn test_minimax_invalid_json_in_parameters() { + // Test handling of invalid JSON in parameter values + let parser = MinimaxM2Parser::new(); + + let input = r#" + +{"key": "value"} +{invalid json: no quotes} +[1, 2, unclosed +Some text {"partial": json} more text + +"#; + + let (normal_text, tools) = parser.parse_complete(input).await.unwrap(); + + // Tool should still be extracted despite invalid JSON in parameters + assert_eq!( + tools.len(), + 1, + "Should extract tool even with invalid JSON in parameters" + ); + assert_eq!(tools[0].function.name, "process"); + + let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap(); + + // Parameters are stored as strings, not parsed as JSON + // Even invalid JSON should be preserved as string values + assert!(args["valid"].is_string()); + assert_eq!(args["valid"], r#"{"key": "value"}"#); + + assert!(args["invalid"].is_string()); + assert_eq!(args["invalid"], "{invalid json: no quotes}"); + + assert!(args["broken"].is_string()); + assert_eq!(args["broken"], "[1, 2, unclosed"); + + assert!(args["mixed"].is_string()); + assert_eq!(args["mixed"], r#"Some text {"partial": json} more text"#); + + assert_eq!(normal_text, ""); +}