From 26704c23c056e426c6bc86ea1289e82b5fd37e59 Mon Sep 17 00:00:00 2001 From: Haoran Wang <70007833+UbeCc@users.noreply.github.com> Date: Sun, 21 Dec 2025 12:07:51 +0800 Subject: [PATCH] [model-gateway]: Tool parser for glm47 (#15520) --- sgl-model-gateway/README.md | 6 +- .../benches/tool_parser_benchmark.rs | 7 +- .../src/reasoning_parser/README.md | 7 +- .../src/reasoning_parser/factory.rs | 1 + sgl-model-gateway/src/tool_parser/factory.rs | 8 +- .../src/tool_parser/parsers/glm4_moe.rs | 30 ++-- .../tests/tool_parser_glm47_moe.rs | 132 ++++++++++++++++++ .../tests/tool_parser_glm4_moe.rs | 14 +- 8 files changed, 179 insertions(+), 26 deletions(-) create mode 100644 sgl-model-gateway/tests/tool_parser_glm47_moe.rs diff --git a/sgl-model-gateway/README.md b/sgl-model-gateway/README.md index 074d45d85..64c0810ea 100644 --- a/sgl-model-gateway/README.md +++ b/sgl-model-gateway/README.md @@ -30,7 +30,7 @@ High-performance model routing control and data plane for large-scale LLM deploy - Multiple load balancing strategies (`random`, `round_robin`, `cache_aware`, `power_of_two`, `bucket`) with DP-aware scheduling. - Multi-model HTTP serving and inference gateway routing with model-specific policies. - Prefill/decode disaggregation, including bootstrap port handling and cache-aware merging. -- gRPC routing with fully Rust tokenizer loading, reasoning parser selection, and tool parser integration for OpenAI-compatible endpoints—supporting streaming and non-streaming modes across DeepSeek, Llama, Kimi K2, Qwen, GPT-OSS, Mistral, Step-3, GLM4, and other reasoning-capable models. +- gRPC routing with fully Rust tokenizer loading, reasoning parser selection, and tool parser integration for OpenAI-compatible endpoints—supporting streaming and non-streaming modes across DeepSeek, Llama, Kimi K2, Qwen, GPT-OSS, Mistral, Step-3, GLM4, GLM4.7 and other reasoning-capable models. - OpenAI-compatible `/v1/chat/completions`, `/v1/responses`, `/v1/conversations`, `/v1/embeddings`, and `/v1/rerank` endpoints. - Native MCP client integration supporting all MCP transport protocols (STDIO, HTTP, SSE, and Streamable) for tool execution loops. - Pluggable history connectors: in-memory, disabled, or Oracle ATP (with pooling and credential support). @@ -224,7 +224,7 @@ Add more workers with the same API; include optional `labels` (for per-model pol --port 8080 ``` The gRPC router tokenizes inputs locally, supports tool-call parsing, and streams responses. It supports both regular HTTP-equivalent serving and PD (prefill/decode) serving when the worker registry contains PD workers. Provide `--model-path` or `--tokenizer-path` (HuggingFace ID or local directory) whenever connection mode resolves to gRPC. -Use `--reasoning-parser` to select built-in reasoning pipelines (DeepSeek-R1, Qwen3, Step-3, GLM4, etc.) and `--tool-call-parser` for JSON/Pythonic/XML tool contracts in streaming or non-streaming modes. +Use `--reasoning-parser` to select built-in reasoning pipelines (DeepSeek-R1, Qwen3, Step-3, GLM4, GLM4.7, etc.) and `--tool-call-parser` for JSON/Pythonic/XML tool contracts in streaming or non-streaming modes. ### OpenAI Backend Mode Route requests to OpenAI or OpenAI-compatible endpoints: @@ -455,7 +455,7 @@ The HTTP router exposes the full OpenAI-compatible surface area (`/generate`, `/ - Industry-first fully Rust implementation of an OpenAI-compatible gRPC inference gateway, including tokenizer, reasoning parser, and tool parser execution in-process for maximum throughput. - Supports both single-stage and PD (prefill/decode) worker topologies; the router automatically selects the appropriate pipeline per model. - Provides the same `/v1/*` APIs as the HTTP router while streaming tokenized requests/responses directly to SRT gRPC workers. -- Built-in reasoning parsers for DeepSeek, Qwen, Llama, Mistral, GPT-OSS, Step-3, GLM4, Kimi K2, and other structured-thought models. +- Built-in reasoning parsers for DeepSeek, Qwen, Llama, Mistral, GPT-OSS, Step-3, GLM4, GLM4.7, Kimi K2, and other structured-thought models. - Tool-call parsers for JSON, Pythonic, XML, and custom schemas with streaming and non-streaming execution loops. - Tokenizer factory supporting HuggingFace models, local tokenizer.json files, and chat template overrides (see `src/tokenizer`). - Explore the code paths in `src/reasoning_parser`, `src/tool_parser`, and `src/tokenizer` for the end-to-end Rust implementations that power gRPC mode. diff --git a/sgl-model-gateway/benches/tool_parser_benchmark.rs b/sgl-model-gateway/benches/tool_parser_benchmark.rs index 30fd76f15..4ade90098 100644 --- a/sgl-model-gateway/benches/tool_parser_benchmark.rs +++ b/sgl-model-gateway/benches/tool_parser_benchmark.rs @@ -80,7 +80,7 @@ Let me examine the scan results and provide recommendations."#; const KIMIK2_FORMAT: &str = r#"⍼validate_and_deploy⍁{"deployment_config": {"application": "payment-service", "version": "2.3.1", "environment": "staging", "region": "us-west-2", "deployment_strategy": "blue_green", "health_check": {"endpoint": "/health", "interval": 30, "timeout": 5, "healthy_threshold": 2, "unhealthy_threshold": 3}, "rollback_on_failure": true, "canary_config": {"percentage": 10, "duration_minutes": 30, "metrics": ["error_rate", "latency_p99", "success_rate"], "thresholds": {"error_rate": 0.01, "latency_p99": 500, "success_rate": 0.99}}, "pre_deployment_hooks": ["run_tests", "security_scan", "backup_database"], "post_deployment_hooks": ["smoke_tests", "notify_team", "update_documentation"]}}"#; -const GLM4_FORMAT: &str = r#" +const GLM45_FORMAT: &str = r#" analyze_customer_behavior dataset_id=customer_interactions_2024 analysis_type=cohort_retention @@ -94,6 +94,8 @@ analyze_customer_behavior confidence_level=0.95 "#; +const GLM47_FORMAT: &str = r#"analyze_customer_behaviordataset_id=customer_interactions_2024analysis_type=cohort_retentioncohort_definition=signup_monthretention_periods=[1, 7, 14, 30, 60, 90, 180, 365]segment_by=["acquisition_channel", "pricing_tier", "industry", "company_size"]metrics=["active_users", "revenue", "feature_usage", "engagement_score"]statistical_tests=["chi_square", "anova", "trend_analysis"]visualization_types=["heatmap", "line_chart", "funnel", "sankey"]export_format=dashboardconfidence_level=0.95"#; + const STEP3_FORMAT: &str = r#" orchestrate_ml_pipeline @@ -285,7 +287,8 @@ fn bench_complete_parsing(c: &mut Criterion) { ("pythonic_multi", "pythonic", PYTHONIC_MULTI), ("deepseek", "deepseek", DEEPSEEK_FORMAT), ("kimik2", "kimik2", KIMIK2_FORMAT), - ("glm4", "glm4_moe", GLM4_FORMAT), + ("glm45", "glm45_moe", GLM45_FORMAT), + ("glm47", "glm47_moe", GLM47_FORMAT), ("step3", "step3", STEP3_FORMAT), ("gpt_oss", "gpt_oss", GPT_OSS_FORMAT), ]; diff --git a/sgl-model-gateway/src/reasoning_parser/README.md b/sgl-model-gateway/src/reasoning_parser/README.md index 5db6f11f1..05aaf9f9e 100644 --- a/sgl-model-gateway/src/reasoning_parser/README.md +++ b/sgl-model-gateway/src/reasoning_parser/README.md @@ -25,7 +25,7 @@ The reasoning parser layer provides a unified interface for detecting and extrac ### Architecture Highlights -- **Model-Specific Parsers**: DeepSeek-R1, Qwen3, Kimi, GLM45, Step3 variants +- **Model-Specific Parsers**: DeepSeek-R1, Qwen3, Kimi, GLM45, GLM47, Step3 variants - **Parser Pooling**: Singleton instances per model type for memory efficiency - **High Concurrency**: Mutex-protected parsers handle 1000+ req/sec - **Buffer Overflow Protection**: Configurable max buffer size (default 64KB) @@ -55,7 +55,7 @@ graph TB PP --> QW[Qwen3] PP --> QWT[Qwen3-Thinking] PP --> KM[Kimi] - PP --> GL[GLM45] + PP --> GL[GLM45/GLM47] PP --> S3[Step3] PP --> PT[Passthrough] end @@ -325,7 +325,7 @@ classDiagram - `qwen3`: Qwen3 base model (initial_in_reasoning=false) - `qwen3_thinking`: Qwen3 thinking variant (initial_in_reasoning=true) - `kimi`: Kimi with Unicode tokens -- `glm45`: GLM-4.5 / GLM-4.6 parser +- `glm45`: GLM-4.5 / GLM-4.6 / GLM-4.7 parser - `step3`: Step3 parser - `passthrough`: No-op fallback parser @@ -337,6 +337,7 @@ classDiagram "qwen3" → "qwen3" "qwen" → "qwen3" "glm45" → "glm45" +"glm47" → "glm45" "kimi" → "kimi" "step3" → "step3" ``` diff --git a/sgl-model-gateway/src/reasoning_parser/factory.rs b/sgl-model-gateway/src/reasoning_parser/factory.rs index 1f0786f05..5269454ff 100644 --- a/sgl-model-gateway/src/reasoning_parser/factory.rs +++ b/sgl-model-gateway/src/reasoning_parser/factory.rs @@ -199,6 +199,7 @@ impl ParserFactory { registry.register_pattern("qwen3", "qwen3"); registry.register_pattern("qwen", "qwen3"); registry.register_pattern("glm45", "glm45"); + registry.register_pattern("glm47", "glm45"); // glm47 uses same reasoning format as glm45 registry.register_pattern("kimi", "kimi"); registry.register_pattern("step3", "step3"); registry.register_pattern("minimax", "minimax"); diff --git a/sgl-model-gateway/src/tool_parser/factory.rs b/sgl-model-gateway/src/tool_parser/factory.rs index c22b2f8c6..265d8b573 100644 --- a/sgl-model-gateway/src/tool_parser/factory.rs +++ b/sgl-model-gateway/src/tool_parser/factory.rs @@ -239,7 +239,8 @@ impl ParserFactory { registry.register_parser("pythonic", || Box::new(PythonicParser::new())); registry.register_parser("llama", || Box::new(LlamaParser::new())); registry.register_parser("deepseek", || Box::new(DeepSeekParser::new())); - registry.register_parser("glm4_moe", || Box::new(Glm4MoeParser::new())); + registry.register_parser("glm45_moe", || Box::new(Glm4MoeParser::glm45())); + registry.register_parser("glm47_moe", || Box::new(Glm4MoeParser::glm47())); registry.register_parser("step3", || Box::new(Step3Parser::new())); registry.register_parser("kimik2", || Box::new(KimiK2Parser::new())); registry.register_parser("minimax_m2", || Box::new(MinimaxM2Parser::new())); @@ -281,8 +282,9 @@ impl ParserFactory { registry.map_model("deepseek-*", "pythonic"); // GLM models - registry.map_model("glm-4.5*", "glm4_moe"); - registry.map_model("glm-4.6*", "glm4_moe"); + registry.map_model("glm-4.5*", "glm45_moe"); + registry.map_model("glm-4.6*", "glm45_moe"); + registry.map_model("glm-4.7*", "glm47_moe"); registry.map_model("glm-*", "json"); // Step3 models diff --git a/sgl-model-gateway/src/tool_parser/parsers/glm4_moe.rs b/sgl-model-gateway/src/tool_parser/parsers/glm4_moe.rs index 74662a761..e72b3b81c 100644 --- a/sgl-model-gateway/src/tool_parser/parsers/glm4_moe.rs +++ b/sgl-model-gateway/src/tool_parser/parsers/glm4_moe.rs @@ -14,8 +14,9 @@ use crate::{ /// GLM-4 MoE format parser for tool calls /// -/// Handles the GLM-4 MoE specific format: -/// `{name}\n{key}\n{value}\n` +/// Handles both GLM-4 MoE and GLM-4.7 MoE formats: +/// - GLM-4: `{name}\n{key}\n{value}\n` +/// - GLM-4.7: `{name}{key}{value}` /// /// Features: /// - XML-style tags for tool calls @@ -47,13 +48,17 @@ pub struct Glm4MoeParser { } impl Glm4MoeParser { - /// Create a new GLM-4 MoE parser - pub fn new() -> Self { + /// Create a new generic GLM MoE parser with a custom func_detail_extractor pattern + /// + /// # Arguments + /// - `func_detail_pattern`: Regex pattern for extracting function name and arguments + /// - For GLM-4: `r"(?s)([^\n]*)\n(.*)"` + /// - For GLM-4.7: `r"(?s)\s*([^<\s]+)\s*(.*?)"` + pub(crate) fn new(func_detail_pattern: &str) -> Self { // Use (?s) flag for DOTALL mode to handle newlines let tool_call_pattern = r"(?s).*?"; let tool_call_extractor = Regex::new(tool_call_pattern).expect("Valid regex pattern"); - let func_detail_pattern = r"(?s)([^\n]*)\n(.*)"; let func_detail_extractor = Regex::new(func_detail_pattern).expect("Valid regex pattern"); let arg_pattern = r"(?s)(.*?)\s*(.*?)"; @@ -72,6 +77,16 @@ impl Glm4MoeParser { } } + /// Create a new GLM-4.5/4.6 MoE parser (with newline-based format) + pub fn glm45() -> Self { + Self::new(r"(?s)([^\n]*)\n(.*)") + } + + /// Create a new GLM-4.7 MoE parser (with whitespace-based format) + pub fn glm47() -> Self { + Self::new(r"(?s)\s*([^<\s]+)\s*(.*?)") + } + /// Parse arguments from key-value pairs fn parse_arguments(&self, args_text: &str) -> ParserResult> { let mut arguments = serde_json::Map::new(); @@ -136,7 +151,6 @@ impl Glm4MoeParser { } } - /// Parse and return StreamingParseResult (mirrors Python's detect_and_parse) /// Parse all tool calls from text (shared logic for complete and incremental parsing) fn parse_tool_calls_from_text(&self, text: &str) -> ParserResult> { let mut tools = Vec::new(); @@ -158,7 +172,7 @@ impl Glm4MoeParser { impl Default for Glm4MoeParser { fn default() -> Self { - Self::new() + Self::glm45() } } @@ -257,7 +271,7 @@ impl ToolParser for Glm4MoeParser { tracing::debug!("Invalid tool name '{}' - skipping", tool_call.function.name); helpers::reset_current_tool_state( &mut self.buffer, - &mut false, // glm4_moe doesn't track name_sent per tool + &mut false, // glm45_moe/glm47_moe doesn't track name_sent per tool &mut self.streamed_args_for_tool, &self.prev_tool_call_arr, ); diff --git a/sgl-model-gateway/tests/tool_parser_glm47_moe.rs b/sgl-model-gateway/tests/tool_parser_glm47_moe.rs new file mode 100644 index 000000000..f72145788 --- /dev/null +++ b/sgl-model-gateway/tests/tool_parser_glm47_moe.rs @@ -0,0 +1,132 @@ +//! GLM-4.7 MoE Parser Integration Tests + +use sgl_model_gateway::tool_parser::{Glm4MoeParser, ToolParser}; + +mod common; +use common::create_test_tools; + +#[tokio::test] +async fn test_glm47_complete_parsing() { + let parser = Glm4MoeParser::glm47(); + + let input = r#"Let me search for that. +get_weathercityBeijingdate2024-12-25 +The weather will be..."#; + + let (normal_text, tools) = parser.parse_complete(input).await.unwrap(); + assert_eq!(tools.len(), 1); + assert_eq!(normal_text, "Let me search for that.\n"); + assert_eq!(tools[0].function.name, "get_weather"); + + let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap(); + assert_eq!(args["city"], "Beijing"); + assert_eq!(args["date"], "2024-12-25"); +} + +#[tokio::test] +async fn test_glm47_multiple_tools() { + let parser = Glm4MoeParser::glm47(); + + let input = r#"searchqueryrust tutorialstranslatetextHello Worldtarget_langzh"#; + + let (normal_text, tools) = parser.parse_complete(input).await.unwrap(); + assert_eq!(tools.len(), 2); + assert_eq!(normal_text, ""); + assert_eq!(tools[0].function.name, "search"); + assert_eq!(tools[1].function.name, "translate"); +} + +#[tokio::test] +async fn test_glm47_type_conversion() { + let parser = Glm4MoeParser::glm47(); + + let input = r#"processcount42rate1.5enabledtruedatanulltextstring value"#; + + let (normal_text, tools) = parser.parse_complete(input).await.unwrap(); + assert_eq!(tools.len(), 1); + assert_eq!(normal_text, ""); + + let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap(); + assert_eq!(args["count"], 42); + assert_eq!(args["rate"], 1.5); + assert_eq!(args["enabled"], true); + assert_eq!(args["data"], serde_json::Value::Null); + assert_eq!(args["text"], "string value"); +} + +#[tokio::test] +async fn test_glm47_streaming() { + let mut parser = Glm4MoeParser::glm47(); + + let tools = create_test_tools(); + + // Simulate streaming chunks + let chunks = vec![ + "", + "get_weather", + "city", + "Shanghai", + "units", + "celsius", + "", + ]; + + let mut found_name = false; + + for chunk in chunks { + let result = parser.parse_incremental(chunk, &tools).await.unwrap(); + + for call in result.calls { + if let Some(name) = call.name { + assert_eq!(name, "get_weather"); + found_name = true; + } + } + } + + assert!(found_name, "Should have found tool name during streaming"); +} + +#[test] +fn test_glm47_format_detection() { + let parser = Glm4MoeParser::glm47(); + + // Should detect GLM-4 format + assert!(parser.has_tool_markers("")); + assert!(parser.has_tool_markers("text with marker")); + + // Should not detect other formats + assert!(!parser.has_tool_markers("[TOOL_CALLS]")); + assert!(!parser.has_tool_markers("<｜tool▁calls▁begin｜>")); + assert!(!parser.has_tool_markers("plain text")); +} + +#[tokio::test] +async fn test_python_literals() { + let parser = Glm4MoeParser::glm47(); + + let input = r#"test_funcbool_trueTruebool_falseFalsenone_valNone"#; + + let (_normal_text, tools) = parser.parse_complete(input).await.unwrap(); + assert_eq!(tools.len(), 1); + assert_eq!(tools[0].function.name, "test_func"); + + let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap(); + assert_eq!(args["bool_true"], true); + assert_eq!(args["bool_false"], false); + assert_eq!(args["none_val"], serde_json::Value::Null); +} + +#[tokio::test] +async fn test_glm47_nested_json_in_arg_values() { + let parser = Glm4MoeParser::glm47(); + + let input = r#"processdata{"nested": {"key": "value"}}list[1, 2, 3]"#; + + let (_normal_text, tools) = parser.parse_complete(input).await.unwrap(); + assert_eq!(tools.len(), 1); + + let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap(); + assert!(args["data"].is_object()); + assert!(args["list"].is_array()); +} diff --git a/sgl-model-gateway/tests/tool_parser_glm4_moe.rs b/sgl-model-gateway/tests/tool_parser_glm4_moe.rs index 1e001a9f6..29486af29 100644 --- a/sgl-model-gateway/tests/tool_parser_glm4_moe.rs +++ b/sgl-model-gateway/tests/tool_parser_glm4_moe.rs @@ -7,7 +7,7 @@ use common::create_test_tools; #[tokio::test] async fn test_glm4_complete_parsing() { - let parser = Glm4MoeParser::new(); + let parser = Glm4MoeParser::glm45(); let input = r#"Let me search for that. get_weather @@ -30,7 +30,7 @@ The weather will be..."#; #[tokio::test] async fn test_glm4_multiple_tools() { - let parser = Glm4MoeParser::new(); + let parser = Glm4MoeParser::glm45(); let input = r#"search query @@ -52,7 +52,7 @@ async fn test_glm4_multiple_tools() { #[tokio::test] async fn test_glm4_type_conversion() { - let parser = Glm4MoeParser::new(); + let parser = Glm4MoeParser::glm45(); let input = r#"process count @@ -81,7 +81,7 @@ async fn test_glm4_type_conversion() { #[tokio::test] async fn test_glm4_streaming() { - let mut parser = Glm4MoeParser::new(); + let mut parser = Glm4MoeParser::glm45(); let tools = create_test_tools(); @@ -114,7 +114,7 @@ async fn test_glm4_streaming() { #[test] fn test_glm4_format_detection() { - let parser = Glm4MoeParser::new(); + let parser = Glm4MoeParser::glm45(); // Should detect GLM-4 format assert!(parser.has_tool_markers("")); @@ -128,7 +128,7 @@ fn test_glm4_format_detection() { #[tokio::test] async fn test_python_literals() { - let parser = Glm4MoeParser::new(); + let parser = Glm4MoeParser::glm45(); let input = r#"test_func bool_true @@ -151,7 +151,7 @@ async fn test_python_literals() { #[tokio::test] async fn test_glm4_nested_json_in_arg_values() { - let parser = Glm4MoeParser::new(); + let parser = Glm4MoeParser::glm45(); let input = r#"process data