chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 11 Packages Projects Releases Wiki Activity

[model-gateway]: Tool parser for glm47 (#15520)

Browse Source
This commit is contained in:
Haoran Wang
2025-12-21 12:07:51 +08:00
committed by GitHub
parent 43b7c17474
commit 26704c23c0
8 changed files with 179 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
sgl-model-gateway/README.md
View File

@@ -30,7 +30,7 @@ High-performance model routing control and data plane for large-scale LLM deploy
- Multiple load balancing strategies (`random`, `round_robin`, `cache_aware`, `power_of_two`, `bucket`) with DP-aware scheduling.
- Multi-model HTTP serving and inference gateway routing with model-specific policies.
- Prefill/decode disaggregation, including bootstrap port handling and cache-aware merging.
- gRPC routing with fully Rust tokenizer loading, reasoning parser selection, and tool parser integration for OpenAI-compatible endpoints—supporting streaming and non-streaming modes across DeepSeek, Llama, Kimi K2, Qwen, GPT-OSS, Mistral, Step-3, GLM4, and other reasoning-capable models.
- gRPC routing with fully Rust tokenizer loading, reasoning parser selection, and tool parser integration for OpenAI-compatible endpoints—supporting streaming and non-streaming modes across DeepSeek, Llama, Kimi K2, Qwen, GPT-OSS, Mistral, Step-3, GLM4, GLM4.7 and other reasoning-capable models.
- OpenAI-compatible `/v1/chat/completions`, `/v1/responses`, `/v1/conversations`, `/v1/embeddings`, and `/v1/rerank` endpoints.
- Native MCP client integration supporting all MCP transport protocols (STDIO, HTTP, SSE, and Streamable) for tool execution loops.
- Pluggable history connectors: in-memory, disabled, or Oracle ATP (with pooling and credential support).
@@ -224,7 +224,7 @@ Add more workers with the same API; include optional `labels` (for per-model pol
--port 8080
```
The gRPC router tokenizes inputs locally, supports tool-call parsing, and streams responses. It supports both regular HTTP-equivalent serving and PD (prefill/decode) serving when the worker registry contains PD workers. Provide `--model-path` or `--tokenizer-path` (HuggingFace ID or local directory) whenever connection mode resolves to gRPC.
Use `--reasoning-parser` to select built-in reasoning pipelines (DeepSeek-R1, Qwen3, Step-3, GLM4, etc.) and `--tool-call-parser` for JSON/Pythonic/XML tool contracts in streaming or non-streaming modes.
Use `--reasoning-parser` to select built-in reasoning pipelines (DeepSeek-R1, Qwen3, Step-3, GLM4, GLM4.7, etc.) and `--tool-call-parser` for JSON/Pythonic/XML tool contracts in streaming or non-streaming modes.
### OpenAI Backend Mode
Route requests to OpenAI or OpenAI-compatible endpoints:
@@ -455,7 +455,7 @@ The HTTP router exposes the full OpenAI-compatible surface area (`/generate`, `/
- Industry-first fully Rust implementation of an OpenAI-compatible gRPC inference gateway, including tokenizer, reasoning parser, and tool parser execution in-process for maximum throughput.
- Supports both single-stage and PD (prefill/decode) worker topologies; the router automatically selects the appropriate pipeline per model.
- Provides the same `/v1/*` APIs as the HTTP router while streaming tokenized requests/responses directly to SRT gRPC workers.
- Built-in reasoning parsers for DeepSeek, Qwen, Llama, Mistral, GPT-OSS, Step-3, GLM4, Kimi K2, and other structured-thought models.
- Built-in reasoning parsers for DeepSeek, Qwen, Llama, Mistral, GPT-OSS, Step-3, GLM4, GLM4.7, Kimi K2, and other structured-thought models.
- Tool-call parsers for JSON, Pythonic, XML, and custom schemas with streaming and non-streaming execution loops.
- Tokenizer factory supporting HuggingFace models, local tokenizer.json files, and chat template overrides (see `src/tokenizer`).
- Explore the code paths in `src/reasoning_parser`, `src/tool_parser`, and `src/tokenizer` for the end-to-end Rust implementations that power gRPC mode.

7
sgl-model-gateway/benches/tool_parser_benchmark.rs
View File

@@ -80,7 +80,7 @@ Let me examine the scan results and provide recommendations."#;
const KIMIK2_FORMAT: &str = r#"⍼validate_and_deploy⍁{"deployment_config": {"application": "payment-service", "version": "2.3.1", "environment": "staging", "region": "us-west-2", "deployment_strategy": "blue_green", "health_check": {"endpoint": "/health", "interval": 30, "timeout": 5, "healthy_threshold": 2, "unhealthy_threshold": 3}, "rollback_on_failure": true, "canary_config": {"percentage": 10, "duration_minutes": 30, "metrics": ["error_rate", "latency_p99", "success_rate"], "thresholds": {"error_rate": 0.01, "latency_p99": 500, "success_rate": 0.99}}, "pre_deployment_hooks": ["run_tests", "security_scan", "backup_database"], "post_deployment_hooks": ["smoke_tests", "notify_team", "update_documentation"]}}"#;
const GLM4_FORMAT: &str = r#"<tool>
const GLM45_FORMAT: &str = r#"<tool>
analyze_customer_behavior
<parameter>dataset_id=customer_interactions_2024</parameter>
<parameter>analysis_type=cohort_retention</parameter>
@@ -94,6 +94,8 @@ analyze_customer_behavior
<parameter>confidence_level=0.95</parameter>
</tool>"#;
const GLM47_FORMAT: &str = r#"<tool>analyze_customer_behavior<parameter>dataset_id=customer_interactions_2024</parameter><parameter>analysis_type=cohort_retention</parameter><parameter>cohort_definition=signup_month</parameter><parameter>retention_periods=[1, 7, 14, 30, 60, 90, 180, 365]</parameter><parameter>segment_by=["acquisition_channel", "pricing_tier", "industry", "company_size"]</parameter><parameter>metrics=["active_users", "revenue", "feature_usage", "engagement_score"]</parameter><parameter>statistical_tests=["chi_square", "anova", "trend_analysis"]</parameter><parameter>visualization_types=["heatmap", "line_chart", "funnel", "sankey"]</parameter><parameter>export_format=dashboard</parameter><parameter>confidence_level=0.95</parameter></tool>"#;
const STEP3_FORMAT: &str = r#"<step.tML version="0.1">
<call>
<name>orchestrate_ml_pipeline</name>
@@ -285,7 +287,8 @@ fn bench_complete_parsing(c: &mut Criterion) {
("pythonic_multi", "pythonic", PYTHONIC_MULTI),
("deepseek", "deepseek", DEEPSEEK_FORMAT),
("kimik2", "kimik2", KIMIK2_FORMAT),
("glm4", "glm4_moe", GLM4_FORMAT),
("glm45", "glm45_moe", GLM45_FORMAT),
("glm47", "glm47_moe", GLM47_FORMAT),
("step3", "step3", STEP3_FORMAT),
("gpt_oss", "gpt_oss", GPT_OSS_FORMAT),
];

7
sgl-model-gateway/src/reasoning_parser/README.md
View File

@@ -25,7 +25,7 @@ The reasoning parser layer provides a unified interface for detecting and extrac
### Architecture Highlights
- **Model-Specific Parsers**: DeepSeek-R1, Qwen3, Kimi, GLM45, Step3 variants
- **Model-Specific Parsers**: DeepSeek-R1, Qwen3, Kimi, GLM45, GLM47, Step3 variants
- **Parser Pooling**: Singleton instances per model type for memory efficiency
- **High Concurrency**: Mutex-protected parsers handle 1000+ req/sec
- **Buffer Overflow Protection**: Configurable max buffer size (default 64KB)
@@ -55,7 +55,7 @@ graph TB
PP --> QW[Qwen3]
PP --> QWT[Qwen3-Thinking]
PP --> KM[Kimi]
PP --> GL[GLM45]
PP --> GL[GLM45/GLM47]
PP --> S3[Step3]
PP --> PT[Passthrough]
end
@@ -325,7 +325,7 @@ classDiagram
- `qwen3`: Qwen3 base model (initial_in_reasoning=false)
- `qwen3_thinking`: Qwen3 thinking variant (initial_in_reasoning=true)
- `kimi`: Kimi with Unicode tokens
- `glm45`: GLM-4.5 / GLM-4.6 parser
- `glm45`: GLM-4.5 / GLM-4.6 / GLM-4.7 parser
- `step3`: Step3 parser
- `passthrough`: No-op fallback parser
@@ -337,6 +337,7 @@ classDiagram
"qwen3" → "qwen3"
"qwen" → "qwen3"
"glm45" → "glm45"
"glm47" → "glm45"
"kimi" → "kimi"
"step3" → "step3"
```

1
sgl-model-gateway/src/reasoning_parser/factory.rs
View File

@@ -199,6 +199,7 @@ impl ParserFactory {
registry.register_pattern("qwen3", "qwen3");
registry.register_pattern("qwen", "qwen3");
registry.register_pattern("glm45", "glm45");
registry.register_pattern("glm47", "glm45"); // glm47 uses same reasoning format as glm45
registry.register_pattern("kimi", "kimi");
registry.register_pattern("step3", "step3");
registry.register_pattern("minimax", "minimax");

8
sgl-model-gateway/src/tool_parser/factory.rs
View File

@@ -239,7 +239,8 @@ impl ParserFactory {
registry.register_parser("pythonic", || Box::new(PythonicParser::new()));
registry.register_parser("llama", || Box::new(LlamaParser::new()));
registry.register_parser("deepseek", || Box::new(DeepSeekParser::new()));
registry.register_parser("glm4_moe", || Box::new(Glm4MoeParser::new()));
registry.register_parser("glm45_moe", || Box::new(Glm4MoeParser::glm45()));
registry.register_parser("glm47_moe", || Box::new(Glm4MoeParser::glm47()));
registry.register_parser("step3", || Box::new(Step3Parser::new()));
registry.register_parser("kimik2", || Box::new(KimiK2Parser::new()));
registry.register_parser("minimax_m2", || Box::new(MinimaxM2Parser::new()));
@@ -281,8 +282,9 @@ impl ParserFactory {
registry.map_model("deepseek-*", "pythonic");
// GLM models
registry.map_model("glm-4.5*", "glm4_moe");
registry.map_model("glm-4.6*", "glm4_moe");
registry.map_model("glm-4.5*", "glm45_moe");
registry.map_model("glm-4.6*", "glm45_moe");
registry.map_model("glm-4.7*", "glm47_moe");
registry.map_model("glm-*", "json");
// Step3 models

30
sgl-model-gateway/src/tool_parser/parsers/glm4_moe.rs
View File

@@ -14,8 +14,9 @@ use crate::{
/// GLM-4 MoE format parser for tool calls
///
/// Handles the GLM-4 MoE specific format:
/// `<tool_call>{name}\n<arg_key>{key}</arg_key>\n<arg_value>{value}</arg_value>\n</tool_call>`
/// Handles both GLM-4 MoE and GLM-4.7 MoE formats:
/// - GLM-4: `<tool_call>{name}\n<arg_key>{key}</arg_key>\n<arg_value>{value}</arg_value>\n</tool_call>`
/// - GLM-4.7: `<tool_call>{name}<arg_key>{key}</arg_key><arg_value>{value}</arg_value></tool_call>`
///
/// Features:
/// - XML-style tags for tool calls
@@ -47,13 +48,17 @@ pub struct Glm4MoeParser {
}
impl Glm4MoeParser {
/// Create a new GLM-4 MoE parser
pub fn new() -> Self {
/// Create a new generic GLM MoE parser with a custom func_detail_extractor pattern
///
/// # Arguments
/// - `func_detail_pattern`: Regex pattern for extracting function name and arguments
/// - For GLM-4: `r"(?s)<tool_call>([^\n]*)\n(.*)</tool_call>"`
/// - For GLM-4.7: `r"(?s)<tool_call>\s*([^<\s]+)\s*(.*?)</tool_call>"`
pub(crate) fn new(func_detail_pattern: &str) -> Self {
// Use (?s) flag for DOTALL mode to handle newlines
let tool_call_pattern = r"(?s)<tool_call>.*?</tool_call>";
let tool_call_extractor = Regex::new(tool_call_pattern).expect("Valid regex pattern");
let func_detail_pattern = r"(?s)<tool_call>([^\n]*)\n(.*)</tool_call>";
let func_detail_extractor = Regex::new(func_detail_pattern).expect("Valid regex pattern");
let arg_pattern = r"(?s)<arg_key>(.*?)</arg_key>\s*<arg_value>(.*?)</arg_value>";
@@ -72,6 +77,16 @@ impl Glm4MoeParser {
}
}
/// Create a new GLM-4.5/4.6 MoE parser (with newline-based format)
pub fn glm45() -> Self {
Self::new(r"(?s)<tool_call>([^\n]*)\n(.*)</tool_call>")
}
/// Create a new GLM-4.7 MoE parser (with whitespace-based format)
pub fn glm47() -> Self {
Self::new(r"(?s)<tool_call>\s*([^<\s]+)\s*(.*?)</tool_call>")
}
/// Parse arguments from key-value pairs
fn parse_arguments(&self, args_text: &str) -> ParserResult<serde_json::Map<String, Value>> {
let mut arguments = serde_json::Map::new();
@@ -136,7 +151,6 @@ impl Glm4MoeParser {
}
}
/// Parse and return StreamingParseResult (mirrors Python's detect_and_parse)
/// Parse all tool calls from text (shared logic for complete and incremental parsing)
fn parse_tool_calls_from_text(&self, text: &str) -> ParserResult<Vec<ToolCall>> {
let mut tools = Vec::new();
@@ -158,7 +172,7 @@ impl Glm4MoeParser {
impl Default for Glm4MoeParser {
fn default() -> Self {
Self::new()
Self::glm45()
}
}
@@ -257,7 +271,7 @@ impl ToolParser for Glm4MoeParser {
tracing::debug!("Invalid tool name '{}' - skipping", tool_call.function.name);
helpers::reset_current_tool_state(
&mut self.buffer,
&mut false, // glm4_moe doesn't track name_sent per tool
&mut false, // glm45_moe/glm47_moe doesn't track name_sent per tool
&mut self.streamed_args_for_tool,
&self.prev_tool_call_arr,
);

132
sgl-model-gateway/tests/tool_parser_glm47_moe.rs Normal file
View File

@@ -0,0 +1,132 @@
//! GLM-4.7 MoE Parser Integration Tests
use sgl_model_gateway::tool_parser::{Glm4MoeParser, ToolParser};
mod common;
use common::create_test_tools;
#[tokio::test]
async fn test_glm47_complete_parsing() {
let parser = Glm4MoeParser::glm47();
let input = r#"Let me search for that.
<tool_call>get_weather<arg_key>city</arg_key><arg_value>Beijing</arg_value><arg_key>date</arg_key><arg_value>2024-12-25</arg_value></tool_call>
The weather will be..."#;
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(normal_text, "Let me search for that.\n");
assert_eq!(tools[0].function.name, "get_weather");
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert_eq!(args["city"], "Beijing");
assert_eq!(args["date"], "2024-12-25");
}
#[tokio::test]
async fn test_glm47_multiple_tools() {
let parser = Glm4MoeParser::glm47();
let input = r#"<tool_call>search<arg_key>query</arg_key><arg_value>rust tutorials</arg_value></tool_call><tool_call>translate<arg_key>text</arg_key><arg_value>Hello World</arg_value><arg_key>target_lang</arg_key><arg_value>zh</arg_value></tool_call>"#;
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 2);
assert_eq!(normal_text, "");
assert_eq!(tools[0].function.name, "search");
assert_eq!(tools[1].function.name, "translate");
}
#[tokio::test]
async fn test_glm47_type_conversion() {
let parser = Glm4MoeParser::glm47();
let input = r#"<tool_call>process<arg_key>count</arg_key><arg_value>42</arg_value><arg_key>rate</arg_key><arg_value>1.5</arg_value><arg_key>enabled</arg_key><arg_value>true</arg_value><arg_key>data</arg_key><arg_value>null</arg_value><arg_key>text</arg_key><arg_value>string value</arg_value></tool_call>"#;
let (normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(normal_text, "");
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert_eq!(args["count"], 42);
assert_eq!(args["rate"], 1.5);
assert_eq!(args["enabled"], true);
assert_eq!(args["data"], serde_json::Value::Null);
assert_eq!(args["text"], "string value");
}
#[tokio::test]
async fn test_glm47_streaming() {
let mut parser = Glm4MoeParser::glm47();
let tools = create_test_tools();
// Simulate streaming chunks
let chunks = vec![
"<tool_call>",
"get_weather",
"<arg_key>city</arg_key>",
"<arg_value>Shanghai</arg_value>",
"<arg_key>units</arg_key>",
"<arg_value>celsius</arg_value>",
"</tool_call>",
];
let mut found_name = false;
for chunk in chunks {
let result = parser.parse_incremental(chunk, &tools).await.unwrap();
for call in result.calls {
if let Some(name) = call.name {
assert_eq!(name, "get_weather");
found_name = true;
}
}
}
assert!(found_name, "Should have found tool name during streaming");
}
#[test]
fn test_glm47_format_detection() {
let parser = Glm4MoeParser::glm47();
// Should detect GLM-4 format
assert!(parser.has_tool_markers("<tool_call>"));
assert!(parser.has_tool_markers("text with <tool_call> marker"));
// Should not detect other formats
assert!(!parser.has_tool_markers("[TOOL_CALLS]"));
assert!(!parser.has_tool_markers("<｜tool▁calls▁begin｜>"));
assert!(!parser.has_tool_markers("plain text"));
}
#[tokio::test]
async fn test_python_literals() {
let parser = Glm4MoeParser::glm47();
let input = r#"<tool_call>test_func<arg_key>bool_true</arg_key><arg_value>True</arg_value><arg_key>bool_false</arg_key><arg_value>False</arg_value><arg_key>none_val</arg_key><arg_value>None</arg_value></tool_call>"#;
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, "test_func");
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert_eq!(args["bool_true"], true);
assert_eq!(args["bool_false"], false);
assert_eq!(args["none_val"], serde_json::Value::Null);
}
#[tokio::test]
async fn test_glm47_nested_json_in_arg_values() {
let parser = Glm4MoeParser::glm47();
let input = r#"<tool_call>process<arg_key>data</arg_key><arg_value>{"nested": {"key": "value"}}</arg_value><arg_key>list</arg_key><arg_value>[1, 2, 3]</arg_value></tool_call>"#;
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert!(args["data"].is_object());
assert!(args["list"].is_array());
}

14
sgl-model-gateway/tests/tool_parser_glm4_moe.rs
View File

@@ -7,7 +7,7 @@ use common::create_test_tools;
#[tokio::test]
async fn test_glm4_complete_parsing() {
let parser = Glm4MoeParser::new();
let parser = Glm4MoeParser::glm45();
let input = r#"Let me search for that.
<tool_call>get_weather
@@ -30,7 +30,7 @@ The weather will be..."#;
#[tokio::test]
async fn test_glm4_multiple_tools() {
let parser = Glm4MoeParser::new();
let parser = Glm4MoeParser::glm45();
let input = r#"<tool_call>search
<arg_key>query</arg_key>
@@ -52,7 +52,7 @@ async fn test_glm4_multiple_tools() {
#[tokio::test]
async fn test_glm4_type_conversion() {
let parser = Glm4MoeParser::new();
let parser = Glm4MoeParser::glm45();
let input = r#"<tool_call>process
<arg_key>count</arg_key>
@@ -81,7 +81,7 @@ async fn test_glm4_type_conversion() {
#[tokio::test]
async fn test_glm4_streaming() {
let mut parser = Glm4MoeParser::new();
let mut parser = Glm4MoeParser::glm45();
let tools = create_test_tools();
@@ -114,7 +114,7 @@ async fn test_glm4_streaming() {
#[test]
fn test_glm4_format_detection() {
let parser = Glm4MoeParser::new();
let parser = Glm4MoeParser::glm45();
// Should detect GLM-4 format
assert!(parser.has_tool_markers("<tool_call>"));
@@ -128,7 +128,7 @@ fn test_glm4_format_detection() {
#[tokio::test]
async fn test_python_literals() {
let parser = Glm4MoeParser::new();
let parser = Glm4MoeParser::glm45();
let input = r#"<tool_call>test_func
<arg_key>bool_true</arg_key>
@@ -151,7 +151,7 @@ async fn test_python_literals() {
#[tokio::test]
async fn test_glm4_nested_json_in_arg_values() {
let parser = Glm4MoeParser::new();
let parser = Glm4MoeParser::glm45();
let input = r#"<tool_call>process
<arg_key>data</arg_key>