chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 14 Packages Projects Releases Wiki Activity

[router] Support structured model output for openai and grpc router (#12431)

Browse Source
This commit is contained in:
Keyang Ru
2025-11-07 10:16:45 -08:00
committed by GitHub
parent e316bcacb1
commit 5c9273c032
13 changed files with 469 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files
sgl-router/py_test/e2e_response_api/backends/test_grpc_backend.py
+93 -9
View File
@@ -6,6 +6,7 @@ Run with:
python3 -m unittest e2e_response_api.backends.test_grpc_backend.TestGrpcBackend
"""
import json
import sys
import unittest
from pathlib import Path
@@ -18,11 +19,12 @@ sys.path.insert(0, str(_TEST_DIR))
from mixins.function_call import FunctionCallingBaseTest
from mixins.mcp import MCPTests
from mixins.state_management import StateManagementTests
from mixins.structured_output import StructuredOutputBaseTest
from router_fixtures import popen_launch_workers_and_router
from util import kill_process_tree
class TestGrpcBackend(StateManagementTests, MCPTests):
class TestGrpcBackend(StateManagementTests, MCPTests, StructuredOutputBaseTest):
"""End to end tests for gRPC backend (Regular backend with Llama)."""
@classmethod
@@ -37,7 +39,15 @@ class TestGrpcBackend(StateManagementTests, MCPTests):
num_workers=1,
tp_size=2,
policy="round_robin",
router_args=["--history-backend", "memory", "--tool-call-parser", "llama"],
worker_args=[
"--context-length=1000",
],
router_args=[
"--history-backend",
"memory",
"--tool-call-parser",
"llama",
],
)
cls.base_url = cls.cluster["base_url"]
@@ -62,14 +72,82 @@ class TestGrpcBackend(StateManagementTests, MCPTests):
def test_mcp_basic_tool_call_streaming(self):
return super().test_mcp_basic_tool_call_streaming()
# Inherited from MCPTests:
# - test_mcp_basic_tool_call
# - test_mcp_basic_tool_call_streaming
# - test_mixed_mcp_and_function_tools (requires external MCP server)
# - test_mixed_mcp_and_function_tools_streaming (requires external MCP server)
def test_structured_output_json_schema(self):
"""Override with simpler schema for Llama model (complex schemas not well supported)."""
data = {
"model": self.model,
"input": [
{
"role": "system",
"content": "You are a math solver. Return ONLY a JSON object that matches the schema—no extra text.",
},
{
"role": "user",
"content": "What is 1 + 1?",
},
],
"text": {
"format": {
"type": "json_schema",
"name": "math_answer",
"schema": {
"type": "object",
"properties": {"answer": {"type": "string"}},
"required": ["answer"],
},
}
},
}
create_resp = self.make_request("/v1/responses", "POST", data)
self.assertEqual(create_resp.status_code, 200)
create_data = create_resp.json()
self.assertIn("id", create_data)
self.assertIn("output", create_data)
self.assertIn("text", create_data)
# Verify text format was echoed back correctly
self.assertIn("format", create_data["text"])
self.assertEqual(create_data["text"]["format"]["type"], "json_schema")
self.assertEqual(create_data["text"]["format"]["name"], "math_answer")
self.assertIn("schema", create_data["text"]["format"])
# Find the message output
output_text = next(
(
content.get("text", "")
for item in create_data.get("output", [])
if item.get("type") == "message"
for content in item.get("content", [])
if content.get("type") == "output_text"
),
None,
)
self.assertIsNotNone(output_text, "No output_text found in response")
self.assertTrue(output_text.strip(), "output_text is empty")
# Parse JSON output
output_json = json.loads(output_text)
# Verify simple schema structure (just answer field)
self.assertIn("answer", output_json)
self.assertIsInstance(output_json["answer"], str)
self.assertTrue(output_json["answer"], "Answer is empty")
@unittest.skip("TODO: Temporary skip since deepwiki might hit rate limit")
def test_mcp_basic_tool_call(self):
return super().test_mcp_basic_tool_call()
@unittest.skip("Temporary skip since deepwiki might hit rate limit")
def test_mcp_basic_tool_call_streaming(self):
return super().test_mcp_basic_tool_call_streaming()
class TestGrpcHarmonyBackend(StateManagementTests, MCPTests, FunctionCallingBaseTest):
class TestGrpcHarmonyBackend(
StateManagementTests, MCPTests, FunctionCallingBaseTest, StructuredOutputBaseTest
):
"""End to end tests for Harmony backend."""
@classmethod
@@ -84,7 +162,13 @@ class TestGrpcHarmonyBackend(StateManagementTests, MCPTests, FunctionCallingBase
num_workers=1,
tp_size=2,
policy="round_robin",
router_args=["--history-backend", "memory"],
worker_args=[
"--reasoning-parser=gpt-oss",
],
router_args=[
"--history-backend",
"memory",
],
)
cls.base_url = cls.cluster["base_url"]
sgl-router/py_test/e2e_response_api/backends/test_http_backend.py
+10
View File
@@ -22,6 +22,7 @@ from mixins.basic_crud import ConversationCRUDBaseTest, ResponseCRUDBaseTest
from mixins.function_call import FunctionCallingBaseTest
from mixins.mcp import MCPTests
from mixins.state_management import StateManagementTests
from mixins.structured_output import StructuredOutputBaseTest
from router_fixtures import popen_launch_openai_xai_router
from util import kill_process_tree
@@ -32,6 +33,7 @@ class TestOpenaiBackend(
StateManagementTests,
MCPTests,
FunctionCallingBaseTest,
StructuredOutputBaseTest,
):
"""End to end tests for OpenAI backend."""
@@ -73,6 +75,14 @@ class TestOpenaiBackend(
def test_mixed_mcp_and_function_tools_streaming(self):
super().test_mixed_mcp_and_function_tools_streaming()
@unittest.skip("Temporary skip since deepwiki might hit rate limit")
def test_mcp_basic_tool_call(self):
super().test_mcp_basic_tool_call()
@unittest.skip("Temporary skip since deepwiki might hit rate limit")
def test_mcp_basic_tool_call_streaming(self):
super().test_mcp_basic_tool_call_streaming()
class TestXaiBackend(StateManagementTests):
"""End to end tests for XAI backend."""
sgl-router/py_test/e2e_response_api/conftest.py
+2
View File
@@ -19,6 +19,7 @@ def pytest_collection_modifyitems(config, items):
- MCPTests
- StateManagementTests
- FunctionCallingBaseTest
- StructuredOutputBaseTest
"""
base_class_names = {
"StateManagementBaseTest",
@@ -27,6 +28,7 @@ def pytest_collection_modifyitems(config, items):
"MCPTests",
"StateManagementTests",
"FunctionCallingBaseTest",
"StructuredOutputBaseTest",
}
# Filter out tests from base classes
sgl-router/py_test/e2e_response_api/mixins/structured_output.py
+129
View File
@@ -0,0 +1,129 @@
"""
Structured output tests for Response API.
Tests for text.format field with json_object and json_schema formats.
"""
import json
import sys
from pathlib import Path
# Add current directory for local imports
_TEST_DIR = Path(__file__).parent
sys.path.insert(0, str(_TEST_DIR))
from util import CustomTestCase
class StructuredOutputBaseTest(CustomTestCase):
"""Base class for structured output tests with common utilities."""
# To be set by subclasses
base_url: str = None
api_key: str = None
model: str = None
def make_request(self, endpoint, method="GET", data=None):
"""Make HTTP request to the API."""
url = f"{self.base_url}{endpoint}"
headers = {
"Content-Type": "application/json",
"Authorization": f"Bearer {self.api_key}",
}
if method == "GET":
response = self.session.get(url, headers=headers)
elif method == "POST":
response = self.session.post(url, headers=headers, json=data)
elif method == "DELETE":
response = self.session.delete(url, headers=headers)
else:
raise ValueError(f"Unsupported method: {method}")
return response
def test_structured_output_json_schema(self):
"""Test structured output with json_schema format."""
# Create response with structured output
data = {
"model": self.model,
"input": [
{
"role": "system",
"content": "You are a helpful math tutor. Guide the user through the solution step by step.",
},
{"role": "user", "content": "how can I solve 8x + 7 = -23"},
],
"text": {
"format": {
"type": "json_schema",
"name": "math_reasoning",
"schema": {
"type": "object",
"properties": {
"steps": {
"type": "array",
"items": {
"type": "object",
"properties": {
"explanation": {"type": "string"},
"output": {"type": "string"},
},
"required": ["explanation", "output"],
"additionalProperties": False,
},
},
"final_answer": {"type": "string"},
},
"required": ["steps", "final_answer"],
"additionalProperties": False,
},
"strict": True,
}
},
}
create_resp = self.make_request("/v1/responses", "POST", data)
self.assertEqual(create_resp.status_code, 200)
create_data = create_resp.json()
self.assertIn("id", create_data)
self.assertIn("output", create_data)
self.assertIn("text", create_data)
# Verify text format was echoed back correctly
self.assertIn("format", create_data["text"])
self.assertEqual(create_data["text"]["format"]["type"], "json_schema")
self.assertEqual(create_data["text"]["format"]["name"], "math_reasoning")
self.assertIn("schema", create_data["text"]["format"])
self.assertEqual(create_data["text"]["format"]["strict"], True)
# Find the message output (output[0] may be reasoning, output[1] is message)
output_text = next(
(
content.get("text", "")
for item in create_data.get("output", [])
if item.get("type") == "message"
for content in item.get("content", [])
if content.get("type") == "output_text"
),
None,
)
self.assertIsNotNone(output_text, "No output_text found in response")
self.assertTrue(output_text.strip(), "output_text is empty")
# Parse JSON output
output_json = json.loads(output_text)
# Verify schema structure
self.assertIn("steps", output_json)
self.assertIn("final_answer", output_json)
self.assertIsInstance(output_json["steps"], list)
self.assertGreater(len(output_json["steps"]), 0)
# Verify each step has required fields
for step in output_json["steps"]:
self.assertIn("explanation", step)
self.assertIn("output", step)
sgl-router/src/grpc_client/sglang_scheduler.rs
+41 -24
View File
@@ -240,7 +240,7 @@ impl SglangSchedulerClient {
}
/// Build a single SGLang GenerateRequest from OpenAI ChatCompletionRequest
pub fn build_generate_request(
pub fn build_generate_request_from_chat(
&self,
request_id: String,
body: &ChatCompletionRequest,
@@ -250,7 +250,8 @@ impl SglangSchedulerClient {
tool_call_constraint: Option<(String, String)>, // (constraint_type, constraint_value)
) -> Result<proto::GenerateRequest, String> {
// Build sampling params
let sampling_params = self.build_grpc_sampling_params(body, tool_call_constraint)?;
let sampling_params =
self.build_grpc_sampling_params_from_chat(body, tool_call_constraint)?;
let grpc_request = proto::GenerateRequest {
request_id,
@@ -313,11 +314,11 @@ impl SglangSchedulerClient {
processed_text: String,
token_ids: Vec<u32>,
harmony_stop_ids: Option<Vec<u32>>,
tool_call_constraint: Option<(String, String)>,
constraint: Option<(String, String)>,
) -> Result<proto::GenerateRequest, String> {
// Build sampling params from ResponsesRequest
let mut sampling_params =
self.build_grpc_sampling_params_from_responses(body, tool_call_constraint)?;
self.build_grpc_sampling_params_from_responses(body, constraint)?;
// Inject Harmony stop token IDs if provided
if let Some(stop_ids) = harmony_stop_ids {
@@ -343,8 +344,8 @@ impl SglangSchedulerClient {
Ok(grpc_request)
}
/// Build gRPC SamplingParams from OpenAI request
fn build_grpc_sampling_params(
/// Build gRPC SamplingParams from ChatCompletionRequest
fn build_grpc_sampling_params_from_chat(
&self,
request: &ChatCompletionRequest,
tool_call_constraint: Option<(String, String)>,
@@ -380,7 +381,7 @@ impl SglangSchedulerClient {
ignore_eos: request.ignore_eos,
no_stop_trim: request.no_stop_trim,
n: request.n.unwrap_or(1) as i32,
constraint: self.build_constraint(request, tool_call_constraint)?,
constraint: self.build_constraint_for_chat(request, tool_call_constraint)?,
..Default::default()
})
}
@@ -395,17 +396,30 @@ impl SglangSchedulerClient {
}
/// Build constraint for structured generation
fn build_constraint(
fn build_constraint_for_chat(
&self,
request: &ChatCompletionRequest,
tool_call_constraint: Option<(String, String)>,
) -> Result<Option<proto::sampling_params::Constraint>, String> {
let mut constraints = Vec::new();
if let Some(ResponseFormat::JsonSchema { json_schema }) = &request.response_format {
let schema_str = serde_json::to_string(&json_schema.schema)
.map_err(|e| format!("Failed to serialize JSON schema: {}", e))?;
constraints.push(proto::sampling_params::Constraint::JsonSchema(schema_str));
// Handle response_format constraints
match &request.response_format {
Some(ResponseFormat::JsonObject) => {
// json_object mode - constrain to valid JSON object
let schema = serde_json::json!({"type": "object"});
let schema_str = serde_json::to_string(&schema)
.map_err(|e| format!("Failed to serialize JSON schema: {}", e))?;
constraints.push(proto::sampling_params::Constraint::JsonSchema(schema_str));
}
Some(ResponseFormat::JsonSchema { json_schema }) => {
let schema_str = serde_json::to_string(&json_schema.schema)
.map_err(|e| format!("Failed to serialize JSON schema: {}", e))?;
constraints.push(proto::sampling_params::Constraint::JsonSchema(schema_str));
}
Some(ResponseFormat::Text) | None => {
// No constraint for text format
}
}
if let Some(ebnf) = &request.ebnf {
@@ -418,7 +432,7 @@ impl SglangSchedulerClient {
constraints.push(proto::sampling_params::Constraint::Regex(regex.clone()));
}
// Handle tool call constraint
// Handle tool call constraint from preparation stage
if let Some((constraint_type, constraint_value)) = tool_call_constraint {
if !constraints.is_empty() {
return Err("Constrained decoding is not compatible with tool calls.".to_string());
@@ -446,10 +460,10 @@ impl SglangSchedulerClient {
fn build_grpc_sampling_params_from_responses(
&self,
request: &ResponsesRequest,
tool_call_constraint: Option<(String, String)>,
constraint: Option<(String, String)>,
) -> Result<proto::SamplingParams, String> {
// ResponsesRequest doesn't have stop sequences in the same way
// For Harmony router: Tools are handled via structural_tag constraints
// Used by Harmony models only. Regular models use Chat API path.
// Constraints come from Harmony preparation stage (structural_tag) or tool handling.
let max_new_tokens = request.max_output_tokens.map(|v| v as i32);
@@ -469,22 +483,25 @@ impl SglangSchedulerClient {
ignore_eos: false,
no_stop_trim: false,
n: 1, // Responses API doesn't support n>1
constraint: self.build_constraint_for_responses(tool_call_constraint)?,
constraint: self.build_constraint_for_responses(constraint)?,
..Default::default()
})
}
/// Build constraint for Responses API (simpler than Chat API's build_constraint)
/// Build constraint for Responses API
///
/// Responses API doesn't support response_format, ebnf, or regex constraints,
/// so this only handles tool_call_constraint.
/// Handles constraints from Harmony preparation stage (structural_tag for Harmony models,
/// structured output via text field, or tool call constraints).
///
/// Note: Regular gRPC models use Chat API path with response_format, not this function.
fn build_constraint_for_responses(
&self,
tool_call_constraint: Option<(String, String)>,
constraint: Option<(String, String)>,
) -> Result<Option<proto::sampling_params::Constraint>, String> {
if let Some((constraint_type, constraint_value)) = tool_call_constraint {
let tool_constraint = match constraint_type.as_str() {
if let Some((constraint_type, constraint_value)) = constraint {
let parsed_constraint = match constraint_type.as_str() {
"structural_tag" => {
// Harmony models: structural tag from preparation stage
proto::sampling_params::Constraint::StructuralTag(constraint_value)
}
"json_schema" => proto::sampling_params::Constraint::JsonSchema(constraint_value),
@@ -492,7 +509,7 @@ impl SglangSchedulerClient {
"regex" => proto::sampling_params::Constraint::Regex(constraint_value),
_ => return Err(format!("Unknown constraint type: {}", constraint_type)),
};
Ok(Some(tool_constraint))
Ok(Some(parsed_constraint))
} else {
Ok(None)
}
sgl-router/src/protocols/responses.rs
+32 -6
View File
@@ -291,15 +291,36 @@ pub struct ReasoningInfo {
pub summary: Option<String>,
}
// ============================================================================
// Text Format (structured outputs)
// ============================================================================
/// Text configuration for structured output requests
#[derive(Debug, Clone, Deserialize, Serialize)]
pub struct ResponseTextFormat {
pub format: TextFormatType,
pub struct TextConfig {
#[serde(skip_serializing_if = "Option::is_none")]
pub format: Option<TextFormat>,
}
/// Text format: text (default), json_object (legacy), or json_schema (recommended)
#[derive(Debug, Clone, Deserialize, Serialize)]
pub struct TextFormatType {
#[serde(rename = "type")]
pub format_type: String,
#[serde(tag = "type")]
pub enum TextFormat {
#[serde(rename = "text")]
Text,
#[serde(rename = "json_object")]
JsonObject,
#[serde(rename = "json_schema")]
JsonSchema {
name: String,
schema: Value,
#[serde(skip_serializing_if = "Option::is_none")]
description: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
strict: Option<bool>,
},
}
#[derive(Debug, Clone, Deserialize, Serialize)]
@@ -539,6 +560,10 @@ pub struct ResponsesRequest {
#[serde(skip_serializing_if = "Option::is_none")]
pub truncation: Option<Truncation>,
/// Text format for structured outputs (text, json_object, json_schema)
#[serde(skip_serializing_if = "Option::is_none")]
pub text: Option<TextConfig>,
/// User identifier
#[serde(skip_serializing_if = "Option::is_none")]
pub user: Option<String>,
@@ -607,6 +632,7 @@ impl Default for ResponsesRequest {
top_logprobs: None,
top_p: None,
truncation: None,
text: None,
user: None,
request_id: None,
priority: 0,
@@ -906,7 +932,7 @@ pub struct ResponsesResponse {
/// Text format settings
#[serde(skip_serializing_if = "Option::is_none")]
pub text: Option<ResponseTextFormat>,
pub text: Option<TextConfig>,
/// Tool choice setting
#[serde(default = "default_tool_choice")]
sgl-router/src/routers/grpc/harmony/processor.rs
+1 -1
View File
@@ -269,7 +269,7 @@ impl HarmonyResponseProcessor {
reasoning: None, // Set by caller if needed
store: responses_request.store.unwrap_or(true),
temperature: responses_request.temperature,
text: None,
text: responses_request.text.clone(),
tool_choice: responses_request
.tool_choice
.as_ref()
sgl-router/src/routers/grpc/harmony/stages/preparation.rs
+116 -13
View File
@@ -88,7 +88,7 @@ impl HarmonyPreparationStage {
// Step 2: Build tool constraints
let tool_constraints = if let Some(tools) = body_ref.tools.as_ref() {
Self::generate_harmony_structural_tag(tools, &body_ref.tool_choice).map_err(|e| *e)?
Self::generate_tool_call_constraint(tools, &body_ref.tool_choice).map_err(|e| *e)?
} else {
None
};
@@ -139,26 +139,40 @@ impl HarmonyPreparationStage {
function_tools = filtered;
}
// Step 3: Generate Harmony structural tags from filtered tools
let tool_constraints = if !function_tools.is_empty() {
Self::generate_harmony_structural_tag(&function_tools, &request.tool_choice)
// Step 3: Generate Harmony structural tags
let tool_constraint = if !function_tools.is_empty() {
Self::generate_tool_call_constraint(&function_tools, &request.tool_choice)
.map_err(|e| *e)?
} else {
None
};
let text_constraint = if let Some(text_config) = &request.text {
Self::generate_text_format_constraint(text_config).map_err(|e| *e)?
} else {
None
};
if tool_constraint.is_some() && text_constraint.is_some() {
return Err(error::bad_request(
"Cannot use both tool_choice (required/function) and text format (json_object/json_schema) simultaneously".to_string(),
));
}
let constraint = tool_constraint.or(text_constraint);
// Step 3: Build via Harmony from responses API request
let build_output = self
.builder
.build_from_responses(request)
.map_err(|e| error::bad_request(format!("Harmony build failed: {}", e)))?;
// Step 4: Store results with tool_constraints
// Step 4: Store results with constraint
ctx.state.preparation = Some(PreparationOutput {
original_text: None,
token_ids: build_output.input_ids,
processed_messages: None,
tool_constraints,
tool_constraints: constraint,
filtered_request: None,
harmony_mode: true,
selection_text: Some(build_output.selection_text),
@@ -169,11 +183,39 @@ impl HarmonyPreparationStage {
Ok(None)
}
/// Generate Harmony structural tag for structured output (text field)
///
/// Converts text.format to structural tag that constrains the final channel.
/// Returns None if text.format is not specified or is "text".
fn generate_text_format_constraint(
text_config: &crate::protocols::responses::TextConfig,
) -> Result<Option<(String, String)>, Box<Response>> {
use crate::protocols::responses::TextFormat;
let Some(format) = &text_config.format else {
return Ok(None);
};
match format {
TextFormat::Text => Ok(None),
TextFormat::JsonObject => {
let tag = build_text_format_structural_tag(&serde_json::json!({"type": "object"}))
.map_err(|e| Box::new(error::internal_error(e)))?;
Ok(Some(("structural_tag".to_string(), tag)))
}
TextFormat::JsonSchema { schema, .. } => {
let tag = build_text_format_structural_tag(schema)
.map_err(|e| Box::new(error::internal_error(e)))?;
Ok(Some(("structural_tag".to_string(), tag)))
}
}
}
/// Generate Harmony structural tag for tool constraints
///
/// Uses structural tags with `triggered_tags` format to force Harmony format output.
/// This ensures the model outputs in Harmony format (with channels) even when constrained.
fn generate_harmony_structural_tag(
fn generate_tool_call_constraint(
tools: &[Tool],
tool_choice: &Option<ToolChoice>,
) -> Result<Option<(String, String)>, Box<Response>> {
@@ -183,16 +225,16 @@ impl HarmonyPreparationStage {
match choice {
ToolChoice::Function { function, .. } => {
let tag = Self::build_harmony_structural_tag(tools, Some(&function.name))?;
let tag = Self::build_tool_call_structural_tag(tools, Some(&function.name))?;
Ok(Some(("structural_tag".to_string(), tag)))
}
ToolChoice::Value(ToolChoiceValue::Required) => {
let tag = Self::build_harmony_structural_tag(tools, None)?;
let tag = Self::build_tool_call_structural_tag(tools, None)?;
Ok(Some(("structural_tag".to_string(), tag)))
}
ToolChoice::AllowedTools { mode, .. } => {
if mode == "required" {
let tag = Self::build_harmony_structural_tag(tools, None)?;
let tag = Self::build_tool_call_structural_tag(tools, None)?;
Ok(Some(("structural_tag".to_string(), tag)))
} else {
Ok(None)
@@ -203,7 +245,11 @@ impl HarmonyPreparationStage {
}
/// Build Harmony structural tag for tool calling constraints
fn build_harmony_structural_tag(
///
/// Supports both reasoning-enabled and reasoning-disabled modes:
/// - With reasoning: triggers on `<|start|>assistant<|channel|>commentary` (waits for analysis)
/// - Without reasoning: triggers on `<|channel|>commentary` (goes directly to commentary)
fn build_tool_call_structural_tag(
tools: &[Tool],
specific_function: Option<&str>,
) -> Result<String, Box<Response>> {
@@ -227,11 +273,12 @@ impl HarmonyPreparationStage {
))));
}
// Build tags for each tool
// Build tags for each tool - need two patterns per tool for reasoning on/off
for tool in tools_to_use {
let tool_name = &tool.function.name;
let params_schema = &tool.function.parameters;
// Pattern 1: For reasoning-enabled mode (with analysis channel before commentary)
tags.push(json!({
"begin": format!("<|start|>assistant<|channel|>commentary to=functions.{}<|constrain|>json<|message|>", tool_name),
"content": {
@@ -240,6 +287,16 @@ impl HarmonyPreparationStage {
},
"end": "" // `end` is empty because <|call|> comes naturally from Harmony stop tokens
}));
// Pattern 2: For reasoning-disabled mode (goes directly to commentary channel)
tags.push(json!({
"begin": format!("<|channel|>commentary to=functions.{}<|constrain|>json<|message|>", tool_name),
"content": {
"type": "json_schema",
"json_schema": params_schema
},
"end": ""
}));
}
let stop_after_first = specific_function.is_some();
@@ -247,7 +304,7 @@ impl HarmonyPreparationStage {
let structural_tag = json!({
"format": {
"type": "triggered_tags",
"triggers": ["<|start|>assistant"],
"triggers": ["<|start|>assistant<|channel|>commentary", "<|channel|>commentary"],
"tags": tags,
"at_least_one": true,
"stop_after_first": stop_after_first
@@ -262,3 +319,49 @@ impl HarmonyPreparationStage {
})
}
}
/// Build Harmony structural tag for structured output (JSON schema constraint)
///
/// Creates a structural tag that applies JSON schema constraint to the final channel,
/// supporting both reasoning-enabled and reasoning-disabled modes:
/// - With reasoning: triggers on `<|start|>assistant<|channel|>final` (waits for analysis to complete)
/// - Without reasoning: triggers on `<|channel|>final` (goes directly to final channel)
///
/// This is used for the Responses API text.format field (json_object or json_schema).
pub fn build_text_format_structural_tag(schema: &serde_json::Value) -> Result<String, String> {
let structural_tag = json!({
"format": {
"type": "triggered_tags",
"triggers": ["<|start|>assistant<|channel|>final", "<|channel|>final"],
"tags": [
{
// Pattern 1: For reasoning-enabled mode (with analysis channel before final)
"begin": "<|start|>assistant<|channel|>final<|constrain|>json<|message|>",
"content": {
"type": "json_schema",
"json_schema": schema
},
"end": ""
},
{
// Pattern 2: For reasoning-disabled mode (goes directly to final channel)
"begin": "<|channel|>final<|constrain|>json<|message|>",
"content": {
"type": "json_schema",
"json_schema": schema
},
"end": ""
}
],
"at_least_one": true,
"stop_after_first": true
}
});
serde_json::to_string(&structural_tag).map_err(|e| {
format!(
"Failed to serialize structural tag for structured output: {}",
e
)
})
}
sgl-router/src/routers/grpc/harmony/stages/request_building.rs
+1 -1
View File
@@ -67,7 +67,7 @@ impl PipelineStage for HarmonyRequestBuildingStage {
let body = prep.filtered_request.as_ref().unwrap_or(request.as_ref());
builder_client
.build_generate_request(
.build_generate_request_from_chat(
request_id,
body,
placeholder_processed_text,
sgl-router/src/routers/grpc/regular/responses/conversions.rs
+33 -3
View File
@@ -10,11 +10,14 @@
use crate::{
protocols::{
chat::{ChatCompletionRequest, ChatCompletionResponse, ChatMessage, UserMessageContent},
common::{FunctionCallResponse, StreamOptions, ToolCall, ToolChoice, UsageInfo},
common::{
FunctionCallResponse, JsonSchemaFormat, ResponseFormat, StreamOptions, ToolCall,
ToolChoice, UsageInfo,
},
responses::{
ResponseContentPart, ResponseInput, ResponseInputOutputItem, ResponseOutputItem,
ResponseReasoningContent::ReasoningText, ResponseStatus, ResponsesRequest,
ResponsesResponse, ResponsesUsage, StringOrContentParts,
ResponsesResponse, ResponsesUsage, StringOrContentParts, TextConfig, TextFormat,
},
},
routers::grpc::common::responses::utils::extract_tools_from_response_tools,
@@ -188,6 +191,7 @@ pub fn responses_to_chat(req: &ResponsesRequest) -> Result<ChatCompletionRequest
skip_special_tokens: true,
tools,
tool_choice: req.tool_choice.clone(),
response_format: map_text_to_response_format(&req.text),
..Default::default()
})
}
@@ -232,6 +236,32 @@ fn role_to_chat_message(role: &str, text: String) -> ChatMessage {
}
}
/// Map TextConfig from Responses API to ResponseFormat for Chat API
///
/// Converts the structured output configuration from the Responses API format
/// to the Chat API format for non-Harmony models.
fn map_text_to_response_format(text: &Option<TextConfig>) -> Option<ResponseFormat> {
let text_config = text.as_ref()?;
let format = text_config.format.as_ref()?;
match format {
TextFormat::Text => Some(ResponseFormat::Text),
TextFormat::JsonObject => Some(ResponseFormat::JsonObject),
TextFormat::JsonSchema {
name,
schema,
description: _,
strict,
} => Some(ResponseFormat::JsonSchema {
json_schema: JsonSchemaFormat {
name: name.clone(),
schema: schema.clone(),
strict: *strict,
},
}),
}
}
/// Convert a ChatCompletionResponse to ResponsesResponse
///
/// # Conversion Logic
@@ -338,7 +368,7 @@ pub fn chat_to_responses(
reasoning: None, // TODO: Map reasoning effort if needed
store: original_req.store.unwrap_or(true),
temperature: original_req.temperature,
text: None,
text: original_req.text.clone(),
tool_choice: ToolChoice::serialize_to_string(&original_req.tool_choice),
tools: original_req.tools.clone().unwrap_or_default(),
top_p: original_req.top_p,
sgl-router/src/routers/grpc/regular/responses/tool_loop.rs
+2
View File
@@ -427,6 +427,7 @@ pub(super) async fn execute_tool_loop(
service_tier: current_request.service_tier.clone(),
top_logprobs: current_request.top_logprobs,
truncation: current_request.truncation.clone(),
text: current_request.text.clone(),
request_id: None,
priority: current_request.priority,
frequency_penalty: current_request.frequency_penalty,
@@ -971,6 +972,7 @@ async fn execute_tool_loop_streaming_internal(
service_tier: current_request.service_tier.clone(),
top_logprobs: current_request.top_logprobs,
truncation: current_request.truncation.clone(),
text: current_request.text.clone(),
request_id: None,
priority: current_request.priority,
frequency_penalty: current_request.frequency_penalty,
sgl-router/src/routers/grpc/regular/stages/chat/request_building.rs
+1 -1
View File
@@ -51,7 +51,7 @@ impl PipelineStage for ChatRequestBuildingStage {
let body_ref = prep.filtered_request.as_ref().unwrap_or(&chat_request);
let mut proto_request = builder_client
.build_generate_request(
.build_generate_request_from_chat(
request_id,
body_ref,
prep.processed_messages.as_ref().unwrap().text.clone(),
sgl-router/tests/responses_api_test.rs
+8
View File
@@ -91,6 +91,7 @@ async fn test_non_streaming_mcp_minimal_e2e_with_persistence() {
top_logprobs: Some(0),
top_p: None,
truncation: Some(Truncation::Disabled),
text: None,
user: None,
request_id: Some("resp_test_mcp_e2e".to_string()),
priority: 0,
@@ -312,6 +313,7 @@ fn test_responses_request_creation() {
top_logprobs: Some(5),
top_p: Some(0.9),
truncation: Some(Truncation::Disabled),
text: None,
user: Some("test-user".to_string()),
request_id: Some("resp_test123".to_string()),
priority: 0,
@@ -354,6 +356,7 @@ fn test_responses_request_sglang_extensions() {
top_logprobs: Some(0),
top_p: Some(0.95),
truncation: Some(Truncation::Auto),
text: None,
user: None,
request_id: Some("resp_test456".to_string()),
priority: 0,
@@ -469,6 +472,7 @@ fn test_json_serialization() {
top_logprobs: Some(10),
top_p: Some(0.8),
truncation: Some(Truncation::Auto),
text: None,
user: Some("test_user".to_string()),
request_id: Some("resp_comprehensive_test".to_string()),
priority: 1,
@@ -574,6 +578,7 @@ async fn test_multi_turn_loop_with_mcp() {
top_logprobs: Some(0),
top_p: Some(1.0),
truncation: Some(Truncation::Disabled),
text: None,
user: None,
request_id: Some("resp_multi_turn_test".to_string()),
priority: 0,
@@ -722,6 +727,7 @@ async fn test_max_tool_calls_limit() {
top_logprobs: Some(0),
top_p: Some(1.0),
truncation: Some(Truncation::Disabled),
text: None,
user: None,
request_id: Some("resp_max_calls_test".to_string()),
priority: 0,
@@ -893,6 +899,7 @@ async fn test_streaming_with_mcp_tool_calls() {
top_logprobs: Some(0),
top_p: Some(1.0),
truncation: Some(Truncation::Disabled),
text: None,
user: None,
request_id: Some("resp_streaming_mcp_test".to_string()),
priority: 0,
@@ -1172,6 +1179,7 @@ async fn test_streaming_multi_turn_with_mcp() {
top_logprobs: Some(0),
top_p: Some(1.0),
truncation: Some(Truncation::Disabled),
text: None,
user: None,
request_id: Some("resp_streaming_multiturn_test".to_string()),
priority: 0,