From 5c9273c0323b84593319a26c654ca233ffa76733 Mon Sep 17 00:00:00 2001 From: Keyang Ru Date: Fri, 7 Nov 2025 10:16:45 -0800 Subject: [PATCH] [router] Support structured model output for openai and grpc router (#12431) --- .../backends/test_grpc_backend.py | 102 ++++++++++++-- .../backends/test_http_backend.py | 10 ++ .../py_test/e2e_response_api/conftest.py | 2 + .../mixins/structured_output.py | 129 ++++++++++++++++++ .../src/grpc_client/sglang_scheduler.rs | 65 +++++---- sgl-router/src/protocols/responses.rs | 38 +++++- .../src/routers/grpc/harmony/processor.rs | 2 +- .../grpc/harmony/stages/preparation.rs | 129 ++++++++++++++++-- .../grpc/harmony/stages/request_building.rs | 2 +- .../grpc/regular/responses/conversions.rs | 36 ++++- .../grpc/regular/responses/tool_loop.rs | 2 + .../regular/stages/chat/request_building.rs | 2 +- sgl-router/tests/responses_api_test.rs | 8 ++ 13 files changed, 469 insertions(+), 58 deletions(-) create mode 100644 sgl-router/py_test/e2e_response_api/mixins/structured_output.py diff --git a/sgl-router/py_test/e2e_response_api/backends/test_grpc_backend.py b/sgl-router/py_test/e2e_response_api/backends/test_grpc_backend.py index c75ce6cc3..80b952469 100644 --- a/sgl-router/py_test/e2e_response_api/backends/test_grpc_backend.py +++ b/sgl-router/py_test/e2e_response_api/backends/test_grpc_backend.py @@ -6,6 +6,7 @@ Run with: python3 -m unittest e2e_response_api.backends.test_grpc_backend.TestGrpcBackend """ +import json import sys import unittest from pathlib import Path @@ -18,11 +19,12 @@ sys.path.insert(0, str(_TEST_DIR)) from mixins.function_call import FunctionCallingBaseTest from mixins.mcp import MCPTests from mixins.state_management import StateManagementTests +from mixins.structured_output import StructuredOutputBaseTest from router_fixtures import popen_launch_workers_and_router from util import kill_process_tree -class TestGrpcBackend(StateManagementTests, MCPTests): +class TestGrpcBackend(StateManagementTests, MCPTests, StructuredOutputBaseTest): """End to end tests for gRPC backend (Regular backend with Llama).""" @classmethod @@ -37,7 +39,15 @@ class TestGrpcBackend(StateManagementTests, MCPTests): num_workers=1, tp_size=2, policy="round_robin", - router_args=["--history-backend", "memory", "--tool-call-parser", "llama"], + worker_args=[ + "--context-length=1000", + ], + router_args=[ + "--history-backend", + "memory", + "--tool-call-parser", + "llama", + ], ) cls.base_url = cls.cluster["base_url"] @@ -62,14 +72,82 @@ class TestGrpcBackend(StateManagementTests, MCPTests): def test_mcp_basic_tool_call_streaming(self): return super().test_mcp_basic_tool_call_streaming() - # Inherited from MCPTests: - # - test_mcp_basic_tool_call - # - test_mcp_basic_tool_call_streaming - # - test_mixed_mcp_and_function_tools (requires external MCP server) - # - test_mixed_mcp_and_function_tools_streaming (requires external MCP server) + def test_structured_output_json_schema(self): + """Override with simpler schema for Llama model (complex schemas not well supported).""" + data = { + "model": self.model, + "input": [ + { + "role": "system", + "content": "You are a math solver. Return ONLY a JSON object that matches the schema—no extra text.", + }, + { + "role": "user", + "content": "What is 1 + 1?", + }, + ], + "text": { + "format": { + "type": "json_schema", + "name": "math_answer", + "schema": { + "type": "object", + "properties": {"answer": {"type": "string"}}, + "required": ["answer"], + }, + } + }, + } + + create_resp = self.make_request("/v1/responses", "POST", data) + self.assertEqual(create_resp.status_code, 200) + + create_data = create_resp.json() + self.assertIn("id", create_data) + self.assertIn("output", create_data) + self.assertIn("text", create_data) + + # Verify text format was echoed back correctly + self.assertIn("format", create_data["text"]) + self.assertEqual(create_data["text"]["format"]["type"], "json_schema") + self.assertEqual(create_data["text"]["format"]["name"], "math_answer") + self.assertIn("schema", create_data["text"]["format"]) + + # Find the message output + output_text = next( + ( + content.get("text", "") + for item in create_data.get("output", []) + if item.get("type") == "message" + for content in item.get("content", []) + if content.get("type") == "output_text" + ), + None, + ) + + self.assertIsNotNone(output_text, "No output_text found in response") + self.assertTrue(output_text.strip(), "output_text is empty") + + # Parse JSON output + output_json = json.loads(output_text) + + # Verify simple schema structure (just answer field) + self.assertIn("answer", output_json) + self.assertIsInstance(output_json["answer"], str) + self.assertTrue(output_json["answer"], "Answer is empty") + + @unittest.skip("TODO: Temporary skip since deepwiki might hit rate limit") + def test_mcp_basic_tool_call(self): + return super().test_mcp_basic_tool_call() + + @unittest.skip("Temporary skip since deepwiki might hit rate limit") + def test_mcp_basic_tool_call_streaming(self): + return super().test_mcp_basic_tool_call_streaming() -class TestGrpcHarmonyBackend(StateManagementTests, MCPTests, FunctionCallingBaseTest): +class TestGrpcHarmonyBackend( + StateManagementTests, MCPTests, FunctionCallingBaseTest, StructuredOutputBaseTest +): """End to end tests for Harmony backend.""" @classmethod @@ -84,7 +162,13 @@ class TestGrpcHarmonyBackend(StateManagementTests, MCPTests, FunctionCallingBase num_workers=1, tp_size=2, policy="round_robin", - router_args=["--history-backend", "memory"], + worker_args=[ + "--reasoning-parser=gpt-oss", + ], + router_args=[ + "--history-backend", + "memory", + ], ) cls.base_url = cls.cluster["base_url"] diff --git a/sgl-router/py_test/e2e_response_api/backends/test_http_backend.py b/sgl-router/py_test/e2e_response_api/backends/test_http_backend.py index 5238c71ef..3f97fe915 100644 --- a/sgl-router/py_test/e2e_response_api/backends/test_http_backend.py +++ b/sgl-router/py_test/e2e_response_api/backends/test_http_backend.py @@ -22,6 +22,7 @@ from mixins.basic_crud import ConversationCRUDBaseTest, ResponseCRUDBaseTest from mixins.function_call import FunctionCallingBaseTest from mixins.mcp import MCPTests from mixins.state_management import StateManagementTests +from mixins.structured_output import StructuredOutputBaseTest from router_fixtures import popen_launch_openai_xai_router from util import kill_process_tree @@ -32,6 +33,7 @@ class TestOpenaiBackend( StateManagementTests, MCPTests, FunctionCallingBaseTest, + StructuredOutputBaseTest, ): """End to end tests for OpenAI backend.""" @@ -73,6 +75,14 @@ class TestOpenaiBackend( def test_mixed_mcp_and_function_tools_streaming(self): super().test_mixed_mcp_and_function_tools_streaming() + @unittest.skip("Temporary skip since deepwiki might hit rate limit") + def test_mcp_basic_tool_call(self): + super().test_mcp_basic_tool_call() + + @unittest.skip("Temporary skip since deepwiki might hit rate limit") + def test_mcp_basic_tool_call_streaming(self): + super().test_mcp_basic_tool_call_streaming() + class TestXaiBackend(StateManagementTests): """End to end tests for XAI backend.""" diff --git a/sgl-router/py_test/e2e_response_api/conftest.py b/sgl-router/py_test/e2e_response_api/conftest.py index 2a53ca1ab..19c082a42 100644 --- a/sgl-router/py_test/e2e_response_api/conftest.py +++ b/sgl-router/py_test/e2e_response_api/conftest.py @@ -19,6 +19,7 @@ def pytest_collection_modifyitems(config, items): - MCPTests - StateManagementTests - FunctionCallingBaseTest + - StructuredOutputBaseTest """ base_class_names = { "StateManagementBaseTest", @@ -27,6 +28,7 @@ def pytest_collection_modifyitems(config, items): "MCPTests", "StateManagementTests", "FunctionCallingBaseTest", + "StructuredOutputBaseTest", } # Filter out tests from base classes diff --git a/sgl-router/py_test/e2e_response_api/mixins/structured_output.py b/sgl-router/py_test/e2e_response_api/mixins/structured_output.py new file mode 100644 index 000000000..6d2fbdb95 --- /dev/null +++ b/sgl-router/py_test/e2e_response_api/mixins/structured_output.py @@ -0,0 +1,129 @@ +""" +Structured output tests for Response API. + +Tests for text.format field with json_object and json_schema formats. +""" + +import json +import sys +from pathlib import Path + +# Add current directory for local imports +_TEST_DIR = Path(__file__).parent +sys.path.insert(0, str(_TEST_DIR)) + +from util import CustomTestCase + + +class StructuredOutputBaseTest(CustomTestCase): + """Base class for structured output tests with common utilities.""" + + # To be set by subclasses + base_url: str = None + api_key: str = None + model: str = None + + def make_request(self, endpoint, method="GET", data=None): + """Make HTTP request to the API.""" + url = f"{self.base_url}{endpoint}" + headers = { + "Content-Type": "application/json", + "Authorization": f"Bearer {self.api_key}", + } + + if method == "GET": + response = self.session.get(url, headers=headers) + elif method == "POST": + response = self.session.post(url, headers=headers, json=data) + elif method == "DELETE": + response = self.session.delete(url, headers=headers) + else: + raise ValueError(f"Unsupported method: {method}") + + return response + + def test_structured_output_json_schema(self): + """Test structured output with json_schema format.""" + + # Create response with structured output + data = { + "model": self.model, + "input": [ + { + "role": "system", + "content": "You are a helpful math tutor. Guide the user through the solution step by step.", + }, + {"role": "user", "content": "how can I solve 8x + 7 = -23"}, + ], + "text": { + "format": { + "type": "json_schema", + "name": "math_reasoning", + "schema": { + "type": "object", + "properties": { + "steps": { + "type": "array", + "items": { + "type": "object", + "properties": { + "explanation": {"type": "string"}, + "output": {"type": "string"}, + }, + "required": ["explanation", "output"], + "additionalProperties": False, + }, + }, + "final_answer": {"type": "string"}, + }, + "required": ["steps", "final_answer"], + "additionalProperties": False, + }, + "strict": True, + } + }, + } + + create_resp = self.make_request("/v1/responses", "POST", data) + self.assertEqual(create_resp.status_code, 200) + + create_data = create_resp.json() + self.assertIn("id", create_data) + self.assertIn("output", create_data) + self.assertIn("text", create_data) + + # Verify text format was echoed back correctly + self.assertIn("format", create_data["text"]) + self.assertEqual(create_data["text"]["format"]["type"], "json_schema") + self.assertEqual(create_data["text"]["format"]["name"], "math_reasoning") + self.assertIn("schema", create_data["text"]["format"]) + self.assertEqual(create_data["text"]["format"]["strict"], True) + + # Find the message output (output[0] may be reasoning, output[1] is message) + output_text = next( + ( + content.get("text", "") + for item in create_data.get("output", []) + if item.get("type") == "message" + for content in item.get("content", []) + if content.get("type") == "output_text" + ), + None, + ) + + self.assertIsNotNone(output_text, "No output_text found in response") + self.assertTrue(output_text.strip(), "output_text is empty") + + # Parse JSON output + output_json = json.loads(output_text) + + # Verify schema structure + self.assertIn("steps", output_json) + self.assertIn("final_answer", output_json) + self.assertIsInstance(output_json["steps"], list) + self.assertGreater(len(output_json["steps"]), 0) + + # Verify each step has required fields + for step in output_json["steps"]: + self.assertIn("explanation", step) + self.assertIn("output", step) diff --git a/sgl-router/src/grpc_client/sglang_scheduler.rs b/sgl-router/src/grpc_client/sglang_scheduler.rs index 31195fd88..579f094ea 100644 --- a/sgl-router/src/grpc_client/sglang_scheduler.rs +++ b/sgl-router/src/grpc_client/sglang_scheduler.rs @@ -240,7 +240,7 @@ impl SglangSchedulerClient { } /// Build a single SGLang GenerateRequest from OpenAI ChatCompletionRequest - pub fn build_generate_request( + pub fn build_generate_request_from_chat( &self, request_id: String, body: &ChatCompletionRequest, @@ -250,7 +250,8 @@ impl SglangSchedulerClient { tool_call_constraint: Option<(String, String)>, // (constraint_type, constraint_value) ) -> Result { // Build sampling params - let sampling_params = self.build_grpc_sampling_params(body, tool_call_constraint)?; + let sampling_params = + self.build_grpc_sampling_params_from_chat(body, tool_call_constraint)?; let grpc_request = proto::GenerateRequest { request_id, @@ -313,11 +314,11 @@ impl SglangSchedulerClient { processed_text: String, token_ids: Vec, harmony_stop_ids: Option>, - tool_call_constraint: Option<(String, String)>, + constraint: Option<(String, String)>, ) -> Result { // Build sampling params from ResponsesRequest let mut sampling_params = - self.build_grpc_sampling_params_from_responses(body, tool_call_constraint)?; + self.build_grpc_sampling_params_from_responses(body, constraint)?; // Inject Harmony stop token IDs if provided if let Some(stop_ids) = harmony_stop_ids { @@ -343,8 +344,8 @@ impl SglangSchedulerClient { Ok(grpc_request) } - /// Build gRPC SamplingParams from OpenAI request - fn build_grpc_sampling_params( + /// Build gRPC SamplingParams from ChatCompletionRequest + fn build_grpc_sampling_params_from_chat( &self, request: &ChatCompletionRequest, tool_call_constraint: Option<(String, String)>, @@ -380,7 +381,7 @@ impl SglangSchedulerClient { ignore_eos: request.ignore_eos, no_stop_trim: request.no_stop_trim, n: request.n.unwrap_or(1) as i32, - constraint: self.build_constraint(request, tool_call_constraint)?, + constraint: self.build_constraint_for_chat(request, tool_call_constraint)?, ..Default::default() }) } @@ -395,17 +396,30 @@ impl SglangSchedulerClient { } /// Build constraint for structured generation - fn build_constraint( + fn build_constraint_for_chat( &self, request: &ChatCompletionRequest, tool_call_constraint: Option<(String, String)>, ) -> Result, String> { let mut constraints = Vec::new(); - if let Some(ResponseFormat::JsonSchema { json_schema }) = &request.response_format { - let schema_str = serde_json::to_string(&json_schema.schema) - .map_err(|e| format!("Failed to serialize JSON schema: {}", e))?; - constraints.push(proto::sampling_params::Constraint::JsonSchema(schema_str)); + // Handle response_format constraints + match &request.response_format { + Some(ResponseFormat::JsonObject) => { + // json_object mode - constrain to valid JSON object + let schema = serde_json::json!({"type": "object"}); + let schema_str = serde_json::to_string(&schema) + .map_err(|e| format!("Failed to serialize JSON schema: {}", e))?; + constraints.push(proto::sampling_params::Constraint::JsonSchema(schema_str)); + } + Some(ResponseFormat::JsonSchema { json_schema }) => { + let schema_str = serde_json::to_string(&json_schema.schema) + .map_err(|e| format!("Failed to serialize JSON schema: {}", e))?; + constraints.push(proto::sampling_params::Constraint::JsonSchema(schema_str)); + } + Some(ResponseFormat::Text) | None => { + // No constraint for text format + } } if let Some(ebnf) = &request.ebnf { @@ -418,7 +432,7 @@ impl SglangSchedulerClient { constraints.push(proto::sampling_params::Constraint::Regex(regex.clone())); } - // Handle tool call constraint + // Handle tool call constraint from preparation stage if let Some((constraint_type, constraint_value)) = tool_call_constraint { if !constraints.is_empty() { return Err("Constrained decoding is not compatible with tool calls.".to_string()); @@ -446,10 +460,10 @@ impl SglangSchedulerClient { fn build_grpc_sampling_params_from_responses( &self, request: &ResponsesRequest, - tool_call_constraint: Option<(String, String)>, + constraint: Option<(String, String)>, ) -> Result { - // ResponsesRequest doesn't have stop sequences in the same way - // For Harmony router: Tools are handled via structural_tag constraints + // Used by Harmony models only. Regular models use Chat API path. + // Constraints come from Harmony preparation stage (structural_tag) or tool handling. let max_new_tokens = request.max_output_tokens.map(|v| v as i32); @@ -469,22 +483,25 @@ impl SglangSchedulerClient { ignore_eos: false, no_stop_trim: false, n: 1, // Responses API doesn't support n>1 - constraint: self.build_constraint_for_responses(tool_call_constraint)?, + constraint: self.build_constraint_for_responses(constraint)?, ..Default::default() }) } - /// Build constraint for Responses API (simpler than Chat API's build_constraint) + /// Build constraint for Responses API /// - /// Responses API doesn't support response_format, ebnf, or regex constraints, - /// so this only handles tool_call_constraint. + /// Handles constraints from Harmony preparation stage (structural_tag for Harmony models, + /// structured output via text field, or tool call constraints). + /// + /// Note: Regular gRPC models use Chat API path with response_format, not this function. fn build_constraint_for_responses( &self, - tool_call_constraint: Option<(String, String)>, + constraint: Option<(String, String)>, ) -> Result, String> { - if let Some((constraint_type, constraint_value)) = tool_call_constraint { - let tool_constraint = match constraint_type.as_str() { + if let Some((constraint_type, constraint_value)) = constraint { + let parsed_constraint = match constraint_type.as_str() { "structural_tag" => { + // Harmony models: structural tag from preparation stage proto::sampling_params::Constraint::StructuralTag(constraint_value) } "json_schema" => proto::sampling_params::Constraint::JsonSchema(constraint_value), @@ -492,7 +509,7 @@ impl SglangSchedulerClient { "regex" => proto::sampling_params::Constraint::Regex(constraint_value), _ => return Err(format!("Unknown constraint type: {}", constraint_type)), }; - Ok(Some(tool_constraint)) + Ok(Some(parsed_constraint)) } else { Ok(None) } diff --git a/sgl-router/src/protocols/responses.rs b/sgl-router/src/protocols/responses.rs index 864e5d9ca..ed3ba3a30 100644 --- a/sgl-router/src/protocols/responses.rs +++ b/sgl-router/src/protocols/responses.rs @@ -291,15 +291,36 @@ pub struct ReasoningInfo { pub summary: Option, } +// ============================================================================ +// Text Format (structured outputs) +// ============================================================================ + +/// Text configuration for structured output requests #[derive(Debug, Clone, Deserialize, Serialize)] -pub struct ResponseTextFormat { - pub format: TextFormatType, +pub struct TextConfig { + #[serde(skip_serializing_if = "Option::is_none")] + pub format: Option, } +/// Text format: text (default), json_object (legacy), or json_schema (recommended) #[derive(Debug, Clone, Deserialize, Serialize)] -pub struct TextFormatType { - #[serde(rename = "type")] - pub format_type: String, +#[serde(tag = "type")] +pub enum TextFormat { + #[serde(rename = "text")] + Text, + + #[serde(rename = "json_object")] + JsonObject, + + #[serde(rename = "json_schema")] + JsonSchema { + name: String, + schema: Value, + #[serde(skip_serializing_if = "Option::is_none")] + description: Option, + #[serde(skip_serializing_if = "Option::is_none")] + strict: Option, + }, } #[derive(Debug, Clone, Deserialize, Serialize)] @@ -539,6 +560,10 @@ pub struct ResponsesRequest { #[serde(skip_serializing_if = "Option::is_none")] pub truncation: Option, + /// Text format for structured outputs (text, json_object, json_schema) + #[serde(skip_serializing_if = "Option::is_none")] + pub text: Option, + /// User identifier #[serde(skip_serializing_if = "Option::is_none")] pub user: Option, @@ -607,6 +632,7 @@ impl Default for ResponsesRequest { top_logprobs: None, top_p: None, truncation: None, + text: None, user: None, request_id: None, priority: 0, @@ -906,7 +932,7 @@ pub struct ResponsesResponse { /// Text format settings #[serde(skip_serializing_if = "Option::is_none")] - pub text: Option, + pub text: Option, /// Tool choice setting #[serde(default = "default_tool_choice")] diff --git a/sgl-router/src/routers/grpc/harmony/processor.rs b/sgl-router/src/routers/grpc/harmony/processor.rs index 0d3bdd281..6d6ae8fa8 100644 --- a/sgl-router/src/routers/grpc/harmony/processor.rs +++ b/sgl-router/src/routers/grpc/harmony/processor.rs @@ -269,7 +269,7 @@ impl HarmonyResponseProcessor { reasoning: None, // Set by caller if needed store: responses_request.store.unwrap_or(true), temperature: responses_request.temperature, - text: None, + text: responses_request.text.clone(), tool_choice: responses_request .tool_choice .as_ref() diff --git a/sgl-router/src/routers/grpc/harmony/stages/preparation.rs b/sgl-router/src/routers/grpc/harmony/stages/preparation.rs index eec874bcb..b8d8919df 100644 --- a/sgl-router/src/routers/grpc/harmony/stages/preparation.rs +++ b/sgl-router/src/routers/grpc/harmony/stages/preparation.rs @@ -88,7 +88,7 @@ impl HarmonyPreparationStage { // Step 2: Build tool constraints let tool_constraints = if let Some(tools) = body_ref.tools.as_ref() { - Self::generate_harmony_structural_tag(tools, &body_ref.tool_choice).map_err(|e| *e)? + Self::generate_tool_call_constraint(tools, &body_ref.tool_choice).map_err(|e| *e)? } else { None }; @@ -139,26 +139,40 @@ impl HarmonyPreparationStage { function_tools = filtered; } - // Step 3: Generate Harmony structural tags from filtered tools - let tool_constraints = if !function_tools.is_empty() { - Self::generate_harmony_structural_tag(&function_tools, &request.tool_choice) + // Step 3: Generate Harmony structural tags + let tool_constraint = if !function_tools.is_empty() { + Self::generate_tool_call_constraint(&function_tools, &request.tool_choice) .map_err(|e| *e)? } else { None }; + let text_constraint = if let Some(text_config) = &request.text { + Self::generate_text_format_constraint(text_config).map_err(|e| *e)? + } else { + None + }; + + if tool_constraint.is_some() && text_constraint.is_some() { + return Err(error::bad_request( + "Cannot use both tool_choice (required/function) and text format (json_object/json_schema) simultaneously".to_string(), + )); + } + + let constraint = tool_constraint.or(text_constraint); + // Step 3: Build via Harmony from responses API request let build_output = self .builder .build_from_responses(request) .map_err(|e| error::bad_request(format!("Harmony build failed: {}", e)))?; - // Step 4: Store results with tool_constraints + // Step 4: Store results with constraint ctx.state.preparation = Some(PreparationOutput { original_text: None, token_ids: build_output.input_ids, processed_messages: None, - tool_constraints, + tool_constraints: constraint, filtered_request: None, harmony_mode: true, selection_text: Some(build_output.selection_text), @@ -169,11 +183,39 @@ impl HarmonyPreparationStage { Ok(None) } + /// Generate Harmony structural tag for structured output (text field) + /// + /// Converts text.format to structural tag that constrains the final channel. + /// Returns None if text.format is not specified or is "text". + fn generate_text_format_constraint( + text_config: &crate::protocols::responses::TextConfig, + ) -> Result, Box> { + use crate::protocols::responses::TextFormat; + + let Some(format) = &text_config.format else { + return Ok(None); + }; + + match format { + TextFormat::Text => Ok(None), + TextFormat::JsonObject => { + let tag = build_text_format_structural_tag(&serde_json::json!({"type": "object"})) + .map_err(|e| Box::new(error::internal_error(e)))?; + Ok(Some(("structural_tag".to_string(), tag))) + } + TextFormat::JsonSchema { schema, .. } => { + let tag = build_text_format_structural_tag(schema) + .map_err(|e| Box::new(error::internal_error(e)))?; + Ok(Some(("structural_tag".to_string(), tag))) + } + } + } + /// Generate Harmony structural tag for tool constraints /// /// Uses structural tags with `triggered_tags` format to force Harmony format output. /// This ensures the model outputs in Harmony format (with channels) even when constrained. - fn generate_harmony_structural_tag( + fn generate_tool_call_constraint( tools: &[Tool], tool_choice: &Option, ) -> Result, Box> { @@ -183,16 +225,16 @@ impl HarmonyPreparationStage { match choice { ToolChoice::Function { function, .. } => { - let tag = Self::build_harmony_structural_tag(tools, Some(&function.name))?; + let tag = Self::build_tool_call_structural_tag(tools, Some(&function.name))?; Ok(Some(("structural_tag".to_string(), tag))) } ToolChoice::Value(ToolChoiceValue::Required) => { - let tag = Self::build_harmony_structural_tag(tools, None)?; + let tag = Self::build_tool_call_structural_tag(tools, None)?; Ok(Some(("structural_tag".to_string(), tag))) } ToolChoice::AllowedTools { mode, .. } => { if mode == "required" { - let tag = Self::build_harmony_structural_tag(tools, None)?; + let tag = Self::build_tool_call_structural_tag(tools, None)?; Ok(Some(("structural_tag".to_string(), tag))) } else { Ok(None) @@ -203,7 +245,11 @@ impl HarmonyPreparationStage { } /// Build Harmony structural tag for tool calling constraints - fn build_harmony_structural_tag( + /// + /// Supports both reasoning-enabled and reasoning-disabled modes: + /// - With reasoning: triggers on `<|start|>assistant<|channel|>commentary` (waits for analysis) + /// - Without reasoning: triggers on `<|channel|>commentary` (goes directly to commentary) + fn build_tool_call_structural_tag( tools: &[Tool], specific_function: Option<&str>, ) -> Result> { @@ -227,11 +273,12 @@ impl HarmonyPreparationStage { )))); } - // Build tags for each tool + // Build tags for each tool - need two patterns per tool for reasoning on/off for tool in tools_to_use { let tool_name = &tool.function.name; let params_schema = &tool.function.parameters; + // Pattern 1: For reasoning-enabled mode (with analysis channel before commentary) tags.push(json!({ "begin": format!("<|start|>assistant<|channel|>commentary to=functions.{}<|constrain|>json<|message|>", tool_name), "content": { @@ -240,6 +287,16 @@ impl HarmonyPreparationStage { }, "end": "" // `end` is empty because <|call|> comes naturally from Harmony stop tokens })); + + // Pattern 2: For reasoning-disabled mode (goes directly to commentary channel) + tags.push(json!({ + "begin": format!("<|channel|>commentary to=functions.{}<|constrain|>json<|message|>", tool_name), + "content": { + "type": "json_schema", + "json_schema": params_schema + }, + "end": "" + })); } let stop_after_first = specific_function.is_some(); @@ -247,7 +304,7 @@ impl HarmonyPreparationStage { let structural_tag = json!({ "format": { "type": "triggered_tags", - "triggers": ["<|start|>assistant"], + "triggers": ["<|start|>assistant<|channel|>commentary", "<|channel|>commentary"], "tags": tags, "at_least_one": true, "stop_after_first": stop_after_first @@ -262,3 +319,49 @@ impl HarmonyPreparationStage { }) } } + +/// Build Harmony structural tag for structured output (JSON schema constraint) +/// +/// Creates a structural tag that applies JSON schema constraint to the final channel, +/// supporting both reasoning-enabled and reasoning-disabled modes: +/// - With reasoning: triggers on `<|start|>assistant<|channel|>final` (waits for analysis to complete) +/// - Without reasoning: triggers on `<|channel|>final` (goes directly to final channel) +/// +/// This is used for the Responses API text.format field (json_object or json_schema). +pub fn build_text_format_structural_tag(schema: &serde_json::Value) -> Result { + let structural_tag = json!({ + "format": { + "type": "triggered_tags", + "triggers": ["<|start|>assistant<|channel|>final", "<|channel|>final"], + "tags": [ + { + // Pattern 1: For reasoning-enabled mode (with analysis channel before final) + "begin": "<|start|>assistant<|channel|>final<|constrain|>json<|message|>", + "content": { + "type": "json_schema", + "json_schema": schema + }, + "end": "" + }, + { + // Pattern 2: For reasoning-disabled mode (goes directly to final channel) + "begin": "<|channel|>final<|constrain|>json<|message|>", + "content": { + "type": "json_schema", + "json_schema": schema + }, + "end": "" + } + ], + "at_least_one": true, + "stop_after_first": true + } + }); + + serde_json::to_string(&structural_tag).map_err(|e| { + format!( + "Failed to serialize structural tag for structured output: {}", + e + ) + }) +} diff --git a/sgl-router/src/routers/grpc/harmony/stages/request_building.rs b/sgl-router/src/routers/grpc/harmony/stages/request_building.rs index 5d64f01d6..5e63aab67 100644 --- a/sgl-router/src/routers/grpc/harmony/stages/request_building.rs +++ b/sgl-router/src/routers/grpc/harmony/stages/request_building.rs @@ -67,7 +67,7 @@ impl PipelineStage for HarmonyRequestBuildingStage { let body = prep.filtered_request.as_ref().unwrap_or(request.as_ref()); builder_client - .build_generate_request( + .build_generate_request_from_chat( request_id, body, placeholder_processed_text, diff --git a/sgl-router/src/routers/grpc/regular/responses/conversions.rs b/sgl-router/src/routers/grpc/regular/responses/conversions.rs index 58306f397..89a66bbb2 100644 --- a/sgl-router/src/routers/grpc/regular/responses/conversions.rs +++ b/sgl-router/src/routers/grpc/regular/responses/conversions.rs @@ -10,11 +10,14 @@ use crate::{ protocols::{ chat::{ChatCompletionRequest, ChatCompletionResponse, ChatMessage, UserMessageContent}, - common::{FunctionCallResponse, StreamOptions, ToolCall, ToolChoice, UsageInfo}, + common::{ + FunctionCallResponse, JsonSchemaFormat, ResponseFormat, StreamOptions, ToolCall, + ToolChoice, UsageInfo, + }, responses::{ ResponseContentPart, ResponseInput, ResponseInputOutputItem, ResponseOutputItem, ResponseReasoningContent::ReasoningText, ResponseStatus, ResponsesRequest, - ResponsesResponse, ResponsesUsage, StringOrContentParts, + ResponsesResponse, ResponsesUsage, StringOrContentParts, TextConfig, TextFormat, }, }, routers::grpc::common::responses::utils::extract_tools_from_response_tools, @@ -188,6 +191,7 @@ pub fn responses_to_chat(req: &ResponsesRequest) -> Result ChatMessage { } } +/// Map TextConfig from Responses API to ResponseFormat for Chat API +/// +/// Converts the structured output configuration from the Responses API format +/// to the Chat API format for non-Harmony models. +fn map_text_to_response_format(text: &Option) -> Option { + let text_config = text.as_ref()?; + let format = text_config.format.as_ref()?; + + match format { + TextFormat::Text => Some(ResponseFormat::Text), + TextFormat::JsonObject => Some(ResponseFormat::JsonObject), + TextFormat::JsonSchema { + name, + schema, + description: _, + strict, + } => Some(ResponseFormat::JsonSchema { + json_schema: JsonSchemaFormat { + name: name.clone(), + schema: schema.clone(), + strict: *strict, + }, + }), + } +} + /// Convert a ChatCompletionResponse to ResponsesResponse /// /// # Conversion Logic @@ -338,7 +368,7 @@ pub fn chat_to_responses( reasoning: None, // TODO: Map reasoning effort if needed store: original_req.store.unwrap_or(true), temperature: original_req.temperature, - text: None, + text: original_req.text.clone(), tool_choice: ToolChoice::serialize_to_string(&original_req.tool_choice), tools: original_req.tools.clone().unwrap_or_default(), top_p: original_req.top_p, diff --git a/sgl-router/src/routers/grpc/regular/responses/tool_loop.rs b/sgl-router/src/routers/grpc/regular/responses/tool_loop.rs index 54480e821..3595d281d 100644 --- a/sgl-router/src/routers/grpc/regular/responses/tool_loop.rs +++ b/sgl-router/src/routers/grpc/regular/responses/tool_loop.rs @@ -427,6 +427,7 @@ pub(super) async fn execute_tool_loop( service_tier: current_request.service_tier.clone(), top_logprobs: current_request.top_logprobs, truncation: current_request.truncation.clone(), + text: current_request.text.clone(), request_id: None, priority: current_request.priority, frequency_penalty: current_request.frequency_penalty, @@ -971,6 +972,7 @@ async fn execute_tool_loop_streaming_internal( service_tier: current_request.service_tier.clone(), top_logprobs: current_request.top_logprobs, truncation: current_request.truncation.clone(), + text: current_request.text.clone(), request_id: None, priority: current_request.priority, frequency_penalty: current_request.frequency_penalty, diff --git a/sgl-router/src/routers/grpc/regular/stages/chat/request_building.rs b/sgl-router/src/routers/grpc/regular/stages/chat/request_building.rs index 2d69b7de1..e91c8b64e 100644 --- a/sgl-router/src/routers/grpc/regular/stages/chat/request_building.rs +++ b/sgl-router/src/routers/grpc/regular/stages/chat/request_building.rs @@ -51,7 +51,7 @@ impl PipelineStage for ChatRequestBuildingStage { let body_ref = prep.filtered_request.as_ref().unwrap_or(&chat_request); let mut proto_request = builder_client - .build_generate_request( + .build_generate_request_from_chat( request_id, body_ref, prep.processed_messages.as_ref().unwrap().text.clone(), diff --git a/sgl-router/tests/responses_api_test.rs b/sgl-router/tests/responses_api_test.rs index 9d10db992..beed0669c 100644 --- a/sgl-router/tests/responses_api_test.rs +++ b/sgl-router/tests/responses_api_test.rs @@ -91,6 +91,7 @@ async fn test_non_streaming_mcp_minimal_e2e_with_persistence() { top_logprobs: Some(0), top_p: None, truncation: Some(Truncation::Disabled), + text: None, user: None, request_id: Some("resp_test_mcp_e2e".to_string()), priority: 0, @@ -312,6 +313,7 @@ fn test_responses_request_creation() { top_logprobs: Some(5), top_p: Some(0.9), truncation: Some(Truncation::Disabled), + text: None, user: Some("test-user".to_string()), request_id: Some("resp_test123".to_string()), priority: 0, @@ -354,6 +356,7 @@ fn test_responses_request_sglang_extensions() { top_logprobs: Some(0), top_p: Some(0.95), truncation: Some(Truncation::Auto), + text: None, user: None, request_id: Some("resp_test456".to_string()), priority: 0, @@ -469,6 +472,7 @@ fn test_json_serialization() { top_logprobs: Some(10), top_p: Some(0.8), truncation: Some(Truncation::Auto), + text: None, user: Some("test_user".to_string()), request_id: Some("resp_comprehensive_test".to_string()), priority: 1, @@ -574,6 +578,7 @@ async fn test_multi_turn_loop_with_mcp() { top_logprobs: Some(0), top_p: Some(1.0), truncation: Some(Truncation::Disabled), + text: None, user: None, request_id: Some("resp_multi_turn_test".to_string()), priority: 0, @@ -722,6 +727,7 @@ async fn test_max_tool_calls_limit() { top_logprobs: Some(0), top_p: Some(1.0), truncation: Some(Truncation::Disabled), + text: None, user: None, request_id: Some("resp_max_calls_test".to_string()), priority: 0, @@ -893,6 +899,7 @@ async fn test_streaming_with_mcp_tool_calls() { top_logprobs: Some(0), top_p: Some(1.0), truncation: Some(Truncation::Disabled), + text: None, user: None, request_id: Some("resp_streaming_mcp_test".to_string()), priority: 0, @@ -1172,6 +1179,7 @@ async fn test_streaming_multi_turn_with_mcp() { top_logprobs: Some(0), top_p: Some(1.0), truncation: Some(Truncation::Disabled), + text: None, user: None, request_id: Some("resp_streaming_multiturn_test".to_string()), priority: 0,