diff --git a/sgl-model-gateway/src/protocols/builders/chat/mod.rs b/sgl-model-gateway/src/protocols/builders/chat/mod.rs deleted file mode 100644 index 61386b505..000000000 --- a/sgl-model-gateway/src/protocols/builders/chat/mod.rs +++ /dev/null @@ -1,7 +0,0 @@ -//! Builders for Chat Completion API response types - -pub mod response; -pub mod stream_response; - -pub use response::ChatCompletionResponseBuilder; -pub use stream_response::ChatCompletionStreamResponseBuilder; diff --git a/sgl-model-gateway/src/protocols/builders/chat/response.rs b/sgl-model-gateway/src/protocols/builders/chat/response.rs deleted file mode 100644 index 7267e2ee2..000000000 --- a/sgl-model-gateway/src/protocols/builders/chat/response.rs +++ /dev/null @@ -1,218 +0,0 @@ -//! Builder for ChatCompletionResponse -//! -//! Provides an ergonomic fluent API for constructing chat completion responses. - -use crate::protocols::{chat::*, common::Usage}; - -/// Builder for ChatCompletionResponse -/// -/// Provides a fluent interface for constructing chat completion responses with sensible defaults. -#[must_use = "Builder does nothing until .build() is called"] -#[derive(Clone, Debug)] -pub struct ChatCompletionResponseBuilder { - id: String, - object: String, - created: u64, - model: String, - choices: Vec, - usage: Option, - system_fingerprint: Option, -} - -impl ChatCompletionResponseBuilder { - /// Create a new builder with required fields - /// - /// # Arguments - /// - `id`: Completion ID (e.g., "chatcmpl_abc123") - /// - `model`: Model name used for generation - pub fn new(id: impl Into, model: impl Into) -> Self { - Self { - id: id.into(), - object: "chat.completion".to_string(), - created: chrono::Utc::now().timestamp() as u64, - model: model.into(), - choices: Vec::new(), - usage: None, - system_fingerprint: None, - } - } - - /// Copy common fields from a ChatCompletionRequest - /// - /// This populates the model field from the request. - pub fn copy_from_request(mut self, request: &ChatCompletionRequest) -> Self { - self.model = request.model.clone(); - self - } - - /// Set the object type (default: "chat.completion") - pub fn object(mut self, object: impl Into) -> Self { - self.object = object.into(); - self - } - - /// Set the creation timestamp (default: current time) - pub fn created(mut self, timestamp: u64) -> Self { - self.created = timestamp; - self - } - - /// Set the choices - pub fn choices(mut self, choices: Vec) -> Self { - self.choices = choices; - self - } - - /// Add a single choice - pub fn add_choice(mut self, choice: ChatChoice) -> Self { - self.choices.push(choice); - self - } - - /// Set usage information - pub fn usage(mut self, usage: Usage) -> Self { - self.usage = Some(usage); - self - } - - /// Set usage if provided (handles Option) - pub fn maybe_usage(mut self, usage: Option) -> Self { - if let Some(u) = usage { - self.usage = Some(u); - } - self - } - - /// Set system fingerprint if provided (handles Option) - pub fn maybe_system_fingerprint(mut self, fingerprint: Option>) -> Self { - if let Some(fp) = fingerprint { - self.system_fingerprint = Some(fp.into()); - } - self - } - - /// Build the ChatCompletionResponse - pub fn build(self) -> ChatCompletionResponse { - ChatCompletionResponse { - id: self.id, - object: self.object, - created: self.created, - model: self.model, - choices: self.choices, - usage: self.usage, - system_fingerprint: self.system_fingerprint, - } - } -} - -// ============================================================================ -// Tests -// ============================================================================ - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_build_minimal() { - let response = ChatCompletionResponse::builder("chatcmpl_123", "gpt-4").build(); - - assert_eq!(response.id, "chatcmpl_123"); - assert_eq!(response.model, "gpt-4"); - assert_eq!(response.object, "chat.completion"); - assert!(response.choices.is_empty()); - assert!(response.usage.is_none()); - assert!(response.system_fingerprint.is_none()); - } - - #[test] - fn test_build_complete() { - let choice = ChatChoice { - index: 0, - message: ChatCompletionMessage { - role: "assistant".to_string(), - content: Some("Hello!".to_string()), - tool_calls: None, - reasoning_content: None, - }, - logprobs: None, - finish_reason: Some("stop".to_string()), - matched_stop: None, - hidden_states: None, - }; - - let usage = Usage { - prompt_tokens: 10, - completion_tokens: 20, - total_tokens: 30, - completion_tokens_details: None, - }; - - let response = ChatCompletionResponse::builder("chatcmpl_456", "gpt-4") - .choices(vec![choice.clone()]) - .maybe_usage(Some(usage)) - .maybe_system_fingerprint(Some("fp_123abc")) - .build(); - - assert_eq!(response.id, "chatcmpl_456"); - assert_eq!(response.choices.len(), 1); - assert_eq!(response.choices[0].index, 0); - assert!(response.usage.is_some()); - assert_eq!(response.system_fingerprint.as_ref().unwrap(), "fp_123abc"); - } - - #[test] - fn test_add_multiple_choices() { - let choice1 = ChatChoice { - index: 0, - message: ChatCompletionMessage { - role: "assistant".to_string(), - content: Some("Option 1".to_string()), - tool_calls: None, - reasoning_content: None, - }, - logprobs: None, - finish_reason: Some("stop".to_string()), - matched_stop: None, - hidden_states: None, - }; - - let choice2 = ChatChoice { - index: 1, - message: ChatCompletionMessage { - role: "assistant".to_string(), - content: Some("Option 2".to_string()), - tool_calls: None, - reasoning_content: None, - }, - logprobs: None, - finish_reason: Some("stop".to_string()), - matched_stop: None, - hidden_states: None, - }; - - let response = ChatCompletionResponse::builder("chatcmpl_789", "gpt-4") - .add_choice(choice1) - .add_choice(choice2) - .build(); - - assert_eq!(response.choices.len(), 2); - assert_eq!(response.choices[0].index, 0); - assert_eq!(response.choices[1].index, 1); - } - - #[test] - fn test_copy_from_request() { - let request = ChatCompletionRequest { - messages: vec![], - model: "gpt-3.5-turbo".to_string(), - ..Default::default() - }; - - let response = ChatCompletionResponse::builder("chatcmpl_101", "gpt-4") - .copy_from_request(&request) - .build(); - - assert_eq!(response.model, "gpt-3.5-turbo"); // Copied from request - } -} diff --git a/sgl-model-gateway/src/protocols/builders/chat/stream_response.rs b/sgl-model-gateway/src/protocols/builders/chat/stream_response.rs deleted file mode 100644 index 9afe486aa..000000000 --- a/sgl-model-gateway/src/protocols/builders/chat/stream_response.rs +++ /dev/null @@ -1,421 +0,0 @@ -//! Builder for ChatCompletionStreamResponse -//! -//! Provides an ergonomic fluent API for constructing streaming chat completion responses. - -use std::borrow::Cow; - -use crate::protocols::{ - chat::*, - common::{FunctionCallDelta, ToolCallDelta, Usage}, -}; - -/// Builder for ChatCompletionStreamResponse -/// -/// Provides a fluent interface for constructing streaming chat completion chunks with sensible defaults. -#[must_use = "Builder does nothing until .build() is called"] -#[derive(Clone, Debug)] -pub struct ChatCompletionStreamResponseBuilder { - id: String, - object: String, - created: u64, - model: String, - choices: Vec, - usage: Option, - system_fingerprint: Option, -} - -impl ChatCompletionStreamResponseBuilder { - /// Create a new builder with required fields - /// - /// # Arguments - /// - `id`: Completion ID (e.g., "chatcmpl_abc123") - /// - `model`: Model name used for generation - pub fn new(id: impl Into, model: impl Into) -> Self { - Self { - id: id.into(), - object: "chat.completion.chunk".to_string(), - created: chrono::Utc::now().timestamp() as u64, - model: model.into(), - choices: Vec::new(), - usage: None, - system_fingerprint: None, - } - } - - /// Copy common fields from a ChatCompletionRequest - /// - /// This populates the model field from the request. - pub fn copy_from_request(mut self, request: &ChatCompletionRequest) -> Self { - self.model = request.model.clone(); - self - } - - /// Set the object type (default: "chat.completion.chunk") - pub fn object(mut self, object: impl Into) -> Self { - self.object = object.into(); - self - } - - /// Set the creation timestamp (default: current time) - pub fn created(mut self, timestamp: u64) -> Self { - self.created = timestamp; - self - } - - /// Set the choices - pub fn choices(mut self, choices: Vec) -> Self { - self.choices = choices; - self - } - - /// Add a single choice (delta) - pub fn add_choice(mut self, choice: ChatStreamChoice) -> Self { - self.choices.push(choice); - self - } - - /// Set usage information (typically sent in final chunk) - pub fn usage(mut self, usage: Usage) -> Self { - self.usage = Some(usage); - self - } - - /// Set system fingerprint if provided (handles Option) - pub fn maybe_system_fingerprint(mut self, fingerprint: Option>) -> Self { - if let Some(fp) = fingerprint { - self.system_fingerprint = Some(fp.into()); - } - self - } - - /// Set usage if provided (handles Option) - pub fn maybe_usage(mut self, usage: Option) -> Self { - if let Some(u) = usage { - self.usage = Some(u); - } - self - } - - /// Add a choice delta that sets `role` and `content` - pub fn add_choice_content( - mut self, - index: u32, - role: impl Into, - content: impl Into, - ) -> Self { - self.choices.push(ChatStreamChoice { - index, - delta: ChatMessageDelta { - role: Some(role.into()), - content: Some(content.into()), - tool_calls: None, - reasoning_content: None, - }, - logprobs: None, - finish_reason: None, - matched_stop: None, - }); - self - } - - /// Add a choice delta that sets `role`, `content`, and `logprobs` - pub fn add_choice_content_with_logprobs( - mut self, - index: u32, - role: impl Into, - content: impl Into, - logprobs: Option, - ) -> Self { - self.choices.push(ChatStreamChoice { - index, - delta: ChatMessageDelta { - role: Some(role.into()), - content: Some(content.into()), - tool_calls: None, - reasoning_content: None, - }, - logprobs, - finish_reason: None, - matched_stop: None, - }); - self - } - - /// Add a choice delta that only sets `role` - pub fn add_choice_role(mut self, index: u32, role: impl Into) -> Self { - self.choices.push(ChatStreamChoice { - index, - delta: ChatMessageDelta { - role: Some(role.into()), - content: None, - tool_calls: None, - reasoning_content: None, - }, - logprobs: None, - finish_reason: None, - matched_stop: None, - }); - self - } - - /// Add a choice delta that appends a tool-call *arguments delta* - /// Uses `Cow` so you can pass `&str` or `String` without extra clones - pub fn add_choice_tool_args( - mut self, - index: u32, - args_delta: impl Into>, - ) -> Self { - self.choices.push(ChatStreamChoice { - index, - delta: ChatMessageDelta { - role: Some("assistant".to_string()), - content: None, - tool_calls: Some(vec![ToolCallDelta { - index: 0, - id: None, - tool_type: None, - function: Some(FunctionCallDelta { - name: None, - arguments: Some(args_delta.into().into_owned()), - }), - }]), - reasoning_content: None, - }, - logprobs: None, - finish_reason: None, - matched_stop: None, - }); - self - } - - /// Add a choice delta that sets reasoning content (for models that stream reasoning) - pub fn add_choice_reasoning(mut self, index: u32, reasoning: impl Into) -> Self { - self.choices.push(ChatStreamChoice { - index, - delta: ChatMessageDelta { - role: Some("assistant".to_string()), - content: None, - tool_calls: None, - reasoning_content: Some(reasoning.into()), - }, - logprobs: None, - finish_reason: None, - matched_stop: None, - }); - self - } - - /// Add a choice delta for tool call with function name and ID - pub fn add_choice_tool_name( - mut self, - index: u32, - tool_call_id: impl Into, - function_name: impl Into, - ) -> Self { - self.choices.push(ChatStreamChoice { - index, - delta: ChatMessageDelta { - role: Some("assistant".to_string()), - content: None, - tool_calls: Some(vec![ToolCallDelta { - index: 0, - id: Some(tool_call_id.into()), - tool_type: Some("function".to_string()), - function: Some(FunctionCallDelta { - name: Some(function_name.into()), - arguments: None, - }), - }]), - reasoning_content: None, - }, - logprobs: None, - finish_reason: None, - matched_stop: None, - }); - self - } - - /// Add a choice delta with a pre-constructed ToolCallDelta - /// Useful when you already have a ToolCallDelta object to emit - pub fn add_choice_tool_call_delta( - mut self, - index: u32, - tool_call_delta: ToolCallDelta, - ) -> Self { - self.choices.push(ChatStreamChoice { - index, - delta: ChatMessageDelta { - role: Some("assistant".to_string()), - content: None, - tool_calls: Some(vec![tool_call_delta]), - reasoning_content: None, - }, - logprobs: None, - finish_reason: None, - matched_stop: None, - }); - self - } - - /// Add a choice with finish_reason (final chunk) - /// This is used for the last chunk in a stream to signal completion - pub fn add_choice_finish_reason( - mut self, - index: u32, - finish_reason: impl Into, - matched_stop: Option, - ) -> Self { - self.choices.push(ChatStreamChoice { - index, - delta: ChatMessageDelta { - role: None, - content: None, - tool_calls: None, - reasoning_content: None, - }, - logprobs: None, - finish_reason: Some(finish_reason.into()), - matched_stop, - }); - self - } - - /// Build the ChatCompletionStreamResponse - pub fn build(self) -> ChatCompletionStreamResponse { - ChatCompletionStreamResponse { - id: self.id, - object: self.object, - created: self.created, - model: self.model, - system_fingerprint: self.system_fingerprint, - choices: self.choices, - usage: self.usage, - } - } -} - -// ============================================================================ -// Tests -// ============================================================================ - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_build_minimal() { - let chunk = ChatCompletionStreamResponseBuilder::new("chatcmpl_123", "gpt-4").build(); - - assert_eq!(chunk.id, "chatcmpl_123"); - assert_eq!(chunk.model, "gpt-4"); - assert_eq!(chunk.object, "chat.completion.chunk"); - assert!(chunk.choices.is_empty()); - assert!(chunk.usage.is_none()); - } - - #[test] - fn test_with_content_delta() { - let chunk = ChatCompletionStreamResponseBuilder::new("chatcmpl_456", "gpt-4") - .add_choice_content(0, "assistant", "Hello") - .build(); - - assert_eq!(chunk.choices.len(), 1); - assert_eq!(chunk.choices[0].index, 0); - assert_eq!(chunk.choices[0].delta.content.as_ref().unwrap(), "Hello"); - assert_eq!(chunk.choices[0].delta.role.as_ref().unwrap(), "assistant"); - assert!(chunk.choices[0].finish_reason.is_none()); - } - - #[test] - fn test_with_role_delta() { - let chunk = ChatCompletionStreamResponseBuilder::new("chatcmpl_789", "gpt-4") - .add_choice_role(0, "assistant") - .build(); - - assert_eq!(chunk.choices.len(), 1); - assert_eq!(chunk.choices[0].delta.role.as_ref().unwrap(), "assistant"); - assert!(chunk.choices[0].delta.content.is_none()); - } - - #[test] - fn test_with_finish_reason() { - let chunk = ChatCompletionStreamResponseBuilder::new("chatcmpl_101", "gpt-4") - .add_choice_finish_reason(0, "stop", None) - .build(); - - assert_eq!(chunk.choices.len(), 1); - assert_eq!(chunk.choices[0].finish_reason.as_ref().unwrap(), "stop"); - assert!(chunk.choices[0].delta.content.is_none()); - assert!(chunk.choices[0].delta.role.is_none()); - } - - #[test] - fn test_multiple_deltas() { - let chunk = ChatCompletionStreamResponseBuilder::new("chatcmpl_202", "gpt-4") - .add_choice_role(0, "assistant") - .add_choice_content(0, "assistant", "Hello") - .add_choice_content(0, "assistant", " world") - .add_choice_finish_reason(0, "stop", None) - .build(); - - assert_eq!(chunk.choices.len(), 4); // role + 2 content + finish - } - - #[test] - fn test_with_usage() { - let usage = Usage { - prompt_tokens: 10, - completion_tokens: 20, - total_tokens: 30, - completion_tokens_details: None, - }; - - let chunk = ChatCompletionStreamResponseBuilder::new("chatcmpl_303", "gpt-4") - .add_choice_finish_reason(0, "stop", None) - .usage(usage) - .build(); - - assert!(chunk.usage.is_some()); - assert_eq!(chunk.usage.as_ref().unwrap().total_tokens, 30); - } - - #[test] - fn test_copy_from_request() { - let request = ChatCompletionRequest { - messages: vec![], - model: "gpt-3.5-turbo".to_string(), - ..Default::default() - }; - - let chunk = ChatCompletionStreamResponseBuilder::new("chatcmpl_404", "gpt-4") - .copy_from_request(&request) - .add_choice_content(0, "assistant", "test") - .build(); - - assert_eq!(chunk.model, "gpt-3.5-turbo"); // Copied from request - } - - #[test] - fn test_add_choice_explicit() { - let choice = ChatStreamChoice { - index: 0, - delta: ChatMessageDelta { - role: Some("assistant".to_string()), - content: Some("Hello".to_string()), - tool_calls: None, - reasoning_content: None, - }, - logprobs: None, - finish_reason: None, - matched_stop: None, - }; - - let chunk = ChatCompletionStreamResponseBuilder::new("chatcmpl_505", "gpt-4") - .add_choice(choice) - .build(); - - assert_eq!(chunk.choices.len(), 1); - assert_eq!(chunk.choices[0].delta.role.as_ref().unwrap(), "assistant"); - assert_eq!(chunk.choices[0].delta.content.as_ref().unwrap(), "Hello"); - } -} diff --git a/sgl-model-gateway/src/protocols/builders/mod.rs b/sgl-model-gateway/src/protocols/builders/mod.rs deleted file mode 100644 index 920844124..000000000 --- a/sgl-model-gateway/src/protocols/builders/mod.rs +++ /dev/null @@ -1,27 +0,0 @@ -//! Builder patterns for protocol response types -//! -//! This module provides ergonomic builders for response types with many optional fields. -//! Builders help avoid telescoping constructors and make construction intent clear. -//! -//! # Organization -//! -//! Builders are organized by API: -//! - `chat/` - Chat Completion API builders (response, stream_response) -//! - `responses/` - Responses API builder (response) -//! -//! # Optional Fields -//! -//! For optional fields, builders provide `maybe_*` methods that handle `Option` directly: -//! ```ignore -//! builder -//! .field(value) -//! .maybe_optional_field(optional_value) // Accepts Option -//! .build() -//! ``` - -pub mod chat; -pub mod responses; - -// Re-export all builders for convenient access -pub use chat::{ChatCompletionResponseBuilder, ChatCompletionStreamResponseBuilder}; -pub use responses::ResponsesResponseBuilder; diff --git a/sgl-model-gateway/src/protocols/builders/responses/mod.rs b/sgl-model-gateway/src/protocols/builders/responses/mod.rs deleted file mode 100644 index aca9d2492..000000000 --- a/sgl-model-gateway/src/protocols/builders/responses/mod.rs +++ /dev/null @@ -1,5 +0,0 @@ -//! Builders for Responses API response types - -pub mod response; - -pub use response::ResponsesResponseBuilder; diff --git a/sgl-model-gateway/src/protocols/builders/responses/response.rs b/sgl-model-gateway/src/protocols/builders/responses/response.rs deleted file mode 100644 index 50d3f1bc5..000000000 --- a/sgl-model-gateway/src/protocols/builders/responses/response.rs +++ /dev/null @@ -1,411 +0,0 @@ -//! Builder for ResponsesResponse -//! -//! Provides an ergonomic fluent API for constructing ResponsesResponse instances. - -use std::collections::HashMap; - -use serde_json::Value; - -use crate::protocols::responses::*; - -/// Builder for ResponsesResponse -/// -/// Provides a fluent interface for constructing responses with sensible defaults. -#[must_use = "Builder does nothing until .build() is called"] -#[derive(Clone, Debug)] -pub struct ResponsesResponseBuilder { - id: String, - object: String, - created_at: i64, - status: ResponseStatus, - error: Option, - incomplete_details: Option, - instructions: Option, - max_output_tokens: Option, - model: String, - output: Vec, - parallel_tool_calls: bool, - previous_response_id: Option, - reasoning: Option, - store: bool, - temperature: Option, - text: Option, - tool_choice: String, - tools: Vec, - top_p: Option, - truncation: Option, - usage: Option, - user: Option, - safety_identifier: Option, - metadata: HashMap, -} - -impl ResponsesResponseBuilder { - /// Create a new builder with required fields - /// - /// # Arguments - /// - `id`: Response ID (e.g., "resp_abc123") - /// - `model`: Model name used for generation - pub fn new(id: impl Into, model: impl Into) -> Self { - Self { - id: id.into(), - object: "response".to_string(), - created_at: chrono::Utc::now().timestamp(), - status: ResponseStatus::InProgress, - error: None, - incomplete_details: None, - instructions: None, - max_output_tokens: None, - model: model.into(), - output: Vec::new(), - parallel_tool_calls: true, - previous_response_id: None, - reasoning: None, - store: true, - temperature: None, - text: None, - tool_choice: "auto".to_string(), - tools: Vec::new(), - top_p: None, - truncation: None, - usage: None, - user: None, - safety_identifier: None, - metadata: HashMap::new(), - } - } - - /// Copy common fields from a ResponsesRequest - /// - /// This populates fields like instructions, max_output_tokens, temperature, etc. - /// from the original request, making it easy to construct a response that mirrors - /// the request parameters. - /// - /// Note: `safety_identifier` is intentionally NOT copied as it is for content moderation - /// and should be set independently from the request's `user` field (which is for billing/tracking). - pub fn copy_from_request(mut self, request: &ResponsesRequest) -> Self { - self.instructions = request.instructions.clone(); - self.max_output_tokens = request.max_output_tokens; - self.parallel_tool_calls = request.parallel_tool_calls.unwrap_or(true); - self.previous_response_id = request.previous_response_id.clone(); - self.store = request.store.unwrap_or(true); - self.temperature = request.temperature; - self.tool_choice = if let Some(ref tc) = request.tool_choice { - serde_json::to_string(tc).unwrap_or_else(|_| "auto".to_string()) - } else { - "auto".to_string() - }; - self.tools = request.tools.clone().unwrap_or_default(); - self.top_p = request.top_p; - self.user = request.user.clone(); - self.metadata = request.metadata.clone().unwrap_or_default(); - self - } - - /// Set the object type (default: "response") - pub fn object(mut self, object: impl Into) -> Self { - self.object = object.into(); - self - } - - /// Set the creation timestamp (default: current time) - pub fn created_at(mut self, timestamp: i64) -> Self { - self.created_at = timestamp; - self - } - - /// Set the response status - pub fn status(mut self, status: ResponseStatus) -> Self { - self.status = status; - self - } - - /// Set error information (if status is failed) - pub fn error(mut self, error: Value) -> Self { - self.error = Some(error); - self - } - - /// Set incomplete details (if response was truncated) - pub fn incomplete_details(mut self, details: Value) -> Self { - self.incomplete_details = Some(details); - self - } - - /// Set system instructions - pub fn instructions(mut self, instructions: impl Into) -> Self { - self.instructions = Some(instructions.into()); - self - } - - /// Set max output tokens - pub fn max_output_tokens(mut self, tokens: u32) -> Self { - self.max_output_tokens = Some(tokens); - self - } - - /// Set output items - pub fn output(mut self, output: Vec) -> Self { - self.output = output; - self - } - - /// Add a single output item - pub fn add_output(mut self, item: ResponseOutputItem) -> Self { - self.output.push(item); - self - } - - /// Set whether parallel tool calls are enabled - pub fn parallel_tool_calls(mut self, enabled: bool) -> Self { - self.parallel_tool_calls = enabled; - self - } - - /// Set previous response ID (if continuation) - pub fn previous_response_id(mut self, id: impl Into) -> Self { - self.previous_response_id = Some(id.into()); - self - } - - /// Set reasoning information - pub fn reasoning(mut self, reasoning: ReasoningInfo) -> Self { - self.reasoning = Some(reasoning); - self - } - - /// Set whether the response is stored - pub fn store(mut self, store: bool) -> Self { - self.store = store; - self - } - - /// Set temperature setting - pub fn temperature(mut self, temperature: f32) -> Self { - self.temperature = Some(temperature); - self - } - - /// Set text format settings if provided (handles Option) - pub fn maybe_text(mut self, text: Option) -> Self { - if let Some(t) = text { - self.text = Some(t); - } - self - } - - /// Set tool choice setting - pub fn tool_choice(mut self, tool_choice: impl Into) -> Self { - self.tool_choice = tool_choice.into(); - self - } - - /// Set available tools - pub fn tools(mut self, tools: Vec) -> Self { - self.tools = tools; - self - } - - /// Set top-p setting - pub fn top_p(mut self, top_p: f32) -> Self { - self.top_p = Some(top_p); - self - } - - /// Set truncation strategy - pub fn truncation(mut self, truncation: impl Into) -> Self { - self.truncation = Some(truncation.into()); - self - } - - /// Set usage information - pub fn usage(mut self, usage: ResponsesUsage) -> Self { - self.usage = Some(usage); - self - } - - /// Set usage if provided (handles Option) - pub fn maybe_usage(mut self, usage: Option) -> Self { - if let Some(u) = usage { - self.usage = Some(u); - } - self - } - - /// Copy from request if provided (handles Option) - pub fn maybe_copy_from_request(mut self, request: Option<&ResponsesRequest>) -> Self { - if let Some(req) = request { - self = self.copy_from_request(req); - } - self - } - - /// Set user identifier - pub fn user(mut self, user: impl Into) -> Self { - self.user = Some(user.into()); - self - } - - /// Set safety identifier - pub fn safety_identifier(mut self, identifier: impl Into) -> Self { - self.safety_identifier = Some(identifier.into()); - self - } - - /// Set metadata - pub fn metadata(mut self, metadata: HashMap) -> Self { - self.metadata = metadata; - self - } - - /// Add a single metadata entry - pub fn add_metadata(mut self, key: impl Into, value: Value) -> Self { - self.metadata.insert(key.into(), value); - self - } - - /// Build the ResponsesResponse - pub fn build(self) -> ResponsesResponse { - ResponsesResponse { - id: self.id, - object: self.object, - created_at: self.created_at, - status: self.status, - error: self.error, - incomplete_details: self.incomplete_details, - instructions: self.instructions, - max_output_tokens: self.max_output_tokens, - model: self.model, - output: self.output, - parallel_tool_calls: self.parallel_tool_calls, - previous_response_id: self.previous_response_id, - reasoning: self.reasoning, - store: self.store, - temperature: self.temperature, - text: self.text, - tool_choice: self.tool_choice, - tools: self.tools, - top_p: self.top_p, - truncation: self.truncation, - usage: self.usage, - user: self.user, - safety_identifier: self.safety_identifier, - metadata: self.metadata, - } - } -} - -// ============================================================================ -// Tests -// ============================================================================ - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_build_minimal() { - let response = ResponsesResponse::builder("resp_123", "gpt-4").build(); - - assert_eq!(response.id, "resp_123"); - assert_eq!(response.model, "gpt-4"); - assert_eq!(response.object, "response"); - assert_eq!(response.status, ResponseStatus::InProgress); - assert!(response.output.is_empty()); - assert!(response.parallel_tool_calls); - assert!(response.store); - } - - #[test] - fn test_build_complete() { - let response = ResponsesResponse::builder("resp_123", "gpt-4") - .status(ResponseStatus::Completed) - .instructions("You are a helpful assistant") - .max_output_tokens(1000) - .temperature(0.7) - .top_p(0.9) - .parallel_tool_calls(false) - .store(false) - .build(); - - assert_eq!(response.status, ResponseStatus::Completed); - assert_eq!( - response.instructions.as_ref().unwrap(), - "You are a helpful assistant" - ); - assert_eq!(response.max_output_tokens, Some(1000)); - assert_eq!(response.temperature, Some(0.7)); - assert_eq!(response.top_p, Some(0.9)); - assert!(!response.parallel_tool_calls); - assert!(!response.store); - } - - #[test] - fn test_copy_from_request() { - let request = ResponsesRequest { - model: "gpt-4".to_string(), - input: ResponseInput::Text("test".to_string()), - instructions: Some("Be helpful".to_string()), - max_output_tokens: Some(500), - temperature: Some(0.8), - top_p: Some(0.95), - parallel_tool_calls: Some(false), - store: Some(false), - user: Some("user_123".to_string()), - metadata: Some(HashMap::from([( - "key".to_string(), - serde_json::json!("value"), - )])), - ..Default::default() - }; - - let response = ResponsesResponse::builder("resp_456", "gpt-4") - .copy_from_request(&request) - .status(ResponseStatus::Completed) - .build(); - - assert_eq!(response.instructions.as_ref().unwrap(), "Be helpful"); - assert_eq!(response.max_output_tokens, Some(500)); - assert_eq!(response.temperature, Some(0.8)); - assert_eq!(response.top_p, Some(0.95)); - assert!(!response.parallel_tool_calls); - assert!(!response.store); - assert_eq!(response.user.as_ref().unwrap(), "user_123"); - assert_eq!( - response.metadata.get("key").unwrap(), - &serde_json::json!("value") - ); - } - - #[test] - fn test_add_output_items() { - let response = ResponsesResponse::builder("resp_789", "gpt-4") - .add_output(ResponseOutputItem::Message { - id: "msg_1".to_string(), - role: "assistant".to_string(), - content: vec![], - status: "completed".to_string(), - }) - .add_output(ResponseOutputItem::Message { - id: "msg_2".to_string(), - role: "assistant".to_string(), - content: vec![], - status: "completed".to_string(), - }) - .build(); - - assert_eq!(response.output.len(), 2); - } - - #[test] - fn test_add_metadata() { - let response = ResponsesResponse::builder("resp_101", "gpt-4") - .add_metadata("key1", serde_json::json!("value1")) - .add_metadata("key2", serde_json::json!(42)) - .build(); - - assert_eq!(response.metadata.len(), 2); - assert_eq!(response.metadata.get("key1").unwrap(), "value1"); - assert_eq!(response.metadata.get("key2").unwrap(), 42); - } -} diff --git a/sgl-model-gateway/src/protocols/chat.rs b/sgl-model-gateway/src/protocols/chat.rs deleted file mode 100644 index 0a0900165..000000000 --- a/sgl-model-gateway/src/protocols/chat.rs +++ /dev/null @@ -1,799 +0,0 @@ -use std::collections::HashMap; - -use serde::{Deserialize, Serialize}; -use serde_json::Value; -use validator::Validate; - -use super::{ - common::{ - default_model, default_true, validate_stop, ChatLogProbs, ContentPart, Function, - FunctionCall, FunctionChoice, GenerationRequest, ResponseFormat, StreamOptions, - StringOrArray, Tool, ToolCall, ToolCallDelta, ToolChoice, ToolChoiceValue, ToolReference, - Usage, - }, - sampling_params::{validate_top_k_value, validate_top_p_value}, -}; -use crate::protocols::{ - builders::{ChatCompletionResponseBuilder, ChatCompletionStreamResponseBuilder}, - validated::Normalizable, -}; - -// ============================================================================ -// Chat Messages -// ============================================================================ - -#[derive(Debug, Clone, Deserialize, Serialize)] -#[serde(tag = "role")] -pub enum ChatMessage { - #[serde(rename = "system")] - System { - content: MessageContent, - #[serde(skip_serializing_if = "Option::is_none")] - name: Option, - }, - #[serde(rename = "user")] - User { - content: MessageContent, - #[serde(skip_serializing_if = "Option::is_none")] - name: Option, - }, - #[serde(rename = "assistant")] - Assistant { - #[serde(skip_serializing_if = "Option::is_none")] - content: Option, - #[serde(skip_serializing_if = "Option::is_none")] - name: Option, - #[serde(skip_serializing_if = "Option::is_none")] - tool_calls: Option>, - /// Reasoning content for O1-style models (SGLang extension) - #[serde(skip_serializing_if = "Option::is_none")] - reasoning_content: Option, - }, - #[serde(rename = "tool")] - Tool { - content: MessageContent, - tool_call_id: String, - }, - #[serde(rename = "function")] - Function { content: String, name: String }, - #[serde(rename = "developer")] - Developer { - content: MessageContent, - #[serde(skip_serializing_if = "Option::is_none")] - tools: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - name: Option, - }, -} - -#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)] -#[serde(untagged)] -pub enum MessageContent { - Text(String), - Parts(Vec), -} - -impl MessageContent { - /// Returns the text content, cloning only when necessary. - /// For simple text, returns a clone of the string. - /// For parts, concatenates text parts with spaces. - /// Optimized to avoid intermediate Vec allocation. - pub fn to_simple_string(&self) -> String { - match self { - MessageContent::Text(text) => text.clone(), - MessageContent::Parts(parts) => { - // Use fold to build string directly without intermediate Vec allocation - let mut result = String::new(); - let mut first = true; - for part in parts { - if let ContentPart::Text { text } = part { - if !first { - result.push(' '); - } - result.push_str(text); - first = false; - } - } - result - } - } - } - - /// Appends text content directly to a buffer, avoiding intermediate allocations. - /// Returns true if any content was appended. - #[inline] - pub fn append_text_to(&self, buffer: &mut String) -> bool { - match self { - MessageContent::Text(text) => { - if !text.is_empty() { - buffer.push_str(text); - true - } else { - false - } - } - MessageContent::Parts(parts) => { - let mut appended = false; - for part in parts { - if let ContentPart::Text { text } = part { - if !text.is_empty() { - if appended { - buffer.push(' '); - } - buffer.push_str(text); - appended = true; - } - } - } - appended - } - } - } - - /// Returns true if this content contains any non-empty text. - #[inline] - pub fn has_text(&self) -> bool { - match self { - MessageContent::Text(text) => !text.is_empty(), - MessageContent::Parts(parts) => parts - .iter() - .any(|part| matches!(part, ContentPart::Text { text } if !text.is_empty())), - } - } -} - -// ============================================================================ -// Chat Completion Request -// ============================================================================ - -#[derive(Debug, Clone, Deserialize, Serialize, Default, Validate)] -#[validate(schema(function = "validate_chat_cross_parameters"))] -pub struct ChatCompletionRequest { - /// A list of messages comprising the conversation so far - #[validate(custom(function = "validate_messages"))] - pub messages: Vec, - - /// ID of the model to use - #[serde(default = "default_model")] - pub model: String, - - /// Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far - #[serde(skip_serializing_if = "Option::is_none")] - #[validate(range(min = -2.0, max = 2.0))] - pub frequency_penalty: Option, - - /// Deprecated: Replaced by tool_choice - #[serde(skip_serializing_if = "Option::is_none")] - #[deprecated(note = "Use tool_choice instead")] - pub function_call: Option, - - /// Deprecated: Replaced by tools - #[serde(skip_serializing_if = "Option::is_none")] - #[deprecated(note = "Use tools instead")] - pub functions: Option>, - - /// Modify the likelihood of specified tokens appearing in the completion - #[serde(skip_serializing_if = "Option::is_none")] - pub logit_bias: Option>, - - /// Whether to return log probabilities of the output tokens - #[serde(default)] - pub logprobs: bool, - - /// Deprecated: Replaced by max_completion_tokens - #[serde(skip_serializing_if = "Option::is_none")] - #[deprecated(note = "Use max_completion_tokens instead")] - #[validate(range(min = 1))] - pub max_tokens: Option, - - /// An upper bound for the number of tokens that can be generated for a completion - #[serde(skip_serializing_if = "Option::is_none")] - #[validate(range(min = 1))] - pub max_completion_tokens: Option, - - /// Developer-defined tags and values used for filtering completions in the dashboard - #[serde(skip_serializing_if = "Option::is_none")] - pub metadata: Option>, - - /// Output types that you would like the model to generate for this request - #[serde(skip_serializing_if = "Option::is_none")] - pub modalities: Option>, - - /// How many chat completion choices to generate for each input message - #[serde(skip_serializing_if = "Option::is_none")] - #[validate(range(min = 1, max = 10))] - pub n: Option, - - /// Whether to enable parallel function calling during tool use - #[serde(skip_serializing_if = "Option::is_none")] - pub parallel_tool_calls: Option, - - /// Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far - #[serde(skip_serializing_if = "Option::is_none")] - #[validate(range(min = -2.0, max = 2.0))] - pub presence_penalty: Option, - - /// Cache key for prompts (beta feature) - #[serde(skip_serializing_if = "Option::is_none")] - pub prompt_cache_key: Option, - - /// Effort level for reasoning models (low, medium, high) - #[serde(skip_serializing_if = "Option::is_none")] - pub reasoning_effort: Option, - - /// An object specifying the format that the model must output - #[serde(skip_serializing_if = "Option::is_none")] - pub response_format: Option, - - /// Safety identifier for content moderation - #[serde(skip_serializing_if = "Option::is_none")] - pub safety_identifier: Option, - - /// Deprecated: This feature is in Legacy mode - #[serde(skip_serializing_if = "Option::is_none")] - #[deprecated(note = "This feature is in Legacy mode")] - pub seed: Option, - - /// The service tier to use for this request - #[serde(skip_serializing_if = "Option::is_none")] - pub service_tier: Option, - - /// Up to 4 sequences where the API will stop generating further tokens - #[serde(skip_serializing_if = "Option::is_none")] - #[validate(custom(function = "validate_stop"))] - pub stop: Option, - - /// If set, partial message deltas will be sent - #[serde(default)] - pub stream: bool, - - /// Options for streaming response - #[serde(skip_serializing_if = "Option::is_none")] - pub stream_options: Option, - - /// What sampling temperature to use, between 0 and 2 - #[serde(skip_serializing_if = "Option::is_none")] - #[validate(range(min = 0.0, max = 2.0))] - pub temperature: Option, - - /// Controls which (if any) tool is called by the model - #[serde(skip_serializing_if = "Option::is_none")] - pub tool_choice: Option, - - /// A list of tools the model may call - #[serde(skip_serializing_if = "Option::is_none")] - pub tools: Option>, - - /// An integer between 0 and 20 specifying the number of most likely tokens to return - #[serde(skip_serializing_if = "Option::is_none")] - #[validate(range(min = 0, max = 20))] - pub top_logprobs: Option, - - /// An alternative to sampling with temperature - #[serde(skip_serializing_if = "Option::is_none")] - #[validate(custom(function = "validate_top_p_value"))] - pub top_p: Option, - - /// Verbosity level for debugging - #[serde(skip_serializing_if = "Option::is_none")] - pub verbosity: Option, - - // ============================================================================= - // Engine-Specific Sampling Parameters - // ============================================================================= - // These parameters are extensions beyond the OpenAI API specification and - // control model generation behavior in engine-specific ways. - // ============================================================================= - /// Top-k sampling parameter (-1 to disable) - #[serde(skip_serializing_if = "Option::is_none")] - #[validate(custom(function = "validate_top_k_value"))] - pub top_k: Option, - - /// Min-p nucleus sampling parameter - #[serde(skip_serializing_if = "Option::is_none")] - #[validate(range(min = 0.0, max = 1.0))] - pub min_p: Option, - - /// Minimum number of tokens to generate - #[serde(skip_serializing_if = "Option::is_none")] - #[validate(range(min = 1))] - pub min_tokens: Option, - - /// Repetition penalty for reducing repetitive text - #[serde(skip_serializing_if = "Option::is_none")] - #[validate(range(min = 0.0, max = 2.0))] - pub repetition_penalty: Option, - - /// Regex constraint for output generation - #[serde(skip_serializing_if = "Option::is_none")] - pub regex: Option, - - /// EBNF grammar constraint for structured output - #[serde(skip_serializing_if = "Option::is_none")] - pub ebnf: Option, - - /// Specific token IDs to use as stop conditions - #[serde(skip_serializing_if = "Option::is_none")] - pub stop_token_ids: Option>, - - /// Skip trimming stop tokens from output - #[serde(default)] - pub no_stop_trim: bool, - - /// Ignore end-of-sequence tokens during generation - #[serde(default)] - pub ignore_eos: bool, - - /// Continue generating from final assistant message - #[serde(default)] - pub continue_final_message: bool, - - /// Skip special tokens during detokenization - #[serde(default = "default_true")] - pub skip_special_tokens: bool, - - /// Path to LoRA adapter(s) for model customization - #[serde(skip_serializing_if = "Option::is_none")] - pub lora_path: Option, - - /// Session parameters for continual prompting - #[serde(skip_serializing_if = "Option::is_none")] - pub session_params: Option>, - - /// Separate reasoning content from final answer (O1-style models) - #[serde(default = "default_true")] - pub separate_reasoning: bool, - - /// Stream reasoning tokens during generation - #[serde(default = "default_true")] - pub stream_reasoning: bool, - - /// Chat template kwargs - #[serde(skip_serializing_if = "Option::is_none")] - pub chat_template_kwargs: Option>, - - /// Return model hidden states - #[serde(default)] - pub return_hidden_states: bool, - - /// Random seed for sampling for deterministic outputs - #[serde(skip_serializing_if = "Option::is_none")] - pub sampling_seed: Option, -} - -// ============================================================================ -// Validation Functions -// ============================================================================ - -/// Validates messages array is not empty and has valid content -fn validate_messages(messages: &[ChatMessage]) -> Result<(), validator::ValidationError> { - if messages.is_empty() { - return Err(validator::ValidationError::new("messages cannot be empty")); - } - - for msg in messages.iter() { - if let ChatMessage::User { content, .. } = msg { - match content { - MessageContent::Text(text) if text.is_empty() => { - return Err(validator::ValidationError::new( - "message content cannot be empty", - )); - } - MessageContent::Parts(parts) if parts.is_empty() => { - return Err(validator::ValidationError::new( - "message content parts cannot be empty", - )); - } - _ => {} - } - } - } - Ok(()) -} - -/// Schema-level validation for cross-field dependencies -fn validate_chat_cross_parameters( - req: &ChatCompletionRequest, -) -> Result<(), validator::ValidationError> { - // 1. Validate logprobs dependency - if req.top_logprobs.is_some() && !req.logprobs { - let mut e = validator::ValidationError::new("top_logprobs_requires_logprobs"); - e.message = Some("top_logprobs is only allowed when logprobs is enabled".into()); - return Err(e); - } - - // 2. Validate stream_options dependency - if req.stream_options.is_some() && !req.stream { - let mut e = validator::ValidationError::new("stream_options_requires_stream"); - e.message = - Some("The 'stream_options' parameter is only allowed when 'stream' is enabled".into()); - return Err(e); - } - - // 3. Validate token limits - min <= max - if let (Some(min), Some(max)) = (req.min_tokens, req.max_completion_tokens) { - if min > max { - let mut e = validator::ValidationError::new("min_tokens_exceeds_max"); - e.message = Some("min_tokens cannot exceed max_tokens/max_completion_tokens".into()); - return Err(e); - } - } - - // 4. Validate structured output conflicts - let has_json_format = matches!( - req.response_format, - Some(ResponseFormat::JsonObject | ResponseFormat::JsonSchema { .. }) - ); - - if has_json_format && req.regex.is_some() { - let mut e = validator::ValidationError::new("regex_conflicts_with_json"); - e.message = Some("cannot use regex constraint with JSON response format".into()); - return Err(e); - } - - if has_json_format && req.ebnf.is_some() { - let mut e = validator::ValidationError::new("ebnf_conflicts_with_json"); - e.message = Some("cannot use EBNF constraint with JSON response format".into()); - return Err(e); - } - - // 5. Validate mutually exclusive structured output constraints - let constraint_count = [ - req.regex.is_some(), - req.ebnf.is_some(), - matches!(req.response_format, Some(ResponseFormat::JsonSchema { .. })), - ] - .iter() - .filter(|&&x| x) - .count(); - - if constraint_count > 1 { - let mut e = validator::ValidationError::new("multiple_constraints"); - e.message = Some("only one structured output constraint (regex, ebnf, or json_schema) can be active at a time".into()); - return Err(e); - } - - // 6. Validate response format JSON schema name - if let Some(ResponseFormat::JsonSchema { json_schema }) = &req.response_format { - if json_schema.name.is_empty() { - let mut e = validator::ValidationError::new("json_schema_name_empty"); - e.message = Some("JSON schema name cannot be empty".into()); - return Err(e); - } - } - - // 7. Validate tool_choice requires tools (except for "none") - if let Some(ref tool_choice) = req.tool_choice { - let has_tools = req.tools.as_ref().is_some_and(|t| !t.is_empty()); - - // Check if tool_choice is anything other than "none" - let is_some_choice = !matches!(tool_choice, ToolChoice::Value(ToolChoiceValue::None)); - - if is_some_choice && !has_tools { - let mut e = validator::ValidationError::new("tool_choice_requires_tools"); - e.message = Some("Invalid value for 'tool_choice': 'tool_choice' is only allowed when 'tools' are specified.".into()); - return Err(e); - } - - // Additional validation when tools are present - if has_tools { - let tools = req.tools.as_ref().unwrap(); - - match tool_choice { - ToolChoice::Function { function, .. } => { - // Validate that the specified function name exists in tools - let function_exists = tools.iter().any(|tool| { - tool.tool_type == "function" && tool.function.name == function.name - }); - - if !function_exists { - let mut e = - validator::ValidationError::new("tool_choice_function_not_found"); - e.message = Some( - format!( - "Invalid value for 'tool_choice': function '{}' not found in 'tools'.", - function.name - ) - .into(), - ); - return Err(e); - } - } - ToolChoice::AllowedTools { - mode, - tools: allowed_tools, - .. - } => { - // Validate mode is "auto" or "required" - if mode != "auto" && mode != "required" { - let mut e = validator::ValidationError::new("tool_choice_invalid_mode"); - e.message = Some(format!( - "Invalid value for 'tool_choice.mode': must be 'auto' or 'required', got '{}'.", - mode - ).into()); - return Err(e); - } - - // Validate that all ToolReferences are Function type (Chat API only supports function tools) - for tool_ref in allowed_tools { - match tool_ref { - ToolReference::Function { name } => { - // Validate that the function exists in tools array - let tool_exists = tools.iter().any(|tool| { - tool.tool_type == "function" && tool.function.name == *name - }); - - if !tool_exists { - let mut e = validator::ValidationError::new( - "tool_choice_tool_not_found", - ); - e.message = Some( - format!( - "Invalid value for 'tool_choice.tools': tool '{}' not found in 'tools'.", - name - ) - .into(), - ); - return Err(e); - } - } - _ => { - // Chat Completion API only supports function tools in tool_choice - let mut e = validator::ValidationError::new( - "tool_choice_invalid_tool_type", - ); - e.message = Some( - format!( - "Invalid value for 'tool_choice.tools': Chat Completion API only supports function tools, got '{}'.", - tool_ref.identifier() - ) - .into(), - ); - return Err(e); - } - } - } - } - _ => {} - } - } - } - - Ok(()) -} - -// ============================================================================ -// Normalizable Implementation -// ============================================================================ - -impl Normalizable for ChatCompletionRequest { - /// Normalize the request by applying migrations and defaults: - /// 1. Migrate deprecated fields to their replacements - /// 2. Clear deprecated fields and log warnings - /// 3. Apply OpenAI defaults for tool_choice - fn normalize(&mut self) { - // Migrate deprecated max_tokens → max_completion_tokens - #[allow(deprecated)] - if self.max_completion_tokens.is_none() && self.max_tokens.is_some() { - self.max_completion_tokens = self.max_tokens; - self.max_tokens = None; // Clear deprecated field - } - - // Migrate deprecated functions → tools - #[allow(deprecated)] - if self.tools.is_none() && self.functions.is_some() { - tracing::warn!("functions is deprecated, use tools instead"); - self.tools = self.functions.as_ref().map(|functions| { - functions - .iter() - .map(|func| Tool { - tool_type: "function".to_string(), - function: func.clone(), - }) - .collect() - }); - self.functions = None; // Clear deprecated field - } - - // Migrate deprecated function_call → tool_choice - #[allow(deprecated)] - if self.tool_choice.is_none() && self.function_call.is_some() { - tracing::warn!("function_call is deprecated, use tool_choice instead"); - self.tool_choice = self.function_call.as_ref().map(|fc| match fc { - FunctionCall::None => ToolChoice::Value(ToolChoiceValue::None), - FunctionCall::Auto => ToolChoice::Value(ToolChoiceValue::Auto), - FunctionCall::Function { name } => ToolChoice::Function { - tool_type: "function".to_string(), - function: FunctionChoice { name: name.clone() }, - }, - }); - self.function_call = None; // Clear deprecated field - } - - // Apply tool_choice defaults - if self.tool_choice.is_none() { - if let Some(tools) = &self.tools { - let choice_value = if !tools.is_empty() { - ToolChoiceValue::Auto - } else { - ToolChoiceValue::None - }; - self.tool_choice = Some(ToolChoice::Value(choice_value)); - } - // If tools is None, leave tool_choice as None (don't set it) - } - } -} - -// ============================================================================ -// GenerationRequest Trait Implementation -// ============================================================================ - -impl GenerationRequest for ChatCompletionRequest { - fn is_stream(&self) -> bool { - self.stream - } - - fn get_model(&self) -> Option<&str> { - Some(&self.model) - } - - fn extract_text_for_routing(&self) -> String { - // Extract text from messages for routing decisions - // Use a single buffer to avoid intermediate Vec allocations - let mut buffer = String::new(); - let mut has_content = false; - - for msg in &self.messages { - match msg { - ChatMessage::System { content, .. } - | ChatMessage::User { content, .. } - | ChatMessage::Tool { content, .. } - | ChatMessage::Developer { content, .. } => { - if has_content && content.has_text() { - buffer.push(' '); - } - if content.append_text_to(&mut buffer) { - has_content = true; - } - } - ChatMessage::Assistant { - content, - reasoning_content, - .. - } => { - // Append main content - if let Some(c) = content { - if has_content && c.has_text() { - buffer.push(' '); - } - if c.append_text_to(&mut buffer) { - has_content = true; - } - } - // Append reasoning content - if let Some(reasoning) = reasoning_content { - if !reasoning.is_empty() { - if has_content { - buffer.push(' '); - } - buffer.push_str(reasoning); - has_content = true; - } - } - } - ChatMessage::Function { content, .. } => { - if !content.is_empty() { - if has_content { - buffer.push(' '); - } - buffer.push_str(content); - has_content = true; - } - } - } - } - - buffer - } -} - -// ============================================================================ -// Response Types -// ============================================================================ - -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct ChatCompletionResponse { - pub id: String, - pub object: String, // "chat.completion" - pub created: u64, - pub model: String, - pub choices: Vec, - #[serde(skip_serializing_if = "Option::is_none")] - pub usage: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub system_fingerprint: Option, -} - -impl ChatCompletionResponse { - /// Create a new builder for ChatCompletionResponse - pub fn builder( - id: impl Into, - model: impl Into, - ) -> ChatCompletionResponseBuilder { - ChatCompletionResponseBuilder::new(id, model) - } -} - -/// Response message structure for ChatCompletionResponse (different from request ChatMessage) -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct ChatCompletionMessage { - pub role: String, // Always "assistant" for responses - #[serde(skip_serializing_if = "Option::is_none")] - pub content: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub tool_calls: Option>, - pub reasoning_content: Option, - // Note: function_call is deprecated and not included - // Note: refusal, annotations, audio are not added yet -} - -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct ChatChoice { - pub index: u32, - pub message: ChatCompletionMessage, - #[serde(skip_serializing_if = "Option::is_none")] - pub logprobs: Option, - pub finish_reason: Option, // "stop", "length", "tool_calls", "content_filter", "function_call" - /// Information about which stop condition was matched - #[serde(skip_serializing_if = "Option::is_none")] - pub matched_stop: Option, // Can be string or integer - /// Hidden states from the model (SGLang extension) - #[serde(skip_serializing_if = "Option::is_none")] - pub hidden_states: Option>, -} - -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct ChatCompletionStreamResponse { - pub id: String, - pub object: String, // "chat.completion.chunk" - pub created: u64, - pub model: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub system_fingerprint: Option, - pub choices: Vec, - #[serde(skip_serializing_if = "Option::is_none")] - pub usage: Option, -} - -impl ChatCompletionStreamResponse { - /// Create a new builder for ChatCompletionStreamResponse - pub fn builder( - id: impl Into, - model: impl Into, - ) -> ChatCompletionStreamResponseBuilder { - ChatCompletionStreamResponseBuilder::new(id, model) - } -} - -/// Delta structure for streaming chat completion responses -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct ChatMessageDelta { - #[serde(skip_serializing_if = "Option::is_none")] - pub role: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub content: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub tool_calls: Option>, - pub reasoning_content: Option, -} - -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct ChatStreamChoice { - pub index: u32, - pub delta: ChatMessageDelta, - pub logprobs: Option, - pub finish_reason: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub matched_stop: Option, -} diff --git a/sgl-model-gateway/src/protocols/classify.rs b/sgl-model-gateway/src/protocols/classify.rs deleted file mode 100644 index 492bf88b1..000000000 --- a/sgl-model-gateway/src/protocols/classify.rs +++ /dev/null @@ -1,121 +0,0 @@ -//! Classify API protocol definitions. -//! -//! This module defines the request and response types for the `/v1/classify` API, -//! which is compatible with vLLM's classification endpoint. -//! -//! Classification reuses the embedding backend - the scheduler returns logits as -//! "embeddings", and the classify layer applies softmax + label mapping. - -use serde::{Deserialize, Serialize}; -use serde_json::Value; - -use super::common::{GenerationRequest, UsageInfo}; - -// ============================================================================ -// Classify API -// ============================================================================ - -/// Classification request - compatible with vLLM's /v1/classify API -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct ClassifyRequest { - /// ID of the model to use - pub model: String, - - /// Input can be a string, array of strings, or token IDs - /// - Single string: "text to classify" - /// - Array of strings: ["text1", "text2"] - /// - Token IDs: [1, 2, 3] (advanced usage) - pub input: Value, - - /// Optional user identifier - #[serde(skip_serializing_if = "Option::is_none")] - pub user: Option, - - /// SGLang extension: request id for tracking - #[serde(skip_serializing_if = "Option::is_none")] - pub rid: Option, - - /// SGLang extension: request priority - #[serde(skip_serializing_if = "Option::is_none")] - pub priority: Option, - - /// SGLang extension: enable/disable logging of metrics - #[serde(skip_serializing_if = "Option::is_none")] - pub log_metrics: Option, -} - -impl GenerationRequest for ClassifyRequest { - fn is_stream(&self) -> bool { - false // Classification is always non-streaming - } - - fn get_model(&self) -> Option<&str> { - Some(&self.model) - } - - fn extract_text_for_routing(&self) -> String { - match &self.input { - Value::String(s) => s.clone(), - Value::Array(arr) => arr - .iter() - .filter_map(|v| v.as_str()) - .collect::>() - .join(" "), - _ => String::new(), - } - } -} - -// ============================================================================ -// Classify Response -// ============================================================================ - -/// Single classification result -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ClassifyData { - /// Index of this result (for batch requests) - pub index: u32, - /// Predicted class label (from id2label mapping) - pub label: String, - /// Probability distribution over all classes (softmax of logits) - pub probs: Vec, - /// Number of classes - pub num_classes: u32, -} - -/// Classification response -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ClassifyResponse { - /// Unique request ID (format: "classify-{uuid}") - pub id: String, - /// Always "list" - pub object: String, - /// Unix timestamp (seconds since epoch) - pub created: u64, - /// Model name - pub model: String, - /// Classification results (one per input in batch) - pub data: Vec, - /// Token usage info - pub usage: UsageInfo, -} - -impl ClassifyResponse { - /// Create a new ClassifyResponse with the given data - pub fn new( - id: String, - model: String, - created: u64, - data: Vec, - usage: UsageInfo, - ) -> Self { - Self { - id, - object: "list".to_string(), - created, - model, - data, - usage, - } - } -} diff --git a/sgl-model-gateway/src/protocols/common.rs b/sgl-model-gateway/src/protocols/common.rs deleted file mode 100644 index 83e2d238a..000000000 --- a/sgl-model-gateway/src/protocols/common.rs +++ /dev/null @@ -1,524 +0,0 @@ -use std::collections::HashMap; - -use serde::{Deserialize, Serialize}; -use serde_json::Value; -use validator; - -use super::UNKNOWN_MODEL_ID; - -// ============================================================================ -// Default value helpers -// ============================================================================ - -/// Default model value when not specified -pub(crate) fn default_model() -> String { - UNKNOWN_MODEL_ID.to_string() -} - -/// Helper function for serde default value (returns true) -pub fn default_true() -> bool { - true -} - -// ============================================================================ -// GenerationRequest Trait -// ============================================================================ - -/// Trait for unified access to generation request properties -/// Implemented by ChatCompletionRequest, CompletionRequest, GenerateRequest, -/// EmbeddingRequest, RerankRequest, and ResponsesRequest -pub trait GenerationRequest: Send + Sync { - /// Check if the request is for streaming - fn is_stream(&self) -> bool; - - /// Get the model name if specified - fn get_model(&self) -> Option<&str>; - - /// Extract text content for routing decisions - fn extract_text_for_routing(&self) -> String; -} - -// ============================================================================ -// String/Array Utilities -// ============================================================================ - -/// A type that can be either a single string or an array of strings -#[derive(Debug, Clone, PartialEq, Deserialize, Serialize)] -#[serde(untagged)] -pub enum StringOrArray { - String(String), - Array(Vec), -} - -impl StringOrArray { - /// Get the number of items in the StringOrArray - pub fn len(&self) -> usize { - match self { - StringOrArray::String(_) => 1, - StringOrArray::Array(arr) => arr.len(), - } - } - - /// Check if the StringOrArray is empty - pub fn is_empty(&self) -> bool { - match self { - StringOrArray::String(s) => s.is_empty(), - StringOrArray::Array(arr) => arr.is_empty(), - } - } - - /// Convert to a vector of strings (clones the data) - pub fn to_vec(&self) -> Vec { - match self { - StringOrArray::String(s) => vec![s.clone()], - StringOrArray::Array(arr) => arr.clone(), - } - } - - /// Returns an iterator over string references without cloning. - /// Use this instead of `to_vec()` when you only need to iterate. - pub fn iter(&self) -> StringOrArrayIter<'_> { - StringOrArrayIter { - inner: self, - index: 0, - } - } - - /// Returns the first string, or None if empty - pub fn first(&self) -> Option<&str> { - match self { - StringOrArray::String(s) => { - if s.is_empty() { - None - } else { - Some(s) - } - } - StringOrArray::Array(arr) => arr.first().map(|s| s.as_str()), - } - } -} - -/// Iterator over StringOrArray that yields string references without cloning -pub struct StringOrArrayIter<'a> { - inner: &'a StringOrArray, - index: usize, -} - -impl<'a> Iterator for StringOrArrayIter<'a> { - type Item = &'a str; - - fn next(&mut self) -> Option { - match self.inner { - StringOrArray::String(s) => { - if self.index == 0 { - self.index = 1; - Some(s.as_str()) - } else { - None - } - } - StringOrArray::Array(arr) => { - if self.index < arr.len() { - let item = &arr[self.index]; - self.index += 1; - Some(item.as_str()) - } else { - None - } - } - } - } - - fn size_hint(&self) -> (usize, Option) { - let remaining = match self.inner { - StringOrArray::String(_) => 1 - self.index, - StringOrArray::Array(arr) => arr.len() - self.index, - }; - (remaining, Some(remaining)) - } -} - -impl<'a> ExactSizeIterator for StringOrArrayIter<'a> {} - -/// Validates stop sequences (max 4, non-empty strings) -/// Used by both ChatCompletionRequest and ResponsesRequest -pub fn validate_stop(stop: &StringOrArray) -> Result<(), validator::ValidationError> { - match stop { - StringOrArray::String(s) => { - if s.is_empty() { - return Err(validator::ValidationError::new( - "stop sequences cannot be empty", - )); - } - } - StringOrArray::Array(arr) => { - if arr.len() > 4 { - return Err(validator::ValidationError::new( - "maximum 4 stop sequences allowed", - )); - } - for s in arr { - if s.is_empty() { - return Err(validator::ValidationError::new( - "stop sequences cannot be empty", - )); - } - } - } - } - Ok(()) -} - -// ============================================================================ -// Content Parts (for multimodal messages) -// ============================================================================ - -#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)] -#[serde(tag = "type")] -pub enum ContentPart { - #[serde(rename = "text")] - Text { text: String }, - #[serde(rename = "image_url")] - ImageUrl { image_url: ImageUrl }, - #[serde(rename = "video_url")] - VideoUrl { video_url: VideoUrl }, -} - -#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)] -pub struct ImageUrl { - pub url: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub detail: Option, // "auto", "low", or "high" -} - -#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)] -pub struct VideoUrl { - pub url: String, -} - -// ============================================================================ -// Response Format (for structured outputs) -// ============================================================================ - -#[derive(Debug, Clone, Deserialize, Serialize)] -#[serde(tag = "type")] -pub enum ResponseFormat { - #[serde(rename = "text")] - Text, - #[serde(rename = "json_object")] - JsonObject, - #[serde(rename = "json_schema")] - JsonSchema { json_schema: JsonSchemaFormat }, -} - -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct JsonSchemaFormat { - pub name: String, - pub schema: Value, - #[serde(skip_serializing_if = "Option::is_none")] - pub strict: Option, -} - -// ============================================================================ -// Streaming -// ============================================================================ - -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct StreamOptions { - #[serde(skip_serializing_if = "Option::is_none")] - pub include_usage: Option, -} - -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct ToolCallDelta { - pub index: u32, - #[serde(skip_serializing_if = "Option::is_none")] - pub id: Option, - #[serde(skip_serializing_if = "Option::is_none")] - #[serde(rename = "type")] - pub tool_type: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub function: Option, -} - -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct FunctionCallDelta { - #[serde(skip_serializing_if = "Option::is_none")] - pub name: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub arguments: Option, -} - -// ============================================================================ -// Tools and Function Calling -// ============================================================================ - -/// Tool choice value for simple string options -#[derive(Debug, Clone, Deserialize, Serialize)] -#[serde(rename_all = "snake_case")] -pub enum ToolChoiceValue { - Auto, - Required, - None, -} - -/// Tool choice for both Chat Completion and Responses APIs -#[derive(Debug, Clone, Deserialize, Serialize)] -#[serde(untagged)] -pub enum ToolChoice { - Value(ToolChoiceValue), - Function { - #[serde(rename = "type")] - tool_type: String, // "function" - function: FunctionChoice, - }, - AllowedTools { - #[serde(rename = "type")] - tool_type: String, // "allowed_tools" - mode: String, // "auto" | "required" TODO: need validation - tools: Vec, - }, -} - -impl Default for ToolChoice { - fn default() -> Self { - Self::Value(ToolChoiceValue::Auto) - } -} - -impl ToolChoice { - /// Serialize tool_choice to string for ResponsesResponse - /// - /// Returns the JSON-serialized tool_choice or "auto" as default - pub fn serialize_to_string(tool_choice: &Option) -> String { - tool_choice - .as_ref() - .map(|tc| serde_json::to_string(tc).unwrap_or_else(|_| "auto".to_string())) - .unwrap_or_else(|| "auto".to_string()) - } -} - -/// Function choice specification for ToolChoice::Function -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct FunctionChoice { - pub name: String, -} - -/// Tool reference for ToolChoice::AllowedTools -/// -/// Represents a reference to a specific tool in the allowed_tools array. -/// Different tool types have different required fields. -#[derive(Debug, Clone, Deserialize, Serialize)] -#[serde(tag = "type")] -#[serde(rename_all = "snake_case")] -pub enum ToolReference { - /// Reference to a function tool - #[serde(rename = "function")] - Function { name: String }, - - /// Reference to an MCP tool - #[serde(rename = "mcp")] - Mcp { - server_label: String, - #[serde(skip_serializing_if = "Option::is_none")] - name: Option, - }, - - /// File search hosted tool - #[serde(rename = "file_search")] - FileSearch, - - /// Web search preview hosted tool - #[serde(rename = "web_search_preview")] - WebSearchPreview, - - /// Computer use preview hosted tool - #[serde(rename = "computer_use_preview")] - ComputerUsePreview, - - /// Code interpreter hosted tool - #[serde(rename = "code_interpreter")] - CodeInterpreter, - - /// Image generation hosted tool - #[serde(rename = "image_generation")] - ImageGeneration, -} - -impl ToolReference { - /// Get a unique identifier for this tool reference - pub fn identifier(&self) -> String { - match self { - ToolReference::Function { name } => format!("function:{}", name), - ToolReference::Mcp { server_label, name } => { - if let Some(n) = name { - format!("mcp:{}:{}", server_label, n) - } else { - format!("mcp:{}", server_label) - } - } - ToolReference::FileSearch => "file_search".to_string(), - ToolReference::WebSearchPreview => "web_search_preview".to_string(), - ToolReference::ComputerUsePreview => "computer_use_preview".to_string(), - ToolReference::CodeInterpreter => "code_interpreter".to_string(), - ToolReference::ImageGeneration => "image_generation".to_string(), - } - } - - /// Get the tool name if this is a function tool - pub fn function_name(&self) -> Option<&str> { - match self { - ToolReference::Function { name } => Some(name.as_str()), - _ => None, - } - } -} - -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct Tool { - #[serde(rename = "type")] - pub tool_type: String, // "function" - pub function: Function, -} - -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct Function { - pub name: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub description: Option, - pub parameters: Value, // JSON Schema - /// Whether to enable strict schema adherence (OpenAI structured outputs) - #[serde(skip_serializing_if = "Option::is_none")] - pub strict: Option, -} - -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct ToolCall { - pub id: String, - #[serde(rename = "type")] - pub tool_type: String, // "function" - pub function: FunctionCallResponse, -} - -#[derive(Debug, Clone, Deserialize, Serialize)] -#[serde(untagged)] -pub enum FunctionCall { - None, - Auto, - Function { name: String }, -} - -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct FunctionCallResponse { - pub name: String, - #[serde(default)] - pub arguments: Option, // JSON string -} - -// ============================================================================ -// Usage and Logging -// ============================================================================ - -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct Usage { - pub prompt_tokens: u32, - pub completion_tokens: u32, - pub total_tokens: u32, - #[serde(skip_serializing_if = "Option::is_none")] - pub completion_tokens_details: Option, -} - -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct CompletionTokensDetails { - pub reasoning_tokens: Option, -} - -/// Usage information (used by rerank and other endpoints) -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct UsageInfo { - pub prompt_tokens: u32, - pub completion_tokens: u32, - pub total_tokens: u32, - #[serde(skip_serializing_if = "Option::is_none")] - pub reasoning_tokens: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub prompt_tokens_details: Option, -} - -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct PromptTokenUsageInfo { - pub cached_tokens: u32, -} - -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct LogProbs { - pub tokens: Vec, - pub token_logprobs: Vec>, - pub top_logprobs: Vec>>, - pub text_offset: Vec, -} - -#[derive(Debug, Clone, Deserialize, Serialize)] -#[serde(untagged)] -pub enum ChatLogProbs { - Detailed { - #[serde(skip_serializing_if = "Option::is_none")] - content: Option>, - }, - Raw(Value), -} - -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct ChatLogProbsContent { - pub token: String, - pub logprob: f32, - pub bytes: Option>, - pub top_logprobs: Vec, -} - -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct TopLogProb { - pub token: String, - pub logprob: f32, - pub bytes: Option>, -} - -// ============================================================================ -// Error Types -// ============================================================================ - -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct ErrorResponse { - pub error: ErrorDetail, -} - -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct ErrorDetail { - pub message: String, - #[serde(rename = "type")] - pub error_type: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub param: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub code: Option, -} - -// ============================================================================ -// Input Types -// ============================================================================ - -#[derive(Debug, Clone, Deserialize, Serialize)] -#[serde(untagged)] -pub enum InputIds { - Single(Vec), - Batch(Vec>), -} - -/// LoRA adapter path - can be single path or batch of paths (SGLang extension) -#[derive(Debug, Clone, Deserialize, Serialize)] -#[serde(untagged)] -pub enum LoRAPath { - Single(Option), - Batch(Vec>), -} diff --git a/sgl-model-gateway/src/protocols/completion.rs b/sgl-model-gateway/src/protocols/completion.rs deleted file mode 100644 index c6a4f638a..000000000 --- a/sgl-model-gateway/src/protocols/completion.rs +++ /dev/null @@ -1,214 +0,0 @@ -use std::collections::HashMap; - -use serde::{Deserialize, Serialize}; -use serde_json::{Map, Value}; - -use super::common::*; - -// ============================================================================ -// Completions API (v1/completions) - DEPRECATED but still supported -// ============================================================================ - -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct CompletionRequest { - /// ID of the model to use (required for OpenAI, optional for some implementations, such as SGLang) - pub model: String, - - /// The prompt(s) to generate completions for - pub prompt: StringOrArray, - - /// The suffix that comes after a completion of inserted text - #[serde(skip_serializing_if = "Option::is_none")] - pub suffix: Option, - - /// The maximum number of tokens to generate - #[serde(skip_serializing_if = "Option::is_none")] - pub max_tokens: Option, - - /// What sampling temperature to use, between 0 and 2 - #[serde(skip_serializing_if = "Option::is_none")] - pub temperature: Option, - - /// An alternative to sampling with temperature (nucleus sampling) - #[serde(skip_serializing_if = "Option::is_none")] - pub top_p: Option, - - /// How many completions to generate for each prompt - #[serde(skip_serializing_if = "Option::is_none")] - pub n: Option, - - /// Whether to stream back partial progress - #[serde(default)] - pub stream: bool, - - /// Options for streaming response - #[serde(skip_serializing_if = "Option::is_none")] - pub stream_options: Option, - - /// Include the log probabilities on the logprobs most likely tokens - #[serde(skip_serializing_if = "Option::is_none")] - pub logprobs: Option, - - /// Echo back the prompt in addition to the completion - #[serde(default)] - pub echo: bool, - - /// Up to 4 sequences where the API will stop generating further tokens - #[serde(skip_serializing_if = "Option::is_none")] - pub stop: Option, - - /// Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far - #[serde(skip_serializing_if = "Option::is_none")] - pub presence_penalty: Option, - - /// Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far - #[serde(skip_serializing_if = "Option::is_none")] - pub frequency_penalty: Option, - - /// Generates best_of completions server-side and returns the "best" - #[serde(skip_serializing_if = "Option::is_none")] - pub best_of: Option, - - /// Modify the likelihood of specified tokens appearing in the completion - #[serde(skip_serializing_if = "Option::is_none")] - pub logit_bias: Option>, - - /// A unique identifier representing your end-user - #[serde(skip_serializing_if = "Option::is_none")] - pub user: Option, - - /// If specified, our system will make a best effort to sample deterministically - #[serde(skip_serializing_if = "Option::is_none")] - pub seed: Option, - - // -------- Engine Specific Sampling Parameters -------- - /// Top-k sampling parameter (-1 to disable) - #[serde(skip_serializing_if = "Option::is_none")] - pub top_k: Option, - - /// Min-p nucleus sampling parameter - #[serde(skip_serializing_if = "Option::is_none")] - pub min_p: Option, - - /// Minimum number of tokens to generate - #[serde(skip_serializing_if = "Option::is_none")] - pub min_tokens: Option, - - /// Repetition penalty for reducing repetitive text - #[serde(skip_serializing_if = "Option::is_none")] - pub repetition_penalty: Option, - - /// Regex constraint for output generation - #[serde(skip_serializing_if = "Option::is_none")] - pub regex: Option, - - /// EBNF grammar constraint for structured output - #[serde(skip_serializing_if = "Option::is_none")] - pub ebnf: Option, - - /// JSON schema constraint for structured output - #[serde(skip_serializing_if = "Option::is_none")] - pub json_schema: Option, - - /// Specific token IDs to use as stop conditions - #[serde(skip_serializing_if = "Option::is_none")] - pub stop_token_ids: Option>, - - /// Skip trimming stop tokens from output - #[serde(default)] - pub no_stop_trim: bool, - - /// Ignore end-of-sequence tokens during generation - #[serde(default)] - pub ignore_eos: bool, - - /// Skip special tokens during detokenization - #[serde(default = "default_true")] - pub skip_special_tokens: bool, - - /// Path to LoRA adapter(s) for model customization - #[serde(skip_serializing_if = "Option::is_none")] - pub lora_path: Option, - - /// Session parameters for continual prompting - #[serde(skip_serializing_if = "Option::is_none")] - pub session_params: Option>, - - /// Return model hidden states - #[serde(default)] - pub return_hidden_states: bool, - - /// Sampling seed for deterministic outputs - #[serde(skip_serializing_if = "Option::is_none")] - pub sampling_seed: Option, - - /// Additional fields including bootstrap info for PD routing - #[serde(flatten)] - pub other: Map, -} - -impl GenerationRequest for CompletionRequest { - fn is_stream(&self) -> bool { - self.stream - } - - fn get_model(&self) -> Option<&str> { - Some(&self.model) - } - - fn extract_text_for_routing(&self) -> String { - match &self.prompt { - StringOrArray::String(s) => s.clone(), - StringOrArray::Array(v) => v.join(" "), - } - } -} - -// ============================================================================ -// Response Types -// ============================================================================ - -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct CompletionResponse { - pub id: String, - pub object: String, // "text_completion" - pub created: u64, - pub model: String, - pub choices: Vec, - #[serde(skip_serializing_if = "Option::is_none")] - pub usage: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub system_fingerprint: Option, -} - -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct CompletionChoice { - pub text: String, - pub index: u32, - #[serde(skip_serializing_if = "Option::is_none")] - pub logprobs: Option, - pub finish_reason: Option, // "stop", "length", "content_filter", etc. - /// Information about which stop condition was matched - #[serde(skip_serializing_if = "Option::is_none")] - pub matched_stop: Option, // Can be string or integer -} - -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct CompletionStreamResponse { - pub id: String, - pub object: String, // "text_completion" - pub created: u64, - pub choices: Vec, - pub model: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub system_fingerprint: Option, -} - -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct CompletionStreamChoice { - pub text: String, - pub index: u32, - #[serde(skip_serializing_if = "Option::is_none")] - pub logprobs: Option, - pub finish_reason: Option, -} diff --git a/sgl-model-gateway/src/protocols/embedding.rs b/sgl-model-gateway/src/protocols/embedding.rs deleted file mode 100644 index 12e3daf19..000000000 --- a/sgl-model-gateway/src/protocols/embedding.rs +++ /dev/null @@ -1,76 +0,0 @@ -use serde::{Deserialize, Serialize}; -use serde_json::Value; - -use super::common::{GenerationRequest, UsageInfo}; - -// ============================================================================ -// Embedding API -// ============================================================================ - -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct EmbeddingRequest { - /// ID of the model to use - pub model: String, - - /// Input can be a string, array of strings, tokens, or batch inputs - pub input: Value, - - /// Optional encoding format (e.g., "float", "base64") - #[serde(skip_serializing_if = "Option::is_none")] - pub encoding_format: Option, - - /// Optional user identifier - #[serde(skip_serializing_if = "Option::is_none")] - pub user: Option, - - /// Optional number of dimensions for the embedding - #[serde(skip_serializing_if = "Option::is_none")] - pub dimensions: Option, - - /// SGLang extension: request id for tracking - #[serde(skip_serializing_if = "Option::is_none")] - pub rid: Option, - - /// SGLang extension: enable/disable logging of metrics for this request - #[serde(skip_serializing_if = "Option::is_none")] - pub log_metrics: Option, -} - -impl GenerationRequest for EmbeddingRequest { - fn is_stream(&self) -> bool { - // Embeddings are non-streaming - false - } - - fn get_model(&self) -> Option<&str> { - Some(&self.model) - } - - fn extract_text_for_routing(&self) -> String { - // Best effort: extract text content for routing decisions - match &self.input { - Value::String(s) => s.clone(), - Value::Array(arr) => arr - .iter() - .filter_map(|v| v.as_str()) - .collect::>() - .join(" "), - _ => String::new(), - } - } -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct EmbeddingObject { - pub object: String, // "embedding" - pub embedding: Vec, - pub index: u32, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct EmbeddingResponse { - pub object: String, // "list" - pub data: Vec, - pub model: String, - pub usage: UsageInfo, -} diff --git a/sgl-model-gateway/src/protocols/event_types.rs b/sgl-model-gateway/src/protocols/event_types.rs deleted file mode 100644 index c36dc5d45..000000000 --- a/sgl-model-gateway/src/protocols/event_types.rs +++ /dev/null @@ -1,228 +0,0 @@ -use std::fmt; - -/// Response lifecycle events -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub enum ResponseEvent { - Created, - InProgress, - Completed, -} - -impl ResponseEvent { - pub const CREATED: &'static str = "response.created"; - pub const IN_PROGRESS: &'static str = "response.in_progress"; - pub const COMPLETED: &'static str = "response.completed"; - - pub const fn as_str(&self) -> &'static str { - match self { - Self::Created => Self::CREATED, - Self::InProgress => Self::IN_PROGRESS, - Self::Completed => Self::COMPLETED, - } - } -} - -impl fmt::Display for ResponseEvent { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(self.as_str()) - } -} - -/// Output item events for streaming -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub enum OutputItemEvent { - Added, - Done, - Delta, -} - -impl OutputItemEvent { - pub const ADDED: &'static str = "response.output_item.added"; - pub const DONE: &'static str = "response.output_item.done"; - pub const DELTA: &'static str = "response.output_item.delta"; - - pub const fn as_str(&self) -> &'static str { - match self { - Self::Added => Self::ADDED, - Self::Done => Self::DONE, - Self::Delta => Self::DELTA, - } - } -} - -impl fmt::Display for OutputItemEvent { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(self.as_str()) - } -} - -/// Function call argument streaming events -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub enum FunctionCallEvent { - ArgumentsDelta, - ArgumentsDone, -} - -impl FunctionCallEvent { - pub const ARGUMENTS_DELTA: &'static str = "response.function_call_arguments.delta"; - pub const ARGUMENTS_DONE: &'static str = "response.function_call_arguments.done"; - - pub const fn as_str(&self) -> &'static str { - match self { - Self::ArgumentsDelta => Self::ARGUMENTS_DELTA, - Self::ArgumentsDone => Self::ARGUMENTS_DONE, - } - } -} - -impl fmt::Display for FunctionCallEvent { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(self.as_str()) - } -} - -/// Content part streaming events -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub enum ContentPartEvent { - Added, - Done, -} - -impl ContentPartEvent { - pub const ADDED: &'static str = "response.content_part.added"; - pub const DONE: &'static str = "response.content_part.done"; - - pub const fn as_str(&self) -> &'static str { - match self { - Self::Added => Self::ADDED, - Self::Done => Self::DONE, - } - } -} - -impl fmt::Display for ContentPartEvent { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(self.as_str()) - } -} - -/// Output text streaming events -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub enum OutputTextEvent { - Delta, - Done, -} - -impl OutputTextEvent { - pub const DELTA: &'static str = "response.output_text.delta"; - pub const DONE: &'static str = "response.output_text.done"; - - pub const fn as_str(&self) -> &'static str { - match self { - Self::Delta => Self::DELTA, - Self::Done => Self::DONE, - } - } -} - -impl fmt::Display for OutputTextEvent { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(self.as_str()) - } -} - -// ============================================================================ -// MCP Events -// ============================================================================ - -/// MCP (Model Context Protocol) call events -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub enum McpEvent { - CallArgumentsDelta, - CallArgumentsDone, - CallInProgress, - CallCompleted, - CallFailed, - ListToolsInProgress, - ListToolsCompleted, -} - -impl McpEvent { - pub const CALL_ARGUMENTS_DELTA: &'static str = "response.mcp_call_arguments.delta"; - pub const CALL_ARGUMENTS_DONE: &'static str = "response.mcp_call_arguments.done"; - pub const CALL_IN_PROGRESS: &'static str = "response.mcp_call.in_progress"; - pub const CALL_COMPLETED: &'static str = "response.mcp_call.completed"; - pub const CALL_FAILED: &'static str = "response.mcp_call.failed"; - pub const LIST_TOOLS_IN_PROGRESS: &'static str = "response.mcp_list_tools.in_progress"; - pub const LIST_TOOLS_COMPLETED: &'static str = "response.mcp_list_tools.completed"; - - pub const fn as_str(&self) -> &'static str { - match self { - Self::CallArgumentsDelta => Self::CALL_ARGUMENTS_DELTA, - Self::CallArgumentsDone => Self::CALL_ARGUMENTS_DONE, - Self::CallInProgress => Self::CALL_IN_PROGRESS, - Self::CallCompleted => Self::CALL_COMPLETED, - Self::CallFailed => Self::CALL_FAILED, - Self::ListToolsInProgress => Self::LIST_TOOLS_IN_PROGRESS, - Self::ListToolsCompleted => Self::LIST_TOOLS_COMPLETED, - } - } -} - -impl fmt::Display for McpEvent { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(self.as_str()) - } -} - -/// Item type discriminators used in output items -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub enum ItemType { - FunctionCall, - FunctionToolCall, - McpCall, - Function, - McpListTools, -} - -impl ItemType { - pub const FUNCTION_CALL: &'static str = "function_call"; - pub const FUNCTION_TOOL_CALL: &'static str = "function_tool_call"; - pub const MCP_CALL: &'static str = "mcp_call"; - pub const FUNCTION: &'static str = "function"; - pub const MCP_LIST_TOOLS: &'static str = "mcp_list_tools"; - - pub const fn as_str(&self) -> &'static str { - match self { - Self::FunctionCall => Self::FUNCTION_CALL, - Self::FunctionToolCall => Self::FUNCTION_TOOL_CALL, - Self::McpCall => Self::MCP_CALL, - Self::Function => Self::FUNCTION, - Self::McpListTools => Self::MCP_LIST_TOOLS, - } - } - - /// Check if this is a function call variant (FunctionCall or FunctionToolCall) - pub const fn is_function_call(&self) -> bool { - matches!(self, Self::FunctionCall | Self::FunctionToolCall) - } -} - -impl fmt::Display for ItemType { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(self.as_str()) - } -} - -/// Check if an event type string matches any response lifecycle event -pub fn is_response_event(event_type: &str) -> bool { - matches!( - event_type, - ResponseEvent::CREATED | ResponseEvent::IN_PROGRESS | ResponseEvent::COMPLETED - ) -} - -/// Check if an item type string is a function call variant -pub fn is_function_call_type(item_type: &str) -> bool { - item_type == ItemType::FUNCTION_CALL || item_type == ItemType::FUNCTION_TOOL_CALL -} diff --git a/sgl-model-gateway/src/protocols/generate.rs b/sgl-model-gateway/src/protocols/generate.rs deleted file mode 100644 index d5819095a..000000000 --- a/sgl-model-gateway/src/protocols/generate.rs +++ /dev/null @@ -1,297 +0,0 @@ -use std::collections::HashMap; - -use serde::{Deserialize, Serialize}; -use serde_json::Value; -use validator::Validate; - -use super::{ - common::{default_true, GenerationRequest, InputIds}, - sampling_params::SamplingParams, -}; -use crate::protocols::validated::Normalizable; - -// ============================================================================ -// SGLang Generate API (native format) -// ============================================================================ - -#[derive(Clone, Debug, Serialize, Deserialize, Validate)] -#[validate(schema(function = "validate_generate_request"))] -pub struct GenerateRequest { - /// Text input - SGLang native format - #[serde(skip_serializing_if = "Option::is_none")] - pub text: Option, - - pub model: Option, - - /// Input IDs for tokenized input - #[serde(skip_serializing_if = "Option::is_none")] - pub input_ids: Option, - - /// Input embeddings for direct embedding input - /// Can be a 2D array (single request) or 3D array (batch of requests) - /// Placeholder for future use - #[serde(skip_serializing_if = "Option::is_none")] - pub input_embeds: Option, - - /// Image input data - /// Can be an image instance, file name, URL, or base64 encoded string - /// Supports single images, lists of images, or nested lists for batch processing - /// Placeholder for future use - #[serde(skip_serializing_if = "Option::is_none")] - pub image_data: Option, - - /// Video input data - /// Can be a file name, URL, or base64 encoded string - /// Supports single videos, lists of videos, or nested lists for batch processing - /// Placeholder for future use - #[serde(skip_serializing_if = "Option::is_none")] - pub video_data: Option, - - /// Audio input data - /// Can be a file name, URL, or base64 encoded string - /// Supports single audio files, lists of audio, or nested lists for batch processing - /// Placeholder for future use - #[serde(skip_serializing_if = "Option::is_none")] - pub audio_data: Option, - - /// Sampling parameters (sglang style) - #[serde(skip_serializing_if = "Option::is_none")] - pub sampling_params: Option, - - /// Whether to return logprobs - #[serde(skip_serializing_if = "Option::is_none")] - pub return_logprob: Option, - - /// If return logprobs, the start location in the prompt for returning logprobs. - #[serde(skip_serializing_if = "Option::is_none")] - pub logprob_start_len: Option, - - /// If return logprobs, the number of top logprobs to return at each position. - #[serde(skip_serializing_if = "Option::is_none")] - pub top_logprobs_num: Option, - - /// If return logprobs, the token ids to return logprob for. - #[serde(skip_serializing_if = "Option::is_none")] - pub token_ids_logprob: Option>, - - /// Whether to detokenize tokens in text in the returned logprobs. - #[serde(default)] - pub return_text_in_logprobs: bool, - - /// Whether to stream the response - #[serde(default)] - pub stream: bool, - - /// Whether to log metrics for this request (e.g. health_generate calls do not log metrics) - #[serde(default = "default_true")] - pub log_metrics: bool, - - /// Return model hidden states - #[serde(default)] - pub return_hidden_states: bool, - - /// The modalities of the image data [image, multi-images, video] - #[serde(skip_serializing_if = "Option::is_none")] - pub modalities: Option>, - - /// Session parameters for continual prompting - #[serde(skip_serializing_if = "Option::is_none")] - pub session_params: Option>, - - /// Path to LoRA adapter(s) for model customization - #[serde(skip_serializing_if = "Option::is_none")] - pub lora_path: Option, - - /// LoRA adapter ID (if pre-loaded) - #[serde(skip_serializing_if = "Option::is_none")] - pub lora_id: Option, - - /// Custom logit processor for advanced sampling control. Must be a serialized instance - /// of `CustomLogitProcessor` in python/sglang/srt/sampling/custom_logit_processor.py - /// Use the processor's `to_str()` method to generate the serialized string. - #[serde(skip_serializing_if = "Option::is_none")] - pub custom_logit_processor: Option, - - /// For disaggregated inference - #[serde(skip_serializing_if = "Option::is_none")] - pub bootstrap_host: Option, - - /// For disaggregated inference - #[serde(skip_serializing_if = "Option::is_none")] - pub bootstrap_port: Option, - - /// For disaggregated inference - #[serde(skip_serializing_if = "Option::is_none")] - pub bootstrap_room: Option, - - /// For disaggregated inference - #[serde(skip_serializing_if = "Option::is_none")] - pub bootstrap_pair_key: Option, - - /// Data parallel rank routing - #[serde(skip_serializing_if = "Option::is_none")] - pub data_parallel_rank: Option, - - /// Background response - #[serde(default)] - pub background: bool, - - /// Conversation ID for tracking - #[serde(skip_serializing_if = "Option::is_none")] - pub conversation_id: Option, - - /// Priority for the request - #[serde(skip_serializing_if = "Option::is_none")] - pub priority: Option, - - /// Extra key for classifying the request (e.g. cache_salt) - #[serde(skip_serializing_if = "Option::is_none")] - pub extra_key: Option, - - /// Whether to disallow logging for this request (e.g. due to ZDR) - #[serde(default)] - pub no_logs: bool, - - /// Custom metric labels - #[serde(skip_serializing_if = "Option::is_none")] - pub custom_labels: Option>, - - /// Whether to return bytes for image generation - #[serde(default)] - pub return_bytes: bool, - - /// Whether to return entropy - #[serde(default)] - pub return_entropy: bool, - - /// Request ID for tracking (inherited from BaseReq in Python) - #[serde(skip_serializing_if = "Option::is_none")] - pub rid: Option, -} - -impl Normalizable for GenerateRequest { - // Use default no-op implementation - no normalization needed for GenerateRequest -} - -/// Validation function for GenerateRequest - ensure exactly one input type is provided -fn validate_generate_request(req: &GenerateRequest) -> Result<(), validator::ValidationError> { - // Exactly one of text or input_ids must be provided - // Note: input_embeds not yet supported in Rust implementation - let has_text = req.text.is_some(); - let has_input_ids = req.input_ids.is_some(); - - let count = [has_text, has_input_ids].iter().filter(|&&x| x).count(); - - if count == 0 { - return Err(validator::ValidationError::new( - "Either text or input_ids should be provided.", - )); - } - - if count > 1 { - return Err(validator::ValidationError::new( - "Either text or input_ids should be provided.", - )); - } - - Ok(()) -} - -impl GenerationRequest for GenerateRequest { - fn is_stream(&self) -> bool { - self.stream - } - - fn get_model(&self) -> Option<&str> { - // Generate requests have an optional model field - if let Some(s) = &self.model { - Some(s.as_str()) - } else { - None - } - } - - fn extract_text_for_routing(&self) -> String { - // Check fields in priority order: text, input_ids - if let Some(ref text) = self.text { - return text.clone(); - } - - if let Some(ref input_ids) = self.input_ids { - return match input_ids { - InputIds::Single(ids) => ids - .iter() - .map(|&id| id.to_string()) - .collect::>() - .join(" "), - InputIds::Batch(batches) => batches - .iter() - .flat_map(|batch| batch.iter().map(|&id| id.to_string())) - .collect::>() - .join(" "), - }; - } - - // No text input found - String::new() - } -} - -// ============================================================================ -// SGLang Generate Response Types -// ============================================================================ - -/// SGLang generate response (single completion or array for n>1) -/// -/// Format for n=1: -/// ```json -/// { -/// "text": "...", -/// "output_ids": [...], -/// "meta_info": { ... } -/// } -/// ``` -/// -/// Format for n>1: -/// ```json -/// [ -/// {"text": "...", "output_ids": [...], "meta_info": {...}}, -/// {"text": "...", "output_ids": [...], "meta_info": {...}} -/// ] -/// ``` -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct GenerateResponse { - pub text: String, - pub output_ids: Vec, - pub meta_info: GenerateMetaInfo, -} - -/// Metadata for a single generate completion -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct GenerateMetaInfo { - pub id: String, - pub finish_reason: GenerateFinishReason, - pub prompt_tokens: u32, - pub weight_version: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub input_token_logprobs: Option>>>, - #[serde(skip_serializing_if = "Option::is_none")] - pub output_token_logprobs: Option>>>, - pub completion_tokens: u32, - pub cached_tokens: u32, - pub e2e_latency: f64, - #[serde(skip_serializing_if = "Option::is_none")] - pub matched_stop: Option, -} - -/// Finish reason for generate endpoint -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(tag = "type", rename_all = "lowercase")] -pub enum GenerateFinishReason { - Length { - length: u32, - }, - Stop, - #[serde(untagged)] - Other(Value), -} diff --git a/sgl-model-gateway/src/protocols/messages.rs b/sgl-model-gateway/src/protocols/messages.rs deleted file mode 100644 index b180e1224..000000000 --- a/sgl-model-gateway/src/protocols/messages.rs +++ /dev/null @@ -1,1726 +0,0 @@ -//! Anthropic Messages API protocol definitions -//! -//! This module provides Rust types for the Anthropic Messages API. -//! See: https://docs.anthropic.com/en/api/messages - -use std::collections::HashMap; - -use serde::{Deserialize, Serialize}; -use serde_json::Value; - -// ============================================================================ -// Request Types -// ============================================================================ - -/// Request to create a message using the Anthropic Messages API. -/// -/// This is the main request type for `/v1/messages` endpoint. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct CreateMessageRequest { - /// The model that will complete your prompt. - pub model: String, - - /// Input messages for the conversation. - pub messages: Vec, - - /// The maximum number of tokens to generate before stopping. - pub max_tokens: u32, - - /// An object describing metadata about the request. - #[serde(skip_serializing_if = "Option::is_none")] - pub metadata: Option, - - /// Service tier for the request (auto or standard_only). - #[serde(skip_serializing_if = "Option::is_none")] - pub service_tier: Option, - - /// Custom text sequences that will cause the model to stop generating. - #[serde(skip_serializing_if = "Option::is_none")] - pub stop_sequences: Option>, - - /// Whether to incrementally stream the response using server-sent events. - #[serde(skip_serializing_if = "Option::is_none")] - pub stream: Option, - - /// System prompt for providing context and instructions. - #[serde(skip_serializing_if = "Option::is_none")] - pub system: Option, - - /// Amount of randomness injected into the response (0.0 to 1.0). - #[serde(skip_serializing_if = "Option::is_none")] - pub temperature: Option, - - /// Configuration for extended thinking. - #[serde(skip_serializing_if = "Option::is_none")] - pub thinking: Option, - - /// How the model should use the provided tools. - #[serde(skip_serializing_if = "Option::is_none")] - pub tool_choice: Option, - - /// Definitions of tools that the model may use. - #[serde(skip_serializing_if = "Option::is_none")] - pub tools: Option>, - - /// Only sample from the top K options for each subsequent token. - #[serde(skip_serializing_if = "Option::is_none")] - pub top_k: Option, - - /// Use nucleus sampling. - #[serde(skip_serializing_if = "Option::is_none")] - pub top_p: Option, - - // Beta features - /// Container configuration for code execution (beta). - #[serde(skip_serializing_if = "Option::is_none")] - pub container: Option, - - /// MCP servers to be utilized in this request (beta). - #[serde(skip_serializing_if = "Option::is_none")] - pub mcp_servers: Option>, -} - -impl CreateMessageRequest { - /// Check if the request is for streaming - pub fn is_stream(&self) -> bool { - self.stream.unwrap_or(false) - } - - /// Get the model name - pub fn get_model(&self) -> &str { - &self.model - } -} - -/// Request metadata -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct Metadata { - /// An external identifier for the user who is associated with the request. - #[serde(skip_serializing_if = "Option::is_none")] - pub user_id: Option, -} - -/// Service tier options -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum ServiceTier { - Auto, - StandardOnly, -} - -/// System content can be a string or an array of text blocks -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(untagged)] -pub enum SystemContent { - String(String), - Blocks(Vec), -} - -/// A single input message in a conversation -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct InputMessage { - /// The role of the message sender (user or assistant) - pub role: Role, - - /// The content of the message - pub content: InputContent, -} - -/// Role of a message sender -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "lowercase")] -pub enum Role { - User, - Assistant, -} - -/// Input content can be a string or an array of content blocks -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(untagged)] -pub enum InputContent { - String(String), - Blocks(Vec), -} - -// ============================================================================ -// Input Content Blocks -// ============================================================================ - -/// Input content block types -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(tag = "type", rename_all = "snake_case")] -pub enum InputContentBlock { - /// Text content - Text(TextBlock), - /// Image content - Image(ImageBlock), - /// Document content - Document(DocumentBlock), - /// Tool use block (for assistant messages) - ToolUse(ToolUseBlock), - /// Tool result block (for user messages) - ToolResult(ToolResultBlock), - /// Thinking block - Thinking(ThinkingBlock), - /// Redacted thinking block - RedactedThinking(RedactedThinkingBlock), - /// Server tool use block - ServerToolUse(ServerToolUseBlock), - /// Search result block - SearchResult(SearchResultBlock), - /// Web search tool result block - WebSearchToolResult(WebSearchToolResultBlock), -} - -/// Text content block -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct TextBlock { - /// The text content - pub text: String, - - /// Cache control for this block - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_control: Option, - - /// Citations for this text block - #[serde(skip_serializing_if = "Option::is_none")] - pub citations: Option>, -} - -/// Image content block -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ImageBlock { - /// The image source - pub source: ImageSource, - - /// Cache control for this block - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_control: Option, -} - -/// Image source (base64 or URL) -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(tag = "type", rename_all = "snake_case")] -pub enum ImageSource { - Base64 { media_type: String, data: String }, - Url { url: String }, -} - -/// Document content block -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct DocumentBlock { - /// The document source - pub source: DocumentSource, - - /// Cache control for this block - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_control: Option, - - /// Optional title for the document - #[serde(skip_serializing_if = "Option::is_none")] - pub title: Option, - - /// Optional context for the document - #[serde(skip_serializing_if = "Option::is_none")] - pub context: Option, - - /// Citations configuration - #[serde(skip_serializing_if = "Option::is_none")] - pub citations: Option, -} - -/// Document source (base64, text, or URL) -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(tag = "type", rename_all = "snake_case")] -pub enum DocumentSource { - Base64 { media_type: String, data: String }, - Text { data: String }, - Url { url: String }, - Content { content: Vec }, -} - -/// Tool use block (in assistant messages) -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ToolUseBlock { - /// Unique identifier for this tool use - pub id: String, - - /// Name of the tool being used - pub name: String, - - /// Input arguments for the tool - pub input: Value, - - /// Cache control for this block - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_control: Option, -} - -/// Tool result block (in user messages) -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ToolResultBlock { - /// The ID of the tool use this is a result for - pub tool_use_id: String, - - /// The result content (string or blocks) - #[serde(skip_serializing_if = "Option::is_none")] - pub content: Option, - - /// Whether this result indicates an error - #[serde(skip_serializing_if = "Option::is_none")] - pub is_error: Option, - - /// Cache control for this block - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_control: Option, -} - -/// Tool result content (string or blocks) -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(untagged)] -pub enum ToolResultContent { - String(String), - Blocks(Vec), -} - -/// Content blocks allowed in tool results -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(tag = "type", rename_all = "snake_case")] -pub enum ToolResultContentBlock { - Text(TextBlock), - Image(ImageBlock), - Document(DocumentBlock), - SearchResult(SearchResultBlock), -} - -/// Thinking block -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ThinkingBlock { - /// The thinking content - pub thinking: String, - - /// Signature for the thinking block - pub signature: String, -} - -/// Redacted thinking block -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct RedactedThinkingBlock { - /// The encrypted/redacted data - pub data: String, -} - -/// Server tool use block -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ServerToolUseBlock { - /// Unique identifier for this tool use - pub id: String, - - /// Name of the server tool - pub name: String, - - /// Input arguments for the tool - pub input: Value, - - /// Cache control for this block - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_control: Option, -} - -/// Search result block -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct SearchResultBlock { - /// Source URL or identifier - pub source: String, - - /// Title of the search result - pub title: String, - - /// Content of the search result - pub content: Vec, - - /// Cache control for this block - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_control: Option, - - /// Citations configuration - #[serde(skip_serializing_if = "Option::is_none")] - pub citations: Option, -} - -/// Web search tool result block -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct WebSearchToolResultBlock { - /// The tool use ID this result is for - pub tool_use_id: String, - - /// The search results or error - pub content: WebSearchToolResultContent, - - /// Cache control for this block - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_control: Option, -} - -/// Web search tool result content -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(untagged)] -pub enum WebSearchToolResultContent { - Results(Vec), - Error(WebSearchToolResultError), -} - -/// Web search result block -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct WebSearchResultBlock { - /// Title of the search result - pub title: String, - - /// URL of the search result - pub url: String, - - /// Encrypted content - pub encrypted_content: String, - - /// Page age (if available) - #[serde(skip_serializing_if = "Option::is_none")] - pub page_age: Option, -} - -/// Web search tool result error -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct WebSearchToolResultError { - #[serde(rename = "type")] - pub error_type: String, - pub error_code: WebSearchToolResultErrorCode, -} - -/// Web search tool result error codes -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum WebSearchToolResultErrorCode { - InvalidToolInput, - Unavailable, - MaxUsesExceeded, - TooManyRequests, - QueryTooLong, -} - -/// Cache control configuration -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(tag = "type", rename_all = "snake_case")] -pub enum CacheControl { - Ephemeral, -} - -/// Citations configuration -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct CitationsConfig { - #[serde(skip_serializing_if = "Option::is_none")] - pub enabled: Option, -} - -/// Citation types -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(tag = "type", rename_all = "snake_case")] -pub enum Citation { - CharLocation(CharLocationCitation), - PageLocation(PageLocationCitation), - ContentBlockLocation(ContentBlockLocationCitation), - WebSearchResultLocation(WebSearchResultLocationCitation), - SearchResultLocation(SearchResultLocationCitation), -} - -/// Character location citation -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct CharLocationCitation { - pub cited_text: String, - pub document_index: u32, - pub document_title: Option, - pub start_char_index: u32, - pub end_char_index: u32, - #[serde(skip_serializing_if = "Option::is_none")] - pub file_id: Option, -} - -/// Page location citation -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct PageLocationCitation { - pub cited_text: String, - pub document_index: u32, - pub document_title: Option, - pub start_page_number: u32, - pub end_page_number: u32, -} - -/// Content block location citation -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ContentBlockLocationCitation { - pub cited_text: String, - pub document_index: u32, - pub document_title: Option, - pub start_block_index: u32, - pub end_block_index: u32, -} - -/// Web search result location citation -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct WebSearchResultLocationCitation { - pub cited_text: String, - pub url: String, - pub title: Option, - pub encrypted_index: String, -} - -/// Search result location citation -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct SearchResultLocationCitation { - pub cited_text: String, - pub search_result_index: u32, - pub source: String, - pub title: Option, - pub start_block_index: u32, - pub end_block_index: u32, -} - -// ============================================================================ -// Tool Definitions -// ============================================================================ - -/// Tool definition -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(untagged)] -pub enum Tool { - /// Custom tool definition - Custom(CustomTool), - /// Bash tool (computer use) - Bash(BashTool), - /// Text editor tool (computer use) - TextEditor(TextEditorTool), - /// Web search tool - WebSearch(WebSearchTool), -} - -/// Custom tool definition -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct CustomTool { - /// Name of the tool - pub name: String, - - /// Optional type (defaults to "custom") - #[serde(rename = "type", skip_serializing_if = "Option::is_none")] - pub tool_type: Option, - - /// Description of what this tool does - #[serde(skip_serializing_if = "Option::is_none")] - pub description: Option, - - /// JSON schema for the tool's input - pub input_schema: InputSchema, - - /// Cache control for this tool - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_control: Option, -} - -/// JSON Schema for tool input -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct InputSchema { - #[serde(rename = "type")] - pub schema_type: String, - - #[serde(skip_serializing_if = "Option::is_none")] - pub properties: Option>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub required: Option>, - - /// Additional properties can be stored here - #[serde(flatten)] - pub additional: HashMap, -} - -/// Bash tool for computer use -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct BashTool { - #[serde(rename = "type")] - pub tool_type: String, // "bash_20250124" - - pub name: String, // "bash" - - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_control: Option, -} - -/// Text editor tool for computer use -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct TextEditorTool { - #[serde(rename = "type")] - pub tool_type: String, // "text_editor_20250124", etc. - - pub name: String, // "str_replace_editor" - - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_control: Option, -} - -/// Web search tool -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct WebSearchTool { - #[serde(rename = "type")] - pub tool_type: String, // "web_search_20250305" - - pub name: String, // "web_search" - - #[serde(skip_serializing_if = "Option::is_none")] - pub allowed_domains: Option>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub blocked_domains: Option>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub max_uses: Option, - - #[serde(skip_serializing_if = "Option::is_none")] - pub user_location: Option, - - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_control: Option, -} - -/// User location for web search -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct UserLocation { - #[serde(rename = "type")] - pub location_type: String, // "approximate" - - #[serde(skip_serializing_if = "Option::is_none")] - pub city: Option, - - #[serde(skip_serializing_if = "Option::is_none")] - pub region: Option, - - #[serde(skip_serializing_if = "Option::is_none")] - pub country: Option, - - #[serde(skip_serializing_if = "Option::is_none")] - pub timezone: Option, -} - -// ============================================================================ -// Tool Choice -// ============================================================================ - -/// How the model should use the provided tools -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(tag = "type", rename_all = "snake_case")] -pub enum ToolChoice { - /// The model will automatically decide whether to use tools - Auto { - #[serde(skip_serializing_if = "Option::is_none")] - disable_parallel_tool_use: Option, - }, - /// The model will use any available tools - Any { - #[serde(skip_serializing_if = "Option::is_none")] - disable_parallel_tool_use: Option, - }, - /// The model will use the specified tool - Tool { - name: String, - #[serde(skip_serializing_if = "Option::is_none")] - disable_parallel_tool_use: Option, - }, - /// The model will not use tools - None, -} - -// ============================================================================ -// Thinking Configuration -// ============================================================================ - -/// Configuration for extended thinking -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(tag = "type", rename_all = "snake_case")] -pub enum ThinkingConfig { - /// Enable extended thinking - Enabled { - /// Budget in tokens for thinking (minimum 1024) - budget_tokens: u32, - }, - /// Disable extended thinking - Disabled, -} - -// ============================================================================ -// Response Types -// ============================================================================ - -/// Response message from the API -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct Message { - /// Unique object identifier - pub id: String, - - /// Object type (always "message") - #[serde(rename = "type")] - pub message_type: String, - - /// Conversational role (always "assistant") - pub role: String, - - /// Content generated by the model - pub content: Vec, - - /// The model that generated the message - pub model: String, - - /// The reason the model stopped generating - pub stop_reason: Option, - - /// Which custom stop sequence was generated (if any) - pub stop_sequence: Option, - - /// Billing and rate-limit usage - pub usage: Usage, -} - -/// Output content block types -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(tag = "type", rename_all = "snake_case")] -pub enum ContentBlock { - /// Text content - Text { - text: String, - #[serde(skip_serializing_if = "Option::is_none")] - citations: Option>, - }, - /// Tool use by the model - ToolUse { - id: String, - name: String, - input: Value, - }, - /// Thinking content - Thinking { thinking: String, signature: String }, - /// Redacted thinking content - RedactedThinking { data: String }, - /// Server tool use - ServerToolUse { - id: String, - name: String, - input: Value, - }, - /// Web search tool result - WebSearchToolResult { - tool_use_id: String, - content: WebSearchToolResultContent, - }, -} - -/// Stop reasons -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum StopReason { - /// The model reached a natural stopping point - EndTurn, - /// We exceeded the requested max_tokens - MaxTokens, - /// One of the custom stop_sequences was generated - StopSequence, - /// The model invoked one or more tools - ToolUse, - /// We paused a long-running turn - PauseTurn, - /// Streaming classifiers intervened - Refusal, -} - -/// Billing and rate-limit usage -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct Usage { - /// The number of input tokens used - pub input_tokens: u32, - - /// The number of output tokens used - pub output_tokens: u32, - - /// The number of input tokens used to create the cache entry - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_creation_input_tokens: Option, - - /// The number of input tokens read from the cache - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_read_input_tokens: Option, - - /// Breakdown of cached tokens by TTL - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_creation: Option, - - /// Server tool usage information - #[serde(skip_serializing_if = "Option::is_none")] - pub server_tool_use: Option, - - /// Service tier used for the request - #[serde(skip_serializing_if = "Option::is_none")] - pub service_tier: Option, -} - -/// Cache creation breakdown -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct CacheCreation { - #[serde(flatten)] - pub tokens_by_ttl: HashMap, -} - -/// Server tool usage information -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ServerToolUsage { - pub web_search_requests: u32, -} - -// ============================================================================ -// Streaming Event Types -// ============================================================================ - -/// Server-sent event wrapper -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(tag = "type", rename_all = "snake_case")] -pub enum MessageStreamEvent { - /// Start of a new message - MessageStart { message: Message }, - /// Update to a message - MessageDelta { - delta: MessageDelta, - usage: MessageDeltaUsage, - }, - /// End of a message - MessageStop, - /// Start of a content block - ContentBlockStart { - index: u32, - content_block: ContentBlock, - }, - /// Update to a content block - ContentBlockDelta { - index: u32, - delta: ContentBlockDelta, - }, - /// End of a content block - ContentBlockStop { index: u32 }, - /// Ping event (for keep-alive) - Ping, - /// Error event - Error { error: ErrorResponse }, -} - -/// Message delta for streaming updates -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct MessageDelta { - #[serde(skip_serializing_if = "Option::is_none")] - pub stop_reason: Option, - - #[serde(skip_serializing_if = "Option::is_none")] - pub stop_sequence: Option, -} - -/// Usage delta for streaming updates -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct MessageDeltaUsage { - pub output_tokens: u32, - - #[serde(skip_serializing_if = "Option::is_none")] - pub input_tokens: Option, - - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_creation_input_tokens: Option, - - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_read_input_tokens: Option, - - #[serde(skip_serializing_if = "Option::is_none")] - pub server_tool_use: Option, -} - -/// Content block delta for streaming updates -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(tag = "type", rename_all = "snake_case")] -pub enum ContentBlockDelta { - /// Text delta - TextDelta { text: String }, - /// JSON input delta (for tool use) - InputJsonDelta { partial_json: String }, - /// Thinking delta - ThinkingDelta { thinking: String }, - /// Signature delta - SignatureDelta { signature: String }, - /// Citations delta - CitationsDelta { citation: Citation }, -} - -// ============================================================================ -// Error Types -// ============================================================================ - -/// Error response -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ErrorResponse { - #[serde(rename = "type")] - pub error_type: String, - - pub message: String, -} - -/// API error types -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(tag = "type", rename_all = "snake_case")] -pub enum ApiError { - InvalidRequestError { message: String }, - AuthenticationError { message: String }, - BillingError { message: String }, - PermissionError { message: String }, - NotFoundError { message: String }, - RateLimitError { message: String }, - TimeoutError { message: String }, - ApiError { message: String }, - OverloadedError { message: String }, -} - -// ============================================================================ -// Count Tokens Types -// ============================================================================ - -/// Request to count tokens in a message -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct CountMessageTokensRequest { - /// The model to use for token counting - pub model: String, - - /// Input messages - pub messages: Vec, - - /// System prompt - #[serde(skip_serializing_if = "Option::is_none")] - pub system: Option, - - /// Thinking configuration - #[serde(skip_serializing_if = "Option::is_none")] - pub thinking: Option, - - /// Tool choice - #[serde(skip_serializing_if = "Option::is_none")] - pub tool_choice: Option, - - /// Tool definitions - #[serde(skip_serializing_if = "Option::is_none")] - pub tools: Option>, -} - -/// Response from token counting -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct CountMessageTokensResponse { - pub input_tokens: u32, -} - -// ============================================================================ -// Model Info Types -// ============================================================================ - -/// Model information -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ModelInfo { - /// Object type (always "model") - #[serde(rename = "type")] - pub model_type: String, - - /// Model ID - pub id: String, - - /// Display name - pub display_name: String, - - /// When the model was created - pub created_at: String, -} - -/// List of models response -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ListModelsResponse { - pub data: Vec, - pub has_more: bool, - pub first_id: Option, - pub last_id: Option, -} - -// ============================================================================ -// Beta Features - Container & MCP Configuration -// ============================================================================ - -/// Container configuration for code execution (beta) -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ContainerConfig { - /// Container ID for reuse across requests - #[serde(skip_serializing_if = "Option::is_none")] - pub id: Option, - - /// Skills to be loaded in the container - #[serde(skip_serializing_if = "Option::is_none")] - pub skills: Option>, -} - -/// MCP server configuration (beta) -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct McpServerConfig { - /// Name of the MCP server - pub name: String, - - /// MCP server URL - pub url: String, - - /// Authorization token (if required) - #[serde(skip_serializing_if = "Option::is_none")] - pub authorization_token: Option, - - /// Tool configuration for this server - #[serde(skip_serializing_if = "Option::is_none")] - pub tool_configuration: Option, -} - -/// MCP tool configuration -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct McpToolConfiguration { - /// Whether to allow all tools - #[serde(skip_serializing_if = "Option::is_none")] - pub enabled: Option, - - /// Allowed tool names - #[serde(skip_serializing_if = "Option::is_none")] - pub allowed_tools: Option>, -} - -// ============================================================================ -// Beta Features - MCP Tool Types -// ============================================================================ - -/// MCP tool use block (beta) - for assistant messages -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct McpToolUseBlock { - /// Unique identifier for this tool use - pub id: String, - - /// Name of the tool being used - pub name: String, - - /// Name of the MCP server - pub server_name: String, - - /// Input arguments for the tool - pub input: Value, - - /// Cache control for this block - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_control: Option, -} - -/// MCP tool result block (beta) - for user messages -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct McpToolResultBlock { - /// The ID of the tool use this is a result for - pub tool_use_id: String, - - /// The result content - #[serde(skip_serializing_if = "Option::is_none")] - pub content: Option, - - /// Whether this result indicates an error - #[serde(skip_serializing_if = "Option::is_none")] - pub is_error: Option, - - /// Cache control for this block - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_control: Option, -} - -/// MCP toolset definition (beta) -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct McpToolset { - #[serde(rename = "type")] - pub toolset_type: String, // "mcp_toolset" - - /// Name of the MCP server to configure tools for - pub mcp_server_name: String, - - /// Default configuration applied to all tools from this server - #[serde(skip_serializing_if = "Option::is_none")] - pub default_config: Option, - - /// Configuration overrides for specific tools - #[serde(skip_serializing_if = "Option::is_none")] - pub configs: Option>, - - /// Cache control for this toolset - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_control: Option, -} - -/// Default configuration for MCP tools -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct McpToolDefaultConfig { - /// Whether tools are enabled - #[serde(skip_serializing_if = "Option::is_none")] - pub enabled: Option, - - /// Whether to defer loading - #[serde(skip_serializing_if = "Option::is_none")] - pub defer_loading: Option, -} - -/// Per-tool MCP configuration -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct McpToolConfig { - /// Whether this tool is enabled - #[serde(skip_serializing_if = "Option::is_none")] - pub enabled: Option, - - /// Whether to defer loading - #[serde(skip_serializing_if = "Option::is_none")] - pub defer_loading: Option, -} - -// ============================================================================ -// Beta Features - Code Execution Types -// ============================================================================ - -/// Code execution tool (beta) -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct CodeExecutionTool { - #[serde(rename = "type")] - pub tool_type: String, // "code_execution_20250522" or "code_execution_20250825" - - pub name: String, // "code_execution" - - /// Allowed callers for this tool - #[serde(skip_serializing_if = "Option::is_none")] - pub allowed_callers: Option>, - - /// Whether to defer loading - #[serde(skip_serializing_if = "Option::is_none")] - pub defer_loading: Option, - - /// Whether to use strict mode - #[serde(skip_serializing_if = "Option::is_none")] - pub strict: Option, - - /// Cache control for this tool - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_control: Option, -} - -/// Code execution result block (beta) -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct CodeExecutionResultBlock { - /// Stdout output - pub stdout: String, - - /// Stderr output - pub stderr: String, - - /// Return code - pub return_code: i32, - - /// Output files - pub content: Vec, -} - -/// Code execution output file reference -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct CodeExecutionOutputBlock { - #[serde(rename = "type")] - pub block_type: String, // "code_execution_output" - - /// File ID - pub file_id: String, -} - -/// Code execution tool result block (beta) -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct CodeExecutionToolResultBlock { - /// The ID of the tool use this is a result for - pub tool_use_id: String, - - /// The result content (success or error) - pub content: CodeExecutionToolResultContent, - - /// Cache control for this block - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_control: Option, -} - -/// Code execution tool result content -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(untagged)] -pub enum CodeExecutionToolResultContent { - Success(CodeExecutionResultBlock), - Error(CodeExecutionToolResultError), -} - -/// Code execution tool result error -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct CodeExecutionToolResultError { - #[serde(rename = "type")] - pub error_type: String, // "code_execution_tool_result_error" - - pub error_code: CodeExecutionToolResultErrorCode, -} - -/// Code execution error codes -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum CodeExecutionToolResultErrorCode { - Unavailable, - CodeExecutionExceededTimeout, - ContainerExpired, - InvalidToolInput, -} - -/// Bash code execution result block (beta) -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct BashCodeExecutionResultBlock { - /// Stdout output - pub stdout: String, - - /// Stderr output - pub stderr: String, - - /// Return code - pub return_code: i32, - - /// Output files - pub content: Vec, -} - -/// Bash code execution output file reference -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct BashCodeExecutionOutputBlock { - #[serde(rename = "type")] - pub block_type: String, // "bash_code_execution_output" - - /// File ID - pub file_id: String, -} - -/// Bash code execution tool result block (beta) -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct BashCodeExecutionToolResultBlock { - /// The ID of the tool use this is a result for - pub tool_use_id: String, - - /// The result content (success or error) - pub content: BashCodeExecutionToolResultContent, - - /// Cache control for this block - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_control: Option, -} - -/// Bash code execution tool result content -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(untagged)] -pub enum BashCodeExecutionToolResultContent { - Success(BashCodeExecutionResultBlock), - Error(BashCodeExecutionToolResultError), -} - -/// Bash code execution tool result error -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct BashCodeExecutionToolResultError { - #[serde(rename = "type")] - pub error_type: String, // "bash_code_execution_tool_result_error" - - pub error_code: BashCodeExecutionToolResultErrorCode, -} - -/// Bash code execution error codes -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum BashCodeExecutionToolResultErrorCode { - Unavailable, - CodeExecutionExceededTimeout, - ContainerExpired, - InvalidToolInput, -} - -/// Text editor code execution tool result block (beta) -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct TextEditorCodeExecutionToolResultBlock { - /// The ID of the tool use this is a result for - pub tool_use_id: String, - - /// The result content - pub content: TextEditorCodeExecutionToolResultContent, - - /// Cache control for this block - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_control: Option, -} - -/// Text editor code execution result content -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(untagged)] -pub enum TextEditorCodeExecutionToolResultContent { - CreateResult(TextEditorCodeExecutionCreateResultBlock), - StrReplaceResult(TextEditorCodeExecutionStrReplaceResultBlock), - ViewResult(TextEditorCodeExecutionViewResultBlock), - Error(TextEditorCodeExecutionToolResultError), -} - -/// Text editor create result block -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct TextEditorCodeExecutionCreateResultBlock { - #[serde(rename = "type")] - pub block_type: String, // "text_editor_code_execution_create_result" -} - -/// Text editor str_replace result block -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct TextEditorCodeExecutionStrReplaceResultBlock { - #[serde(rename = "type")] - pub block_type: String, // "text_editor_code_execution_str_replace_result" - - /// Snippet of content around the replacement - #[serde(skip_serializing_if = "Option::is_none")] - pub snippet: Option, -} - -/// Text editor view result block -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct TextEditorCodeExecutionViewResultBlock { - #[serde(rename = "type")] - pub block_type: String, // "text_editor_code_execution_view_result" - - /// Content of the viewed file - pub content: String, -} - -/// Text editor code execution tool result error -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct TextEditorCodeExecutionToolResultError { - #[serde(rename = "type")] - pub error_type: String, - - pub error_code: TextEditorCodeExecutionToolResultErrorCode, -} - -/// Text editor code execution error codes -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum TextEditorCodeExecutionToolResultErrorCode { - Unavailable, - InvalidToolInput, - FileNotFound, - ContainerExpired, -} - -// ============================================================================ -// Beta Features - Web Fetch Types -// ============================================================================ - -/// Web fetch tool (beta) -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct WebFetchTool { - #[serde(rename = "type")] - pub tool_type: String, // "web_fetch_20250305" or similar - - pub name: String, // "web_fetch" - - /// Allowed callers for this tool - #[serde(skip_serializing_if = "Option::is_none")] - pub allowed_callers: Option>, - - /// Maximum number of uses - #[serde(skip_serializing_if = "Option::is_none")] - pub max_uses: Option, - - /// Cache control for this tool - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_control: Option, -} - -/// Web fetch result block (beta) -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct WebFetchResultBlock { - #[serde(rename = "type")] - pub block_type: String, // "web_fetch_result" - - /// The URL that was fetched - pub url: String, - - /// The document content - pub content: DocumentBlock, - - /// When the content was retrieved - #[serde(skip_serializing_if = "Option::is_none")] - pub retrieved_at: Option, -} - -/// Web fetch tool result block (beta) -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct WebFetchToolResultBlock { - /// The ID of the tool use this is a result for - pub tool_use_id: String, - - /// The result content (success or error) - pub content: WebFetchToolResultContent, - - /// Cache control for this block - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_control: Option, -} - -/// Web fetch tool result content -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(untagged)] -pub enum WebFetchToolResultContent { - Success(WebFetchResultBlock), - Error(WebFetchToolResultError), -} - -/// Web fetch tool result error -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct WebFetchToolResultError { - #[serde(rename = "type")] - pub error_type: String, // "web_fetch_tool_result_error" - - pub error_code: WebFetchToolResultErrorCode, -} - -/// Web fetch error codes -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum WebFetchToolResultErrorCode { - InvalidToolInput, - Unavailable, - MaxUsesExceeded, - TooManyRequests, - UrlNotAllowed, - FetchFailed, - ContentTooLarge, -} - -// ============================================================================ -// Beta Features - Tool Search Types -// ============================================================================ - -/// Tool search tool (beta) -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ToolSearchTool { - #[serde(rename = "type")] - pub tool_type: String, // "tool_search_tool_regex" or "tool_search_tool_bm25" - - pub name: String, - - /// Allowed callers for this tool - #[serde(skip_serializing_if = "Option::is_none")] - pub allowed_callers: Option>, - - /// Cache control for this tool - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_control: Option, -} - -/// Tool reference block (beta) - returned by tool search -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ToolReferenceBlock { - #[serde(rename = "type")] - pub block_type: String, // "tool_reference" - - /// Tool name - pub tool_name: String, - - /// Tool description - #[serde(skip_serializing_if = "Option::is_none")] - pub description: Option, -} - -/// Tool search result block (beta) -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ToolSearchResultBlock { - #[serde(rename = "type")] - pub block_type: String, // "tool_search_tool_search_result" - - /// Tool name - pub tool_name: String, - - /// Relevance score - #[serde(skip_serializing_if = "Option::is_none")] - pub score: Option, -} - -/// Tool search tool result block (beta) -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ToolSearchToolResultBlock { - /// The ID of the tool use this is a result for - pub tool_use_id: String, - - /// The search results - pub content: Vec, - - /// Cache control for this block - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_control: Option, -} - -// ============================================================================ -// Beta Features - Container Upload Types -// ============================================================================ - -/// Container upload block (beta) -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ContainerUploadBlock { - #[serde(rename = "type")] - pub block_type: String, // "container_upload" - - /// File ID - pub file_id: String, - - /// File name - pub file_name: String, - - /// File path in container - #[serde(skip_serializing_if = "Option::is_none")] - pub file_path: Option, -} - -// ============================================================================ -// Beta Features - Memory Tool Types -// ============================================================================ - -/// Memory tool (beta) -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct MemoryTool { - #[serde(rename = "type")] - pub tool_type: String, // "memory_20250818" - - pub name: String, // "memory" - - /// Allowed callers for this tool - #[serde(skip_serializing_if = "Option::is_none")] - pub allowed_callers: Option>, - - /// Whether to defer loading - #[serde(skip_serializing_if = "Option::is_none")] - pub defer_loading: Option, - - /// Whether to use strict mode - #[serde(skip_serializing_if = "Option::is_none")] - pub strict: Option, - - /// Input examples - #[serde(skip_serializing_if = "Option::is_none")] - pub input_examples: Option>, - - /// Cache control for this tool - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_control: Option, -} - -// ============================================================================ -// Beta Features - Computer Use Tool Types -// ============================================================================ - -/// Computer use tool (beta) -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ComputerUseTool { - #[serde(rename = "type")] - pub tool_type: String, // "computer_20241022" or "computer_20250124" - - pub name: String, // "computer" - - /// Display width - pub display_width_px: u32, - - /// Display height - pub display_height_px: u32, - - /// Display number (optional) - #[serde(skip_serializing_if = "Option::is_none")] - pub display_number: Option, - - /// Allowed callers for this tool - #[serde(skip_serializing_if = "Option::is_none")] - pub allowed_callers: Option>, - - /// Cache control for this tool - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_control: Option, -} - -// ============================================================================ -// Beta Features - Extended Input Content Block Enum -// ============================================================================ - -/// Beta input content block types (extends InputContentBlock) -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(tag = "type", rename_all = "snake_case")] -pub enum BetaInputContentBlock { - // Standard types - Text(TextBlock), - Image(ImageBlock), - Document(DocumentBlock), - ToolUse(ToolUseBlock), - ToolResult(ToolResultBlock), - Thinking(ThinkingBlock), - RedactedThinking(RedactedThinkingBlock), - ServerToolUse(ServerToolUseBlock), - SearchResult(SearchResultBlock), - WebSearchToolResult(WebSearchToolResultBlock), - - // Beta MCP types - McpToolUse(McpToolUseBlock), - McpToolResult(McpToolResultBlock), - - // Beta code execution types - CodeExecutionToolResult(CodeExecutionToolResultBlock), - BashCodeExecutionToolResult(BashCodeExecutionToolResultBlock), - TextEditorCodeExecutionToolResult(TextEditorCodeExecutionToolResultBlock), - - // Beta web fetch types - WebFetchToolResult(WebFetchToolResultBlock), - - // Beta tool search types - ToolSearchToolResult(ToolSearchToolResultBlock), - ToolReference(ToolReferenceBlock), - - // Beta container types - ContainerUpload(ContainerUploadBlock), -} - -/// Beta output content block types (extends ContentBlock) -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(tag = "type", rename_all = "snake_case")] -pub enum BetaContentBlock { - // Standard types - Text { - text: String, - #[serde(skip_serializing_if = "Option::is_none")] - citations: Option>, - }, - ToolUse { - id: String, - name: String, - input: Value, - }, - Thinking { - thinking: String, - signature: String, - }, - RedactedThinking { - data: String, - }, - ServerToolUse { - id: String, - name: String, - input: Value, - }, - WebSearchToolResult { - tool_use_id: String, - content: WebSearchToolResultContent, - }, - - // Beta MCP types - McpToolUse { - id: String, - name: String, - server_name: String, - input: Value, - }, - McpToolResult { - tool_use_id: String, - #[serde(skip_serializing_if = "Option::is_none")] - content: Option, - #[serde(skip_serializing_if = "Option::is_none")] - is_error: Option, - }, - - // Beta code execution types - CodeExecutionToolResult { - tool_use_id: String, - content: CodeExecutionToolResultContent, - }, - BashCodeExecutionToolResult { - tool_use_id: String, - content: BashCodeExecutionToolResultContent, - }, - TextEditorCodeExecutionToolResult { - tool_use_id: String, - content: TextEditorCodeExecutionToolResultContent, - }, - - // Beta web fetch types - WebFetchToolResult { - tool_use_id: String, - content: WebFetchToolResultContent, - }, - - // Beta tool search types - ToolSearchToolResult { - tool_use_id: String, - content: Vec, - }, - ToolReference { - tool_name: String, - #[serde(skip_serializing_if = "Option::is_none")] - description: Option, - }, - - // Beta container types - ContainerUpload { - file_id: String, - file_name: String, - #[serde(skip_serializing_if = "Option::is_none")] - file_path: Option, - }, -} - -/// Beta tool definition (extends Tool) -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(untagged)] -pub enum BetaTool { - // Standard tools - Custom(CustomTool), - Bash(BashTool), - TextEditor(TextEditorTool), - WebSearch(WebSearchTool), - - // Beta tools - CodeExecution(CodeExecutionTool), - McpToolset(McpToolset), - WebFetch(WebFetchTool), - ToolSearch(ToolSearchTool), - Memory(MemoryTool), - ComputerUse(ComputerUseTool), -} - -/// Server tool names for beta features -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum BetaServerToolName { - WebSearch, - WebFetch, - CodeExecution, - BashCodeExecution, - TextEditorCodeExecution, - ToolSearchToolRegex, - ToolSearchToolBm25, -} - -/// Server tool caller types (beta) -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(tag = "type", rename_all = "snake_case")] -pub enum ServerToolCaller { - /// Direct caller (the model itself) - Direct, - /// Code execution caller - #[serde(rename = "code_execution_20250825")] - CodeExecution20250825, -} diff --git a/sgl-model-gateway/src/protocols/mod.rs b/sgl-model-gateway/src/protocols/mod.rs deleted file mode 100644 index 619661147..000000000 --- a/sgl-model-gateway/src/protocols/mod.rs +++ /dev/null @@ -1,25 +0,0 @@ -// Protocol definitions and validation for various LLM APIs -// This module provides a structured approach to handling different API protocols - -/// Default model identifier used when no model is specified. -/// -/// This constant should be used instead of hardcoded "unknown" strings -/// throughout the codebase for consistency. -pub const UNKNOWN_MODEL_ID: &str = "unknown"; - -pub mod builders; -pub mod chat; -pub mod classify; -pub mod common; -pub mod completion; -pub mod embedding; -pub mod event_types; -pub mod generate; -pub mod messages; -pub mod parser; -pub mod rerank; -pub mod responses; -pub mod sampling_params; -pub mod tokenize; -pub mod validated; -pub mod worker_spec; diff --git a/sgl-model-gateway/src/protocols/parser.rs b/sgl-model-gateway/src/protocols/parser.rs deleted file mode 100644 index 47a53a0d5..000000000 --- a/sgl-model-gateway/src/protocols/parser.rs +++ /dev/null @@ -1,23 +0,0 @@ -use serde::Deserialize; - -use crate::protocols::common::Tool; - -/// Request to parse function calls from model output text -#[derive(Deserialize)] -pub struct ParseFunctionCallRequest { - /// The text to parse for function calls - pub text: String, - /// The parser type/name to use for parsing (e.g., "json", "pythonic") - pub tool_call_parser: String, - /// The list of available tools that the model can call - pub tools: Vec, -} - -/// Request to separate reasoning from normal text in model output -#[derive(Deserialize)] -pub struct SeparateReasoningRequest { - /// The text to parse for reasoning content - pub text: String, - /// The parser type/name to use for reasoning detection (e.g., "step3", "deepseek_r1") - pub reasoning_parser: String, -} diff --git a/sgl-model-gateway/src/protocols/rerank.rs b/sgl-model-gateway/src/protocols/rerank.rs deleted file mode 100644 index 6775f5d8a..000000000 --- a/sgl-model-gateway/src/protocols/rerank.rs +++ /dev/null @@ -1,212 +0,0 @@ -use std::collections::HashMap; - -use serde::{Deserialize, Serialize}; -use serde_json::Value; -use validator::Validate; - -use super::common::{default_model, default_true, GenerationRequest, StringOrArray, UsageInfo}; - -fn default_rerank_object() -> String { - "rerank".to_string() -} - -/// TODO: Create timestamp should not be in protocol layer -fn current_timestamp() -> i64 { - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap_or_else(|_| std::time::Duration::from_secs(0)) - .as_secs() as i64 -} - -// ============================================================================ -// Rerank API -// ============================================================================ - -#[derive(Debug, Clone, Deserialize, Serialize, Validate)] -#[validate(schema(function = "validate_rerank_request"))] -pub struct RerankRequest { - /// The query text to rank documents against - #[validate(custom(function = "validate_query"))] - pub query: String, - - /// List of documents to be ranked - #[validate(custom(function = "validate_documents"))] - pub documents: Vec, - - /// Model to use for reranking - #[serde(default = "default_model")] - pub model: String, - - /// Maximum number of documents to return (optional) - #[serde(skip_serializing_if = "Option::is_none")] - #[validate(range(min = 1))] - pub top_k: Option, - - /// Whether to return documents in addition to scores - #[serde(default = "default_true")] - pub return_documents: bool, - - // SGLang specific extensions - /// Request ID for tracking - pub rid: Option, - - /// User identifier - pub user: Option, -} - -impl GenerationRequest for RerankRequest { - fn get_model(&self) -> Option<&str> { - Some(&self.model) - } - - fn is_stream(&self) -> bool { - false // Reranking doesn't support streaming - } - - fn extract_text_for_routing(&self) -> String { - self.query.clone() - } -} - -impl super::validated::Normalizable for RerankRequest { - // Use default no-op normalization -} - -// ============================================================================ -// Validation Functions -// ============================================================================ - -/// Validates that the query is not empty -fn validate_query(query: &str) -> Result<(), validator::ValidationError> { - if query.trim().is_empty() { - return Err(validator::ValidationError::new("query cannot be empty")); - } - Ok(()) -} - -/// Validates that the documents list is not empty -fn validate_documents(documents: &[String]) -> Result<(), validator::ValidationError> { - if documents.is_empty() { - return Err(validator::ValidationError::new( - "documents list cannot be empty", - )); - } - Ok(()) -} - -/// Schema-level validation for cross-field dependencies -fn validate_rerank_request(req: &RerankRequest) -> Result<(), validator::ValidationError> { - // Validate top_k if specified - if let Some(k) = req.top_k { - if k > req.documents.len() { - // This is allowed but we log a warning - tracing::warn!( - "top_k ({}) is greater than number of documents ({})", - k, - req.documents.len() - ); - } - } - Ok(()) -} - -impl RerankRequest { - /// Get the effective top_k value - pub fn effective_top_k(&self) -> usize { - self.top_k.unwrap_or(self.documents.len()) - } -} - -/// Individual rerank result -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct RerankResult { - /// Relevance score for the document - pub score: f32, - - /// The document text (if return_documents was true) - #[serde(skip_serializing_if = "Option::is_none")] - pub document: Option, - - /// Original index of the document in the request - pub index: usize, - - /// Additional metadata about the ranking - #[serde(skip_serializing_if = "Option::is_none")] - pub meta_info: Option>, -} - -/// Rerank response containing sorted results -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct RerankResponse { - /// Ranked results sorted by score (highest first) - pub results: Vec, - - /// Model used for reranking - pub model: String, - - /// Usage information - pub usage: Option, - - /// Response object type - #[serde(default = "default_rerank_object")] - pub object: String, - - /// Response ID - pub id: Option, - - /// Creation timestamp - pub created: i64, -} - -impl RerankResponse { - /// Create a new RerankResponse with the given results and model - pub fn new( - results: Vec, - model: String, - request_id: Option, - ) -> Self { - RerankResponse { - results, - model, - usage: None, - object: default_rerank_object(), - id: request_id, - created: current_timestamp(), - } - } - - /// Apply top_k limit to results - pub fn apply_top_k(&mut self, k: usize) { - self.results.truncate(k); - } - - /// Drop documents from results (when return_documents is false) - pub fn drop_documents(&mut self) { - for result in &mut self.results { - result.document = None; - } - } -} - -/// V1 API compatibility format for rerank requests -/// Matches Python's V1RerankReqInput -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct V1RerankReqInput { - pub query: String, - pub documents: Vec, -} - -/// Convert V1RerankReqInput to RerankRequest -impl From for RerankRequest { - fn from(v1: V1RerankReqInput) -> Self { - RerankRequest { - query: v1.query, - documents: v1.documents, - model: default_model(), - top_k: None, - return_documents: true, - rid: None, - user: None, - } - } -} diff --git a/sgl-model-gateway/src/protocols/responses.rs b/sgl-model-gateway/src/protocols/responses.rs deleted file mode 100644 index f2f41d348..000000000 --- a/sgl-model-gateway/src/protocols/responses.rs +++ /dev/null @@ -1,1298 +0,0 @@ -// OpenAI Responses API types -// https://platform.openai.com/docs/api-reference/responses - -use std::collections::HashMap; - -use serde::{Deserialize, Serialize}; -use serde_json::Value; -use validator::Validate; - -use super::{ - common::{ - default_model, default_true, validate_stop, ChatLogProbs, Function, GenerationRequest, - PromptTokenUsageInfo, StringOrArray, ToolChoice, ToolChoiceValue, ToolReference, UsageInfo, - }, - sampling_params::{validate_top_k_value, validate_top_p_value}, -}; -use crate::protocols::{builders::ResponsesResponseBuilder, validated::Normalizable}; - -// ============================================================================ -// Response Tools (MCP and others) -// ============================================================================ - -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct ResponseTool { - #[serde(rename = "type")] - pub r#type: ResponseToolType, - // Function tool fields (used when type == "function") - // In Responses API, function fields are flattened at the top level - #[serde(flatten)] - #[serde(skip_serializing_if = "Option::is_none")] - pub function: Option, - // MCP-specific fields (used when type == "mcp") - #[serde(skip_serializing_if = "Option::is_none")] - pub server_url: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub authorization: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub server_label: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub server_description: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub require_approval: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub allowed_tools: Option>, -} - -impl Default for ResponseTool { - fn default() -> Self { - Self { - r#type: ResponseToolType::WebSearchPreview, - function: None, - server_url: None, - authorization: None, - server_label: None, - server_description: None, - require_approval: None, - allowed_tools: None, - } - } -} - -#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)] -#[serde(rename_all = "snake_case")] -pub enum ResponseToolType { - Function, - WebSearchPreview, - CodeInterpreter, - Mcp, -} - -// ============================================================================ -// Reasoning Parameters -// ============================================================================ - -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct ResponseReasoningParam { - #[serde(default = "default_reasoning_effort")] - #[serde(skip_serializing_if = "Option::is_none")] - pub effort: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub summary: Option, -} - -fn default_reasoning_effort() -> Option { - Some(ReasoningEffort::Medium) -} - -#[derive(Debug, Clone, Deserialize, Serialize)] -#[serde(rename_all = "snake_case")] -pub enum ReasoningEffort { - Minimal, - Low, - Medium, - High, -} - -#[derive(Debug, Clone, Deserialize, Serialize)] -#[serde(rename_all = "snake_case")] -pub enum ReasoningSummary { - Auto, - Concise, - Detailed, -} - -// ============================================================================ -// Input/Output Items -// ============================================================================ - -/// Content can be either a simple string or array of content parts (for SimpleInputMessage) -#[derive(Debug, Clone, Deserialize, Serialize)] -#[serde(untagged)] -pub enum StringOrContentParts { - String(String), - Array(Vec), -} - -#[derive(Debug, Clone, Deserialize, Serialize)] -#[serde(tag = "type")] -#[serde(rename_all = "snake_case")] -pub enum ResponseInputOutputItem { - #[serde(rename = "message")] - Message { - id: String, - role: String, - content: Vec, - #[serde(skip_serializing_if = "Option::is_none")] - status: Option, - }, - #[serde(rename = "reasoning")] - Reasoning { - id: String, - summary: Vec, - #[serde(skip_serializing_if = "Vec::is_empty")] - #[serde(default)] - content: Vec, - #[serde(skip_serializing_if = "Option::is_none")] - status: Option, - }, - #[serde(rename = "function_call")] - FunctionToolCall { - id: String, - call_id: String, - name: String, - arguments: String, - #[serde(skip_serializing_if = "Option::is_none")] - output: Option, - #[serde(skip_serializing_if = "Option::is_none")] - status: Option, - }, - #[serde(rename = "function_call_output")] - FunctionCallOutput { - id: Option, - call_id: String, - output: String, - #[serde(skip_serializing_if = "Option::is_none")] - status: Option, - }, - #[serde(untagged)] - SimpleInputMessage { - content: StringOrContentParts, - role: String, - #[serde(skip_serializing_if = "Option::is_none")] - #[serde(rename = "type")] - r#type: Option, - }, -} - -#[derive(Debug, Clone, Deserialize, Serialize)] -#[serde(tag = "type")] -#[serde(rename_all = "snake_case")] -pub enum ResponseContentPart { - #[serde(rename = "output_text")] - OutputText { - text: String, - #[serde(default)] - #[serde(skip_serializing_if = "Vec::is_empty")] - annotations: Vec, - #[serde(skip_serializing_if = "Option::is_none")] - logprobs: Option, - }, - #[serde(rename = "input_text")] - InputText { text: String }, - #[serde(other)] - Unknown, -} - -#[derive(Debug, Clone, Deserialize, Serialize)] -#[serde(tag = "type")] -#[serde(rename_all = "snake_case")] -pub enum ResponseReasoningContent { - #[serde(rename = "reasoning_text")] - ReasoningText { text: String }, -} - -/// MCP Tool information for the mcp_list_tools output item -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct McpToolInfo { - pub name: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub description: Option, - pub input_schema: Value, - #[serde(skip_serializing_if = "Option::is_none")] - pub annotations: Option, -} - -#[derive(Debug, Clone, Deserialize, Serialize)] -#[serde(tag = "type")] -#[serde(rename_all = "snake_case")] -pub enum ResponseOutputItem { - #[serde(rename = "message")] - Message { - id: String, - role: String, - content: Vec, - status: String, - }, - #[serde(rename = "reasoning")] - Reasoning { - id: String, - summary: Vec, - content: Vec, - #[serde(skip_serializing_if = "Option::is_none")] - status: Option, - }, - #[serde(rename = "function_call")] - FunctionToolCall { - id: String, - call_id: String, - name: String, - arguments: String, - #[serde(skip_serializing_if = "Option::is_none")] - output: Option, - status: String, - }, - #[serde(rename = "mcp_list_tools")] - McpListTools { - id: String, - server_label: String, - tools: Vec, - }, - #[serde(rename = "mcp_call")] - McpCall { - id: String, - status: String, - #[serde(skip_serializing_if = "Option::is_none")] - approval_request_id: Option, - arguments: String, - #[serde(skip_serializing_if = "Option::is_none")] - error: Option, - name: String, - output: String, - server_label: String, - }, -} - -// ============================================================================ -// Configuration Enums -// ============================================================================ - -#[derive(Debug, Clone, Deserialize, Serialize, Default)] -#[serde(rename_all = "snake_case")] -pub enum ServiceTier { - #[default] - Auto, - Default, - Flex, - Scale, - Priority, -} - -#[derive(Debug, Clone, Deserialize, Serialize, Default)] -#[serde(rename_all = "snake_case")] -pub enum Truncation { - Auto, - #[default] - Disabled, -} - -#[derive(Debug, Clone, PartialEq, Deserialize, Serialize)] -#[serde(rename_all = "snake_case")] -pub enum ResponseStatus { - Queued, - InProgress, - Completed, - Failed, - Cancelled, -} - -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct ReasoningInfo { - #[serde(skip_serializing_if = "Option::is_none")] - pub effort: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub summary: Option, -} - -// ============================================================================ -// Text Format (structured outputs) -// ============================================================================ - -/// Text configuration for structured output requests -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct TextConfig { - #[serde(skip_serializing_if = "Option::is_none")] - pub format: Option, -} - -/// Text format: text (default), json_object (legacy), or json_schema (recommended) -#[derive(Debug, Clone, Deserialize, Serialize)] -#[serde(tag = "type")] -pub enum TextFormat { - #[serde(rename = "text")] - Text, - - #[serde(rename = "json_object")] - JsonObject, - - #[serde(rename = "json_schema")] - JsonSchema { - name: String, - schema: Value, - #[serde(skip_serializing_if = "Option::is_none")] - description: Option, - #[serde(skip_serializing_if = "Option::is_none")] - strict: Option, - }, -} - -#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)] -#[serde(rename_all = "snake_case")] -pub enum IncludeField { - #[serde(rename = "code_interpreter_call.outputs")] - CodeInterpreterCallOutputs, - #[serde(rename = "computer_call_output.output.image_url")] - ComputerCallOutputImageUrl, - #[serde(rename = "file_search_call.results")] - FileSearchCallResults, - #[serde(rename = "message.input_image.image_url")] - MessageInputImageUrl, - #[serde(rename = "message.output_text.logprobs")] - MessageOutputTextLogprobs, - #[serde(rename = "reasoning.encrypted_content")] - ReasoningEncryptedContent, -} - -// ============================================================================ -// Usage Types (Responses API format) -// ============================================================================ - -/// OpenAI Responses API usage format (different from standard UsageInfo) -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct ResponseUsage { - pub input_tokens: u32, - pub output_tokens: u32, - pub total_tokens: u32, - #[serde(skip_serializing_if = "Option::is_none")] - pub input_tokens_details: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub output_tokens_details: Option, -} - -#[derive(Debug, Clone, Deserialize, Serialize)] -#[serde(untagged)] -pub enum ResponsesUsage { - Classic(UsageInfo), - Modern(ResponseUsage), -} - -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct InputTokensDetails { - pub cached_tokens: u32, -} - -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct OutputTokensDetails { - pub reasoning_tokens: u32, -} - -impl UsageInfo { - /// Convert to OpenAI Responses API format - pub fn to_response_usage(&self) -> ResponseUsage { - ResponseUsage { - input_tokens: self.prompt_tokens, - output_tokens: self.completion_tokens, - total_tokens: self.total_tokens, - input_tokens_details: self.prompt_tokens_details.as_ref().map(|details| { - InputTokensDetails { - cached_tokens: details.cached_tokens, - } - }), - output_tokens_details: self.reasoning_tokens.map(|tokens| OutputTokensDetails { - reasoning_tokens: tokens, - }), - } - } -} - -impl From for ResponseUsage { - fn from(usage: UsageInfo) -> Self { - usage.to_response_usage() - } -} - -impl ResponseUsage { - /// Convert back to standard UsageInfo format - pub fn to_usage_info(&self) -> UsageInfo { - UsageInfo { - prompt_tokens: self.input_tokens, - completion_tokens: self.output_tokens, - total_tokens: self.total_tokens, - reasoning_tokens: self - .output_tokens_details - .as_ref() - .map(|details| details.reasoning_tokens), - prompt_tokens_details: self.input_tokens_details.as_ref().map(|details| { - PromptTokenUsageInfo { - cached_tokens: details.cached_tokens, - } - }), - } - } -} - -#[derive(Debug, Clone, Default, Deserialize, Serialize)] -pub struct ResponsesGetParams { - #[serde(default)] - pub include: Vec, - #[serde(default)] - pub include_obfuscation: Option, - #[serde(default)] - pub starting_after: Option, - #[serde(default)] - pub stream: Option, -} - -impl ResponsesUsage { - pub fn to_response_usage(&self) -> ResponseUsage { - match self { - ResponsesUsage::Classic(usage) => usage.to_response_usage(), - ResponsesUsage::Modern(usage) => usage.clone(), - } - } - - pub fn to_usage_info(&self) -> UsageInfo { - match self { - ResponsesUsage::Classic(usage) => usage.clone(), - ResponsesUsage::Modern(usage) => usage.to_usage_info(), - } - } -} - -// ============================================================================ -// Helper Functions for Defaults -// ============================================================================ - -fn default_top_k() -> i32 { - -1 -} - -fn default_repetition_penalty() -> f32 { - 1.0 -} - -fn default_temperature() -> Option { - Some(1.0) -} - -fn default_top_p() -> Option { - Some(1.0) -} - -// ============================================================================ -// Request/Response Types -// ============================================================================ - -#[derive(Debug, Clone, Deserialize, Serialize, Validate)] -#[validate(schema(function = "validate_responses_cross_parameters"))] -pub struct ResponsesRequest { - /// Run the request in the background - #[serde(skip_serializing_if = "Option::is_none")] - pub background: Option, - - /// Fields to include in the response - #[serde(skip_serializing_if = "Option::is_none")] - pub include: Option>, - - /// Input content - can be string or structured items - #[validate(custom(function = "validate_response_input"))] - pub input: ResponseInput, - - /// System instructions for the model - #[serde(skip_serializing_if = "Option::is_none")] - pub instructions: Option, - - /// Maximum number of output tokens - #[serde(skip_serializing_if = "Option::is_none")] - #[validate(range(min = 1))] - pub max_output_tokens: Option, - - /// Maximum number of tool calls - #[serde(skip_serializing_if = "Option::is_none")] - #[validate(range(min = 1))] - pub max_tool_calls: Option, - - /// Additional metadata - #[serde(skip_serializing_if = "Option::is_none")] - pub metadata: Option>, - - /// Model to use - #[serde(default = "default_model")] - pub model: String, - - /// Optional conversation id to persist input/output as items - #[serde(skip_serializing_if = "Option::is_none")] - #[validate(custom(function = "validate_conversation_id"))] - pub conversation: Option, - - /// Whether to enable parallel tool calls - #[serde(skip_serializing_if = "Option::is_none")] - pub parallel_tool_calls: Option, - - /// ID of previous response to continue from - #[serde(skip_serializing_if = "Option::is_none")] - pub previous_response_id: Option, - - /// Reasoning configuration - #[serde(skip_serializing_if = "Option::is_none")] - pub reasoning: Option, - - /// Service tier - #[serde(skip_serializing_if = "Option::is_none")] - pub service_tier: Option, - - /// Whether to store the response - #[serde(skip_serializing_if = "Option::is_none")] - pub store: Option, - - /// Whether to stream the response - #[serde(default)] - pub stream: Option, - - /// Temperature for sampling - #[serde( - default = "default_temperature", - skip_serializing_if = "Option::is_none" - )] - #[validate(range(min = 0.0, max = 2.0))] - pub temperature: Option, - - /// Tool choice behavior - #[serde(skip_serializing_if = "Option::is_none")] - pub tool_choice: Option, - - /// Available tools - #[serde(skip_serializing_if = "Option::is_none")] - #[validate(custom(function = "validate_response_tools"))] - pub tools: Option>, - - /// Number of top logprobs to return - #[serde(skip_serializing_if = "Option::is_none")] - #[validate(range(min = 0, max = 20))] - pub top_logprobs: Option, - - /// Top-p sampling parameter - #[serde(default = "default_top_p", skip_serializing_if = "Option::is_none")] - #[validate(custom(function = "validate_top_p_value"))] - pub top_p: Option, - - /// Truncation behavior - #[serde(skip_serializing_if = "Option::is_none")] - pub truncation: Option, - - /// Text format for structured outputs (text, json_object, json_schema) - #[serde(skip_serializing_if = "Option::is_none")] - #[validate(custom(function = "validate_text_format"))] - pub text: Option, - - /// User identifier - #[serde(skip_serializing_if = "Option::is_none")] - pub user: Option, - - /// Request ID - #[serde(skip_serializing_if = "Option::is_none")] - pub request_id: Option, - - /// Request priority - #[serde(default)] - pub priority: i32, - - /// Frequency penalty - #[serde(skip_serializing_if = "Option::is_none")] - #[validate(range(min = -2.0, max = 2.0))] - pub frequency_penalty: Option, - - /// Presence penalty - #[serde(skip_serializing_if = "Option::is_none")] - #[validate(range(min = -2.0, max = 2.0))] - pub presence_penalty: Option, - - /// Stop sequences - #[serde(skip_serializing_if = "Option::is_none")] - #[validate(custom(function = "validate_stop"))] - pub stop: Option, - - /// Top-k sampling parameter (SGLang extension) - #[serde(default = "default_top_k")] - #[validate(custom(function = "validate_top_k_value"))] - pub top_k: i32, - - /// Min-p sampling parameter (SGLang extension) - #[serde(default)] - #[validate(range(min = 0.0, max = 1.0))] - pub min_p: f32, - - /// Repetition penalty (SGLang extension) - #[serde(default = "default_repetition_penalty")] - #[validate(range(min = 0.0, max = 2.0))] - pub repetition_penalty: f32, -} - -#[derive(Debug, Clone, Deserialize, Serialize)] -#[serde(untagged)] -pub enum ResponseInput { - Items(Vec), - Text(String), -} - -impl Default for ResponsesRequest { - fn default() -> Self { - Self { - background: None, - include: None, - input: ResponseInput::Text(String::new()), - instructions: None, - max_output_tokens: None, - max_tool_calls: None, - metadata: None, - model: default_model(), - conversation: None, - parallel_tool_calls: None, - previous_response_id: None, - reasoning: None, - service_tier: None, - store: None, - stream: None, - temperature: None, - tool_choice: None, - tools: None, - top_logprobs: None, - top_p: None, - truncation: None, - text: None, - user: None, - request_id: None, - priority: 0, - frequency_penalty: None, - presence_penalty: None, - stop: None, - top_k: default_top_k(), - min_p: 0.0, - repetition_penalty: default_repetition_penalty(), - } - } -} - -impl Normalizable for ResponsesRequest { - /// Normalize the request by applying defaults: - /// 1. Apply tool_choice defaults based on tools presence - /// 2. Apply parallel_tool_calls defaults - /// 3. Apply store field defaults - fn normalize(&mut self) { - // 1. Apply tool_choice defaults - if self.tool_choice.is_none() { - if let Some(tools) = &self.tools { - let choice_value = if !tools.is_empty() { - ToolChoiceValue::Auto - } else { - ToolChoiceValue::None - }; - self.tool_choice = Some(ToolChoice::Value(choice_value)); - } - // If tools is None, leave tool_choice as None (don't set it) - } - - // 2. Apply default for parallel_tool_calls if tools are present - if self.parallel_tool_calls.is_none() && self.tools.is_some() { - self.parallel_tool_calls = Some(true); - } - - // 3. Ensure store defaults to true if not specified - if self.store.is_none() { - self.store = Some(true); - } - } -} - -impl GenerationRequest for ResponsesRequest { - fn is_stream(&self) -> bool { - self.stream.unwrap_or(false) - } - - fn get_model(&self) -> Option<&str> { - Some(self.model.as_str()) - } - - fn extract_text_for_routing(&self) -> String { - match &self.input { - ResponseInput::Text(text) => text.clone(), - ResponseInput::Items(items) => items - .iter() - .filter_map(|item| match item { - ResponseInputOutputItem::Message { content, .. } => { - let texts: Vec = content - .iter() - .filter_map(|part| match part { - ResponseContentPart::OutputText { text, .. } => Some(text.clone()), - ResponseContentPart::InputText { text } => Some(text.clone()), - ResponseContentPart::Unknown => None, - }) - .collect(); - if texts.is_empty() { - None - } else { - Some(texts.join(" ")) - } - } - ResponseInputOutputItem::SimpleInputMessage { content, .. } => { - match content { - StringOrContentParts::String(s) => Some(s.clone()), - StringOrContentParts::Array(parts) => { - // SimpleInputMessage only supports InputText - let texts: Vec = parts - .iter() - .filter_map(|part| match part { - ResponseContentPart::InputText { text } => { - Some(text.clone()) - } - _ => None, - }) - .collect(); - if texts.is_empty() { - None - } else { - Some(texts.join(" ")) - } - } - } - } - ResponseInputOutputItem::Reasoning { content, .. } => { - let texts: Vec = content - .iter() - .map(|part| match part { - ResponseReasoningContent::ReasoningText { text } => text.clone(), - }) - .collect(); - if texts.is_empty() { - None - } else { - Some(texts.join(" ")) - } - } - ResponseInputOutputItem::FunctionToolCall { arguments, .. } => { - Some(arguments.clone()) - } - ResponseInputOutputItem::FunctionCallOutput { output, .. } => { - Some(output.clone()) - } - }) - .collect::>() - .join(" "), - } - } -} - -/// Validate conversation ID format -pub fn validate_conversation_id(conv_id: &str) -> Result<(), validator::ValidationError> { - if !conv_id.starts_with("conv_") { - let mut error = validator::ValidationError::new("invalid_conversation_id"); - error.message = Some(std::borrow::Cow::Owned(format!( - "Invalid 'conversation': '{}'. Expected an ID that begins with 'conv_'.", - conv_id - ))); - return Err(error); - } - - // Check if the conversation ID contains only valid characters - let is_valid = conv_id - .chars() - .all(|c| c.is_alphanumeric() || c == '_' || c == '-'); - - if !is_valid { - let mut error = validator::ValidationError::new("invalid_conversation_id"); - error.message = Some(std::borrow::Cow::Owned(format!( - "Invalid 'conversation': '{}'. Expected an ID that contains letters, numbers, underscores, or dashes, but this value contained additional characters.", - conv_id - ))); - return Err(error); - } - Ok(()) -} - -/// Validates tool_choice requires tools and references exist -fn validate_tool_choice_with_tools( - request: &ResponsesRequest, -) -> Result<(), validator::ValidationError> { - let Some(tool_choice) = &request.tool_choice else { - return Ok(()); - }; - - let has_tools = request.tools.as_ref().is_some_and(|t| !t.is_empty()); - let is_some_choice = !matches!(tool_choice, ToolChoice::Value(ToolChoiceValue::None)); - - // Check if tool_choice requires tools but none are provided - if is_some_choice && !has_tools { - let mut e = validator::ValidationError::new("tool_choice_requires_tools"); - e.message = Some("Invalid value for 'tool_choice': 'tool_choice' is only allowed when 'tools' are specified.".into()); - return Err(e); - } - - // Validate tool references exist when tools are present - if !has_tools { - return Ok(()); - } - - // Extract function tool names from ResponseTools - let tools = request.tools.as_ref().unwrap(); - let function_tool_names: Vec<&str> = tools - .iter() - .filter_map(|t| match t.r#type { - ResponseToolType::Function => t.function.as_ref().map(|f| f.name.as_str()), - _ => None, - }) - .collect(); - - // Validate tool references exist - match tool_choice { - ToolChoice::Function { function, .. } => { - if !function_tool_names.contains(&function.name.as_str()) { - let mut e = validator::ValidationError::new("tool_choice_function_not_found"); - e.message = Some( - format!( - "Invalid value for 'tool_choice': function '{}' not found in 'tools'.", - function.name - ) - .into(), - ); - return Err(e); - } - } - ToolChoice::AllowedTools { - mode, - tools: allowed_tools, - .. - } => { - // Validate mode is "auto" or "required" - if mode != "auto" && mode != "required" { - let mut e = validator::ValidationError::new("tool_choice_invalid_mode"); - e.message = Some( - format!( - "Invalid value for 'tool_choice.mode': must be 'auto' or 'required', got '{}'.", - mode - ) - .into(), - ); - return Err(e); - } - - // Validate that all function tool references exist - for tool_ref in allowed_tools { - if let ToolReference::Function { name } = tool_ref { - if !function_tool_names.contains(&name.as_str()) { - let mut e = validator::ValidationError::new("tool_choice_tool_not_found"); - e.message = Some( - format!( - "Invalid value for 'tool_choice.tools': tool '{}' not found in 'tools'.", - name - ) - .into(), - ); - return Err(e); - } - } - // Note: MCP and hosted tools don't need existence validation here - // as they are resolved dynamically at runtime - } - } - _ => {} - } - - Ok(()) -} - -/// Schema-level validation for cross-field dependencies -fn validate_responses_cross_parameters( - request: &ResponsesRequest, -) -> Result<(), validator::ValidationError> { - // 1. Validate tool_choice requires tools (enhanced) - validate_tool_choice_with_tools(request)?; - - // 2. Validate top_logprobs requires include field - if request.top_logprobs.is_some() { - let has_logprobs_include = request - .include - .as_ref() - .is_some_and(|inc| inc.contains(&IncludeField::MessageOutputTextLogprobs)); - - if !has_logprobs_include { - let mut e = validator::ValidationError::new("top_logprobs_requires_include"); - e.message = Some( - "top_logprobs requires include field with 'message.output_text.logprobs'".into(), - ); - return Err(e); - } - } - - // 3. Validate background/stream conflict - if request.background == Some(true) && request.stream == Some(true) { - let mut e = validator::ValidationError::new("background_conflicts_with_stream"); - e.message = Some("Cannot use background mode with streaming".into()); - return Err(e); - } - - // 4. Validate conversation and previous_response_id are mutually exclusive - if request.conversation.is_some() && request.previous_response_id.is_some() { - let mut e = validator::ValidationError::new("mutually_exclusive_parameters"); - e.message = Some("Mutually exclusive parameters. Ensure you are only providing one of: 'previous_response_id' or 'conversation'.".into()); - return Err(e); - } - - // 5. Validate input items structure - if let ResponseInput::Items(items) = &request.input { - // Check for at least one valid input message - let has_valid_input = items.iter().any(|item| { - matches!( - item, - ResponseInputOutputItem::Message { .. } - | ResponseInputOutputItem::SimpleInputMessage { .. } - ) - }); - - if !has_valid_input { - let mut e = validator::ValidationError::new("input_missing_user_message"); - e.message = Some("Input items must contain at least one message".into()); - return Err(e); - } - } - - // 6. Validate text format conflicts (for future structured output constraints) - // Currently, Responses API doesn't have regex/ebnf like Chat API, - // but this is here for completeness and future-proofing - - Ok(()) -} - -// ============================================================================ -// Field-Level Validation Functions -// ============================================================================ - -/// Validates response input is not empty and has valid content -fn validate_response_input(input: &ResponseInput) -> Result<(), validator::ValidationError> { - match input { - ResponseInput::Text(text) => { - if text.is_empty() { - let mut e = validator::ValidationError::new("input_text_empty"); - e.message = Some("Input text cannot be empty".into()); - return Err(e); - } - } - ResponseInput::Items(items) => { - if items.is_empty() { - let mut e = validator::ValidationError::new("input_items_empty"); - e.message = Some("Input items cannot be empty".into()); - return Err(e); - } - // Validate each item has valid content - for item in items { - validate_input_item(item)?; - } - } - } - Ok(()) -} - -/// Validates individual input items have valid content -fn validate_input_item(item: &ResponseInputOutputItem) -> Result<(), validator::ValidationError> { - match item { - ResponseInputOutputItem::Message { content, .. } => { - if content.is_empty() { - let mut e = validator::ValidationError::new("message_content_empty"); - e.message = Some("Message content cannot be empty".into()); - return Err(e); - } - } - ResponseInputOutputItem::SimpleInputMessage { content, .. } => match content { - StringOrContentParts::String(s) if s.is_empty() => { - let mut e = validator::ValidationError::new("message_content_empty"); - e.message = Some("Message content cannot be empty".into()); - return Err(e); - } - StringOrContentParts::Array(parts) if parts.is_empty() => { - let mut e = validator::ValidationError::new("message_content_empty"); - e.message = Some("Message content parts cannot be empty".into()); - return Err(e); - } - _ => {} - }, - ResponseInputOutputItem::Reasoning { .. } => { - // Reasoning content can be empty - no validation needed - } - ResponseInputOutputItem::FunctionCallOutput { output, .. } => { - if output.is_empty() { - let mut e = validator::ValidationError::new("function_output_empty"); - e.message = Some("Function call output cannot be empty".into()); - return Err(e); - } - } - _ => {} - } - Ok(()) -} - -/// Validates ResponseTool structure based on tool type -fn validate_response_tools(tools: &[ResponseTool]) -> Result<(), validator::ValidationError> { - for tool in tools { - match tool.r#type { - ResponseToolType::Function => { - if tool.function.is_none() { - let mut e = validator::ValidationError::new("function_tool_missing_function"); - e.message = Some("Function tool must have a function definition".into()); - return Err(e); - } - } - ResponseToolType::Mcp => { - if tool.server_url.is_none() { - let mut e = validator::ValidationError::new("mcp_tool_missing_server_url"); - e.message = Some("MCP tool must have a server_url".into()); - return Err(e); - } - } - _ => {} - } - } - Ok(()) -} - -/// Validates text format configuration (JSON schema name cannot be empty) -fn validate_text_format(text: &TextConfig) -> Result<(), validator::ValidationError> { - if let Some(TextFormat::JsonSchema { name, .. }) = &text.format { - if name.is_empty() { - let mut e = validator::ValidationError::new("json_schema_name_empty"); - e.message = Some("JSON schema name cannot be empty".into()); - return Err(e); - } - } - Ok(()) -} - -/// Normalize a SimpleInputMessage to a proper Message item -/// -/// This helper converts SimpleInputMessage (which can have flexible content) -/// into a fully-structured Message item with a generated ID, role, and content array. -/// -/// SimpleInputMessage items are converted to Message items with IDs generated using -/// the centralized ID generation pattern with "msg_" prefix for consistency. -/// -/// # Arguments -/// * `item` - The input item to normalize -/// -/// # Returns -/// A normalized ResponseInputOutputItem (either Message if converted, or original if not SimpleInputMessage) -pub fn normalize_input_item(item: &ResponseInputOutputItem) -> ResponseInputOutputItem { - match item { - ResponseInputOutputItem::SimpleInputMessage { content, role, .. } => { - let content_vec = match content { - StringOrContentParts::String(s) => { - vec![ResponseContentPart::InputText { text: s.clone() }] - } - StringOrContentParts::Array(parts) => parts.clone(), - }; - - ResponseInputOutputItem::Message { - id: generate_id("msg"), - role: role.clone(), - content: content_vec, - status: Some("completed".to_string()), - } - } - _ => item.clone(), - } -} - -pub fn generate_id(prefix: &str) -> String { - use rand::RngCore; - let mut rng = rand::rng(); - // Generate exactly 50 hex characters (25 bytes) for the part after the underscore - let mut bytes = [0u8; 25]; - rng.fill_bytes(&mut bytes); - let hex_string: String = bytes.iter().map(|b| format!("{:02x}", b)).collect(); - format!("{}_{}", prefix, hex_string) -} - -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct ResponsesResponse { - /// Response ID - pub id: String, - - /// Object type - #[serde(default = "default_object_type")] - pub object: String, - - /// Creation timestamp - pub created_at: i64, - - /// Response status - pub status: ResponseStatus, - - /// Error information if status is failed - #[serde(skip_serializing_if = "Option::is_none")] - pub error: Option, - - /// Incomplete details if response was truncated - #[serde(skip_serializing_if = "Option::is_none")] - pub incomplete_details: Option, - - /// System instructions used - #[serde(skip_serializing_if = "Option::is_none")] - pub instructions: Option, - - /// Max output tokens setting - #[serde(skip_serializing_if = "Option::is_none")] - pub max_output_tokens: Option, - - /// Model name - pub model: String, - - /// Output items - #[serde(default)] - pub output: Vec, - - /// Whether parallel tool calls are enabled - #[serde(default = "default_true")] - pub parallel_tool_calls: bool, - - /// Previous response ID if this is a continuation - #[serde(skip_serializing_if = "Option::is_none")] - pub previous_response_id: Option, - - /// Reasoning information - #[serde(skip_serializing_if = "Option::is_none")] - pub reasoning: Option, - - /// Whether the response is stored - #[serde(default = "default_true")] - pub store: bool, - - /// Temperature setting used - #[serde(skip_serializing_if = "Option::is_none")] - pub temperature: Option, - - /// Text format settings - #[serde(skip_serializing_if = "Option::is_none")] - pub text: Option, - - /// Tool choice setting - #[serde(default = "default_tool_choice")] - pub tool_choice: String, - - /// Available tools - #[serde(default)] - pub tools: Vec, - - /// Top-p setting used - #[serde(skip_serializing_if = "Option::is_none")] - pub top_p: Option, - - /// Truncation strategy used - #[serde(skip_serializing_if = "Option::is_none")] - pub truncation: Option, - - /// Usage information - #[serde(skip_serializing_if = "Option::is_none")] - pub usage: Option, - - /// User identifier - #[serde(skip_serializing_if = "Option::is_none")] - pub user: Option, - - /// Safety identifier for content moderation - #[serde(skip_serializing_if = "Option::is_none")] - pub safety_identifier: Option, - - /// Additional metadata - #[serde(default)] - pub metadata: HashMap, -} - -fn default_object_type() -> String { - "response".to_string() -} - -fn default_tool_choice() -> String { - "auto".to_string() -} - -impl ResponsesResponse { - /// Create a builder for constructing a ResponsesResponse - pub fn builder(id: impl Into, model: impl Into) -> ResponsesResponseBuilder { - ResponsesResponseBuilder::new(id, model) - } - - /// Check if the response is complete - pub fn is_complete(&self) -> bool { - matches!(self.status, ResponseStatus::Completed) - } - - /// Check if the response is in progress - pub fn is_in_progress(&self) -> bool { - matches!(self.status, ResponseStatus::InProgress) - } - - /// Check if the response failed - pub fn is_failed(&self) -> bool { - matches!(self.status, ResponseStatus::Failed) - } -} - -impl ResponseOutputItem { - /// Create a new message output item - pub fn new_message( - id: String, - role: String, - content: Vec, - status: String, - ) -> Self { - Self::Message { - id, - role, - content, - status, - } - } - - /// Create a new reasoning output item - pub fn new_reasoning( - id: String, - summary: Vec, - content: Vec, - status: Option, - ) -> Self { - Self::Reasoning { - id, - summary, - content, - status, - } - } - - /// Create a new function tool call output item - pub fn new_function_tool_call( - id: String, - call_id: String, - name: String, - arguments: String, - output: Option, - status: String, - ) -> Self { - Self::FunctionToolCall { - id, - call_id, - name, - arguments, - output, - status, - } - } -} - -impl ResponseContentPart { - /// Create a new text content part - pub fn new_text( - text: String, - annotations: Vec, - logprobs: Option, - ) -> Self { - Self::OutputText { - text, - annotations, - logprobs, - } - } -} - -impl ResponseReasoningContent { - /// Create a new reasoning text content - pub fn new_reasoning_text(text: String) -> Self { - Self::ReasoningText { text } - } -} diff --git a/sgl-model-gateway/src/protocols/sampling_params.rs b/sgl-model-gateway/src/protocols/sampling_params.rs deleted file mode 100644 index 9055a53dd..000000000 --- a/sgl-model-gateway/src/protocols/sampling_params.rs +++ /dev/null @@ -1,119 +0,0 @@ -use serde::{Deserialize, Serialize}; -use validator::Validate; - -use super::common::StringOrArray; - -/// Sampling parameters for text generation -#[derive(Debug, Clone, Deserialize, Serialize, Default, Validate)] -#[validate(schema(function = "validate_sampling_params"))] -pub struct SamplingParams { - /// Temperature for sampling (must be >= 0.0, no upper limit) - #[serde(skip_serializing_if = "Option::is_none")] - #[validate(range(min = 0.0))] - pub temperature: Option, - /// Maximum number of new tokens to generate (must be >= 0) - #[serde(skip_serializing_if = "Option::is_none")] - #[validate(range(min = 0))] - pub max_new_tokens: Option, - /// Top-p nucleus sampling (0.0 < top_p <= 1.0) - #[serde(skip_serializing_if = "Option::is_none")] - #[validate(custom(function = "validate_top_p_value"))] - pub top_p: Option, - /// Top-k sampling (-1 to disable, or >= 1) - #[serde(skip_serializing_if = "Option::is_none")] - #[validate(custom(function = "validate_top_k_value"))] - pub top_k: Option, - #[serde(skip_serializing_if = "Option::is_none")] - #[validate(range(min = -2.0, max = 2.0))] - pub frequency_penalty: Option, - #[serde(skip_serializing_if = "Option::is_none")] - #[validate(range(min = -2.0, max = 2.0))] - pub presence_penalty: Option, - #[serde(skip_serializing_if = "Option::is_none")] - #[validate(range(min = 0.0, max = 2.0))] - pub repetition_penalty: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub stop: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub ignore_eos: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub skip_special_tokens: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub json_schema: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub regex: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub ebnf: Option, - #[serde(skip_serializing_if = "Option::is_none")] - #[validate(range(min = 0.0, max = 1.0))] - pub min_p: Option, - /// Minimum number of new tokens (validated in schema function for cross-field check with max_new_tokens) - #[serde(skip_serializing_if = "Option::is_none")] - pub min_new_tokens: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub stop_token_ids: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - pub no_stop_trim: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub n: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub sampling_seed: Option, -} - -// ============================================================================ -// Shared Validation Functions -// ============================================================================ - -/// Validates top_p: 0.0 < top_p <= 1.0 (can't use range validator for open interval) -pub fn validate_top_p_value(top_p: f32) -> Result<(), validator::ValidationError> { - if !(top_p > 0.0 && top_p <= 1.0) { - return Err(validator::ValidationError::new( - "top_p must be in (0, 1] - greater than 0.0 and at most 1.0", - )); - } - Ok(()) -} - -/// Validates top_k: -1 (disabled) or >= 1 (special -1 case - can't use range validator) -pub fn validate_top_k_value(top_k: i32) -> Result<(), validator::ValidationError> { - if top_k != -1 && top_k < 1 { - return Err(validator::ValidationError::new( - "top_k must be -1 (disabled) or at least 1", - )); - } - Ok(()) -} - -// ============================================================================ -// SamplingParams-Specific Validation -// ============================================================================ - -/// Validation function for SamplingParams - cross-field validation only -fn validate_sampling_params(params: &SamplingParams) -> Result<(), validator::ValidationError> { - // 1. Cross-field validation: min_new_tokens <= max_new_tokens - if let (Some(min), Some(max)) = (params.min_new_tokens, params.max_new_tokens) { - if min > max { - return Err(validator::ValidationError::new( - "min_new_tokens cannot exceed max_new_tokens", - )); - } - } - - // 2. Validate mutually exclusive structured output constraints - let constraint_count = [ - params.regex.is_some(), - params.ebnf.is_some(), - params.json_schema.is_some(), - ] - .iter() - .filter(|&&x| x) - .count(); - - if constraint_count > 1 { - return Err(validator::ValidationError::new( - "only one of regex, ebnf, or json_schema can be set", - )); - } - - Ok(()) -} diff --git a/sgl-model-gateway/src/protocols/tokenize.rs b/sgl-model-gateway/src/protocols/tokenize.rs deleted file mode 100644 index 62a19c23b..000000000 --- a/sgl-model-gateway/src/protocols/tokenize.rs +++ /dev/null @@ -1,279 +0,0 @@ -//! Tokenize and Detokenize API protocol types -//! -//! These types mirror the SGLang Python implementation for compatibility. -//! See: python/sglang/srt/entrypoints/openai/protocol.py - -use serde::{Deserialize, Serialize}; - -use super::UNKNOWN_MODEL_ID; - -// ============================================================================ -// Tokenize API -// ============================================================================ - -/// Request schema for the /v1/tokenize endpoint -/// -/// Supports both single string and batch tokenization. -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct TokenizeRequest { - /// Model name for tokenizer selection - #[serde(default = "default_model_name")] - pub model: String, - - /// Text(s) to tokenize - can be a single string or array of strings - pub prompt: StringOrArray, -} - -/// Response schema for the /v1/tokenize endpoint -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct TokenizeResponse { - /// Token IDs - single list for single input, nested list for batch - pub tokens: TokensResult, - - /// Token count(s) - single int for single input, list for batch - pub count: CountResult, - - /// Character count(s) of input - single int for single input, list for batch - pub char_count: CountResult, -} - -/// Token IDs result - either single or batch -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(untagged)] -pub enum TokensResult { - Single(Vec), - Batch(Vec>), -} - -/// Count result - either single or batch -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(untagged)] -pub enum CountResult { - Single(i32), - Batch(Vec), -} - -// ============================================================================ -// Detokenize API -// ============================================================================ - -/// Request schema for the /v1/detokenize endpoint -/// -/// Supports both single sequence and batch detokenization. -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct DetokenizeRequest { - /// Model name for tokenizer selection - #[serde(default = "default_model_name")] - pub model: String, - - /// Token IDs to detokenize - single list or batch (list of lists) - pub tokens: TokensInput, - - /// Whether to skip special tokens (e.g., padding or EOS) during decoding - #[serde(default = "default_true")] - pub skip_special_tokens: bool, -} - -/// Token input - either single sequence or batch -#[derive(Debug, Clone, Deserialize, Serialize)] -#[serde(untagged)] -pub enum TokensInput { - /// Single sequence of token IDs - Single(Vec), - /// Batch of token sequences - Batch(Vec>), -} - -impl TokensInput { - /// Check if this is a batch input - pub fn is_batch(&self) -> bool { - matches!(self, TokensInput::Batch(_)) - } - - /// Get the sequences (always returns a vec of vecs for uniform processing) - pub fn sequences(&self) -> Vec<&[u32]> { - match self { - TokensInput::Single(seq) => vec![seq.as_slice()], - TokensInput::Batch(seqs) => seqs.iter().map(|s| s.as_slice()).collect(), - } - } -} - -/// Response schema for the /v1/detokenize endpoint -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct DetokenizeResponse { - /// Decoded text - single string for single input, list for batch - pub text: TextResult, -} - -/// Text result - either single or batch -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(untagged)] -pub enum TextResult { - Single(String), - Batch(Vec), -} - -// ============================================================================ -// Tokenizer Management API -// ============================================================================ - -/// Request schema for adding a tokenizer -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct AddTokenizerRequest { - /// Name to register the tokenizer under - pub name: String, - - /// Source: either a local path or HuggingFace model ID - pub source: String, - - /// Optional path to chat template file - #[serde(skip_serializing_if = "Option::is_none")] - pub chat_template_path: Option, -} - -/// Response schema for adding a tokenizer (async) -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct AddTokenizerResponse { - /// Unique identifier for the tokenizer (UUID) - pub id: String, - /// Status of the request: "pending", "processing", "completed", "failed" - pub status: String, - pub message: String, - /// Vocabulary size of the loaded tokenizer (only set on completion) - #[serde(skip_serializing_if = "Option::is_none")] - pub vocab_size: Option, -} - -/// Response schema for listing tokenizers -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ListTokenizersResponse { - pub tokenizers: Vec, -} - -/// Information about a registered tokenizer -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct TokenizerInfo { - /// Unique identifier (UUID) - pub id: String, - /// User-provided name - pub name: String, - /// Source path or HuggingFace model ID - pub source: String, - pub vocab_size: usize, -} - -/// Request schema for removing a tokenizer -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct RemoveTokenizerRequest { - /// Name of the tokenizer to remove - pub name: String, -} - -/// Response schema for removing a tokenizer -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct RemoveTokenizerResponse { - pub success: bool, - pub message: String, -} - -// ============================================================================ -// Helper Types -// ============================================================================ - -/// String or array of strings (for flexible input) -#[derive(Debug, Clone, Deserialize, Serialize)] -#[serde(untagged)] -pub enum StringOrArray { - Single(String), - Array(Vec), -} - -impl StringOrArray { - /// Check if this is a batch (array) input - pub fn is_batch(&self) -> bool { - matches!(self, StringOrArray::Array(_)) - } - - /// Get all strings as a slice (converts single to vec) - pub fn as_strings(&self) -> Vec<&str> { - match self { - StringOrArray::Single(s) => vec![s.as_str()], - StringOrArray::Array(arr) => arr.iter().map(|s| s.as_str()).collect(), - } - } -} - -// ============================================================================ -// Default Functions -// ============================================================================ - -fn default_model_name() -> String { - UNKNOWN_MODEL_ID.to_string() -} - -fn default_true() -> bool { - true -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_tokenize_request_single() { - let json = r#"{"prompt": "Hello world"}"#; - let req: TokenizeRequest = serde_json::from_str(json).unwrap(); - assert_eq!(req.model, "unknown"); - assert!(matches!(req.prompt, StringOrArray::Single(_))); - } - - #[test] - fn test_tokenize_request_batch() { - let json = r#"{"model": "llama", "prompt": ["Hello", "World"]}"#; - let req: TokenizeRequest = serde_json::from_str(json).unwrap(); - assert_eq!(req.model, "llama"); - assert!(matches!(req.prompt, StringOrArray::Array(_))); - } - - #[test] - fn test_detokenize_request_single() { - let json = r#"{"tokens": [1, 2, 3]}"#; - let req: DetokenizeRequest = serde_json::from_str(json).unwrap(); - assert!(matches!(req.tokens, TokensInput::Single(_))); - assert!(req.skip_special_tokens); - } - - #[test] - fn test_detokenize_request_batch() { - let json = r#"{"tokens": [[1, 2], [3, 4, 5]], "skip_special_tokens": false}"#; - let req: DetokenizeRequest = serde_json::from_str(json).unwrap(); - assert!(matches!(req.tokens, TokensInput::Batch(_))); - assert!(!req.skip_special_tokens); - } - - #[test] - fn test_tokenize_response_single() { - let resp = TokenizeResponse { - tokens: TokensResult::Single(vec![1, 2, 3]), - count: CountResult::Single(3), - char_count: CountResult::Single(11), - }; - let json = serde_json::to_string(&resp).unwrap(); - assert!(json.contains("[1,2,3]")); - assert!(json.contains("\"count\":3")); - assert!(json.contains("\"char_count\":11")); - } - - #[test] - fn test_tokenize_response_batch() { - let resp = TokenizeResponse { - tokens: TokensResult::Batch(vec![vec![1, 2], vec![3, 4, 5]]), - count: CountResult::Batch(vec![2, 3]), - char_count: CountResult::Batch(vec![5, 5]), - }; - let json = serde_json::to_string(&resp).unwrap(); - assert!(json.contains("[[1,2],[3,4,5]]")); - assert!(json.contains("[2,3]")); - } -} diff --git a/sgl-model-gateway/src/protocols/validated.rs b/sgl-model-gateway/src/protocols/validated.rs deleted file mode 100644 index 7eb5a812b..000000000 --- a/sgl-model-gateway/src/protocols/validated.rs +++ /dev/null @@ -1,164 +0,0 @@ -// Validated JSON extractor for automatic request validation -// -// This module provides a ValidatedJson extractor that automatically validates -// requests using the validator crate's Validate trait. - -use axum::{ - extract::{rejection::JsonRejection, FromRequest, Request}, - http::StatusCode, - response::{IntoResponse, Response}, - Json, -}; -use serde::de::DeserializeOwned; -use serde_json::json; -use validator::Validate; - -/// Trait for request types that need post-deserialization normalization -pub trait Normalizable { - /// Normalize the request by applying defaults and transformations - fn normalize(&mut self) { - // Default: no-op - } -} - -/// A JSON extractor that automatically validates and normalizes the request body -/// -/// This extractor deserializes the request body and automatically calls `.validate()` -/// on types that implement the `Validate` trait. If validation fails, it returns -/// a 400 Bad Request with detailed error information. -/// -/// # Example -/// -/// ```rust,ignore -/// async fn create_chat( -/// ValidatedJson(request): ValidatedJson, -/// ) -> Response { -/// // request is guaranteed to be valid here -/// process_request(request).await -/// } -/// ``` -pub struct ValidatedJson(pub T); - -impl FromRequest for ValidatedJson -where - T: DeserializeOwned + Validate + Normalizable + Send, - S: Send + Sync, -{ - type Rejection = Response; - - async fn from_request(req: Request, state: &S) -> Result { - // First, extract and deserialize the JSON - let Json(mut data) = - Json::::from_request(req, state) - .await - .map_err(|err: JsonRejection| { - let error_message = match err { - JsonRejection::JsonDataError(e) => { - format!("Invalid JSON data: {}", e) - } - JsonRejection::JsonSyntaxError(e) => { - format!("JSON syntax error: {}", e) - } - JsonRejection::MissingJsonContentType(_) => { - "Missing Content-Type: application/json header".to_string() - } - _ => format!("Failed to parse JSON: {}", err), - }; - - ( - StatusCode::BAD_REQUEST, - Json(json!({ - "error": { - "message": error_message, - "type": "invalid_request_error", - "code": "json_parse_error" - } - })), - ) - .into_response() - })?; - - // Normalize the request (apply defaults based on other fields) - data.normalize(); - - // Then, automatically validate the data - data.validate().map_err(|validation_errors| { - ( - StatusCode::BAD_REQUEST, - Json(json!({ - "error": { - "message": validation_errors.to_string(), - "type": "invalid_request_error", - "code": 400 - } - })), - ) - .into_response() - })?; - - Ok(ValidatedJson(data)) - } -} - -// Implement Deref to allow transparent access to the inner value -impl std::ops::Deref for ValidatedJson { - type Target = T; - - fn deref(&self) -> &Self::Target { - &self.0 - } -} - -impl std::ops::DerefMut for ValidatedJson { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.0 - } -} - -#[cfg(test)] -mod tests { - use serde::{Deserialize, Serialize}; - use validator::Validate; - - use super::*; - - #[derive(Debug, Deserialize, Serialize, Validate)] - struct TestRequest { - #[validate(range(min = 0.0, max = 1.0))] - value: f32, - #[validate(length(min = 1))] - name: String, - } - - impl Normalizable for TestRequest { - // Use default no-op implementation - } - - #[tokio::test] - async fn test_validated_json_valid() { - // This test is conceptual - actual testing would require Axum test harness - let request = TestRequest { - value: 0.5, - name: "test".to_string(), - }; - assert!(request.validate().is_ok()); - } - - #[tokio::test] - async fn test_validated_json_invalid_range() { - let request = TestRequest { - value: 1.5, // Out of range - name: "test".to_string(), - }; - assert!(request.validate().is_err()); - } - - #[tokio::test] - async fn test_validated_json_invalid_length() { - let request = TestRequest { - value: 0.5, - name: "".to_string(), // Empty name - }; - assert!(request.validate().is_err()); - } -} diff --git a/sgl-model-gateway/src/protocols/worker_spec.rs b/sgl-model-gateway/src/protocols/worker_spec.rs deleted file mode 100644 index afd2340ea..000000000 --- a/sgl-model-gateway/src/protocols/worker_spec.rs +++ /dev/null @@ -1,375 +0,0 @@ -//! Worker management API specifications -//! -//! Defines the request/response structures for worker management endpoints - -use std::collections::HashMap; - -use serde::{Deserialize, Serialize}; - -use super::UNKNOWN_MODEL_ID; - -/// Worker configuration for API requests -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct WorkerConfigRequest { - /// Worker URL (required) - pub url: String, - - /// Worker API key (optional) - #[serde(skip_serializing_if = "Option::is_none")] - pub api_key: Option, - - /// Model ID (optional, will query from server if not provided) - #[serde(skip_serializing_if = "Option::is_none")] - pub model_id: Option, - - /// Worker priority (optional, default: 50, higher = preferred) - #[serde(skip_serializing_if = "Option::is_none")] - pub priority: Option, - - /// Worker cost factor (optional, default: 1.0) - #[serde(skip_serializing_if = "Option::is_none")] - pub cost: Option, - - /// Worker type (optional: "regular", "prefill", "decode") - #[serde(skip_serializing_if = "Option::is_none")] - pub worker_type: Option, - - /// Bootstrap port for prefill workers (optional) - #[serde(skip_serializing_if = "Option::is_none")] - pub bootstrap_port: Option, - - /// Runtime type (optional: "sglang", "vllm", default: "sglang") - /// Only relevant for gRPC workers - #[serde(skip_serializing_if = "Option::is_none")] - pub runtime: Option, - - // gRPC-specific configuration (optional, ignored in HTTP mode) - /// Tokenizer path for gRPC mode - #[serde(skip_serializing_if = "Option::is_none")] - pub tokenizer_path: Option, - - /// Reasoning parser type for gRPC mode - #[serde(skip_serializing_if = "Option::is_none")] - pub reasoning_parser: Option, - - /// Tool parser type for gRPC mode - #[serde(skip_serializing_if = "Option::is_none")] - pub tool_parser: Option, - - /// Chat template for gRPC mode - #[serde(skip_serializing_if = "Option::is_none")] - pub chat_template: Option, - - /// Additional labels (optional) - #[serde(default, skip_serializing_if = "HashMap::is_empty")] - pub labels: HashMap, - - /// Health check timeout in seconds (default: 30) - #[serde(default = "default_health_check_timeout")] - pub health_check_timeout_secs: u64, - - /// Health check interval in seconds (default: 60) - #[serde(default = "default_health_check_interval")] - pub health_check_interval_secs: u64, - - /// Number of successful health checks needed to mark worker as healthy (default: 2) - #[serde(default = "default_health_success_threshold")] - pub health_success_threshold: u32, - - /// Number of failed health checks before marking worker as unhealthy (default: 3) - #[serde(default = "default_health_failure_threshold")] - pub health_failure_threshold: u32, - - /// Disable periodic health checks for this worker (default: false) - #[serde(default)] - pub disable_health_check: bool, - - /// Maximum connection attempts during worker registration (default: 20) - #[serde(default = "default_max_connection_attempts")] - pub max_connection_attempts: u32, - - /// Enable data parallelism aware scheduling (default: false) - #[serde(default)] - pub dp_aware: bool, -} - -// Default value functions for serde -fn default_health_check_timeout() -> u64 { - 30 -} - -fn default_health_check_interval() -> u64 { - 60 -} - -fn default_health_success_threshold() -> u32 { - 2 -} - -fn default_health_failure_threshold() -> u32 { - 3 -} - -fn default_max_connection_attempts() -> u32 { - 20 -} - -/// Worker information for API responses -#[derive(Debug, Clone, Serialize)] -pub struct WorkerInfo { - /// Worker unique identifier - pub id: String, - - /// Worker URL - pub url: String, - - /// Model ID this worker serves - pub model_id: String, - - /// Worker priority - pub priority: u32, - - /// Worker cost factor - pub cost: f32, - - /// Worker type - pub worker_type: String, - - /// Whether the worker is healthy - pub is_healthy: bool, - - /// Current load on the worker - pub load: usize, - - /// Connection mode (http or grpc) - pub connection_mode: String, - - /// Runtime type (sglang or vllm, for gRPC workers) - #[serde(skip_serializing_if = "Option::is_none")] - pub runtime_type: Option, - - // gRPC-specific fields (None for HTTP workers) - #[serde(skip_serializing_if = "Option::is_none")] - pub tokenizer_path: Option, - - #[serde(skip_serializing_if = "Option::is_none")] - pub reasoning_parser: Option, - - #[serde(skip_serializing_if = "Option::is_none")] - pub tool_parser: Option, - - #[serde(skip_serializing_if = "Option::is_none")] - pub chat_template: Option, - - /// Bootstrap port for prefill workers - #[serde(skip_serializing_if = "Option::is_none")] - pub bootstrap_port: Option, - - /// Additional metadata - #[serde(skip_serializing_if = "HashMap::is_empty")] - pub metadata: HashMap, - - /// Whether health checks are disabled for this worker - pub disable_health_check: bool, - - /// Job status for async operations (if available) - #[serde(skip_serializing_if = "Option::is_none")] - pub job_status: Option, -} - -impl WorkerInfo { - /// Create a partial WorkerInfo for pending workers (not yet registered). - /// Used when a worker ID maps to a URL but the worker is still being registered. - pub fn pending(worker_id: &str, url: String, job_status: Option) -> Self { - Self { - id: worker_id.to_string(), - url, - model_id: UNKNOWN_MODEL_ID.to_string(), - priority: 0, - cost: 1.0, - worker_type: UNKNOWN_MODEL_ID.to_string(), - is_healthy: false, - load: 0, - connection_mode: UNKNOWN_MODEL_ID.to_string(), - runtime_type: None, - tokenizer_path: None, - reasoning_parser: None, - tool_parser: None, - chat_template: None, - bootstrap_port: None, - metadata: HashMap::new(), - disable_health_check: false, - job_status, - } - } -} - -/// Job status for async control plane operations -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct JobStatus { - pub job_type: String, - pub worker_url: String, - pub status: String, - pub message: Option, - pub timestamp: u64, -} - -/// Worker list response -#[derive(Debug, Clone, Serialize)] -pub struct WorkerListResponse { - /// List of workers - pub workers: Vec, - - /// Total count - pub total: usize, - - /// Statistics - pub stats: WorkerStats, -} - -/// Worker statistics -#[derive(Debug, Clone, Serialize)] -pub struct WorkerStats { - pub total_workers: usize, - pub healthy_workers: usize, - pub total_models: usize, - pub total_load: usize, - pub by_type: WorkerTypeStats, -} - -/// Worker statistics by type -#[derive(Debug, Clone, Serialize)] -pub struct WorkerTypeStats { - pub regular: usize, - pub prefill: usize, - pub decode: usize, -} - -/// Worker update request -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct WorkerUpdateRequest { - /// Update priority - #[serde(skip_serializing_if = "Option::is_none")] - pub priority: Option, - - /// Update cost - #[serde(skip_serializing_if = "Option::is_none")] - pub cost: Option, - - /// Update labels - #[serde(skip_serializing_if = "Option::is_none")] - pub labels: Option>, - - /// Update API key (for key rotation) - #[serde(skip_serializing_if = "Option::is_none")] - pub api_key: Option, - - /// Update health check timeout in seconds - #[serde(skip_serializing_if = "Option::is_none")] - pub health_check_timeout_secs: Option, - - /// Update health check interval in seconds - #[serde(skip_serializing_if = "Option::is_none")] - pub health_check_interval_secs: Option, - - /// Update health success threshold - #[serde(skip_serializing_if = "Option::is_none")] - pub health_success_threshold: Option, - - /// Update health failure threshold - #[serde(skip_serializing_if = "Option::is_none")] - pub health_failure_threshold: Option, - - /// Disable periodic health checks for this worker - #[serde(skip_serializing_if = "Option::is_none")] - pub disable_health_check: Option, -} - -/// Generic API response -#[derive(Debug, Clone, Serialize)] -pub struct WorkerApiResponse { - pub success: bool, - pub message: String, - - #[serde(skip_serializing_if = "Option::is_none")] - pub worker: Option, -} - -/// Error response -#[derive(Debug, Clone, Serialize)] -pub struct WorkerErrorResponse { - pub error: String, - pub code: String, -} - -/// Server info response from /get_server_info endpoint -#[derive(Debug, Clone, Deserialize)] -pub struct ServerInfo { - #[serde(skip_serializing_if = "Option::is_none")] - pub model_id: Option, - - #[serde(skip_serializing_if = "Option::is_none")] - pub model_path: Option, - - #[serde(skip_serializing_if = "Option::is_none")] - pub priority: Option, - - #[serde(skip_serializing_if = "Option::is_none")] - pub cost: Option, - - #[serde(skip_serializing_if = "Option::is_none")] - pub worker_type: Option, - - // gRPC-specific - #[serde(skip_serializing_if = "Option::is_none")] - pub tokenizer_path: Option, - - #[serde(skip_serializing_if = "Option::is_none")] - pub reasoning_parser: Option, - - #[serde(skip_serializing_if = "Option::is_none")] - pub tool_parser: Option, - - #[serde(skip_serializing_if = "Option::is_none")] - pub chat_template: Option, -} - -/// Result from flush cache operations across workers -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct FlushCacheResult { - /// URLs of workers where cache flush succeeded - pub successful: Vec, - /// URLs and error messages for workers where cache flush failed - pub failed: Vec<(String, String)>, - /// Total number of workers attempted - pub total_workers: usize, - /// Number of HTTP workers (gRPC workers don't support flush cache) - pub http_workers: usize, - /// Human-readable summary message - pub message: String, -} - -/// Result from getting worker loads -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct WorkerLoadsResult { - /// Worker URL and load pairs - pub loads: Vec, - /// Total number of workers - pub total_workers: usize, - /// Number of workers with successful load fetches - pub successful: usize, - /// Number of workers with failed load fetches - pub failed: usize, -} - -/// Individual worker load information -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct WorkerLoadInfo { - /// Worker URL - pub worker: String, - /// Worker type (regular, prefill, decode) - #[serde(skip_serializing_if = "Option::is_none")] - pub worker_type: Option, - /// Current load (-1 indicates failure to fetch) - pub load: isize, -}